diff --git a/.github/workflows/mutation.yml b/.github/workflows/mutation.yml index f54ec7a6d..de4bd4669 100644 --- a/.github/workflows/mutation.yml +++ b/.github/workflows/mutation.yml @@ -59,6 +59,8 @@ jobs: mutate: "src/streaming/**/*.ts,!src/**/tests/**,!src/**/*.test.ts,!src/**/*.test.tsx,!src/**/*.d.ts" - name: hooks mutate: "src/hooks/**/*.ts,src/hooks/**/*.tsx,!src/**/tests/**,!src/**/*.test.ts,!src/**/*.test.tsx,!src/**/*.d.ts" + - name: biometrics + mutate: "src/biometrics/**/*.ts,!src/**/tests/**,!src/**/*.test.ts,!src/**/*.test.tsx,!src/**/*.d.ts" - name: db mutate: "src/db/**/*.ts,!src/**/tests/**,!src/**/*.test.ts,!src/**/*.test.tsx,!src/**/*.d.ts" name: mutate (${{ matrix.shard.name }}) diff --git a/.github/workflows/python-modules.yml b/.github/workflows/python-modules.yml index 426e2c237..d8997abfe 100644 --- a/.github/workflows/python-modules.yml +++ b/.github/workflows/python-modules.yml @@ -5,17 +5,23 @@ on: branches: [dev] paths: - 'modules/**/*.py' + - 'modules/common/biometrics/*.json' + - 'modules/piezo-processor/fixtures/*.json' - 'modules/**/pyproject.toml' - 'modules/**/uv.lock' - 'modules/.coveragerc' + - 'scripts/**/*.py' - '.github/workflows/python-modules.yml' pull_request: types: [opened, synchronize, reopened] paths: - 'modules/**/*.py' + - 'modules/common/biometrics/*.json' + - 'modules/piezo-processor/fixtures/*.json' - 'modules/**/pyproject.toml' - 'modules/**/uv.lock' - 'modules/.coveragerc' + - 'scripts/**/*.py' - '.github/workflows/python-modules.yml' workflow_dispatch: @@ -48,9 +54,9 @@ jobs: workdir: modules target: common cov_source: common - # cbor2 is needed to decode the real NATS capture fixtures through - # NatsFollower's decode path (test_nats_follower). - deps: "cbor2" + # Build-identity tests inspect installed NumPy/SciPy metadata; + # use the piezo module's locked environment, including cbor2. + deps: "" - name: cover-buttons workdir: modules/cover-buttons target: . @@ -70,12 +76,20 @@ jobs: workdir: modules/piezo-processor target: . cov_source: . + cov_extra: "--cov=../common/biometrics" deps: "numpy scipy" - name: sleep-detector workdir: modules/sleep-detector target: . cov_source: . deps: "" + # Stdlib-only repo scripts; tests load them by path from + # scripts/tests/. + - name: scripts + workdir: scripts + target: tests + cov_source: . + deps: "" steps: - name: Checkout @@ -86,7 +100,7 @@ jobs: with: # Pod runtime is pinned to 3.10 by every module's pyproject # (`requires-python = ">=3.9,<3.11"`). Match it in CI. - python-version: '3.14' + python-version: '3.10' - name: Install uv uses: astral-sh/setup-uv@v4 @@ -96,11 +110,16 @@ jobs: - name: Install test deps working-directory: ${{ matrix.module.workdir }} run: | - if [ -f uv.lock ]; then + if [ "${{ matrix.module.name }}" = "common" ]; then + # Keep common's cwd/import layout while using the numerical lock. + UV_PROJECT_ENVIRONMENT="$PWD/.venv" uv sync --project piezo-processor --locked --no-install-project + elif [ -f uv.lock ]; then uv sync --locked --no-install-project else uv venv .venv - uv pip install --python .venv/bin/python ${{ matrix.module.deps }} + if [ -n "${{ matrix.module.deps }}" ]; then + uv pip install --python .venv/bin/python ${{ matrix.module.deps }} + fi fi # shellcheck source=/dev/null . .venv/bin/activate @@ -114,12 +133,13 @@ jobs: pytest ${{ matrix.module.target }} \ --cov-config=${{ github.workspace }}/modules/.coveragerc \ --cov=${{ matrix.module.cov_source }} \ + ${{ matrix.module.cov_extra }} \ --cov-report=xml:coverage.xml \ --junit-xml=junit.xml \ -o junit_family=legacy - name: Upload coverage to Codecov - if: ${{ !cancelled() }} + if: ${{ github.repository == 'sleepypod/core' && !cancelled() }} uses: codecov/codecov-action@v5 with: use_oidc: true @@ -132,7 +152,7 @@ jobs: working-directory: ${{ matrix.module.workdir }} - name: Upload test results to Codecov - if: ${{ !cancelled() }} + if: ${{ github.repository == 'sleepypod/core' && !cancelled() }} uses: codecov/codecov-action@v5 with: report_type: test_results diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index eba7cbba0..728688b48 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -29,6 +29,7 @@ jobs: - { name: "Lint", cmd: "CHANGED=$(git diff --name-only --diff-filter=d origin/${BASE_REF#refs/heads/}...HEAD -- '*.ts' '*.tsx' | head -200); if [ -n \"$CHANGED\" ]; then echo \"$CHANGED\" | xargs -d '\\n' pnpm eslint --; else echo 'No TS/TSX files changed'; fi" } - { name: "Typecheck", cmd: "pnpm tsc" } - { name: "Unit Tests", cmd: "pnpm test run --coverage --passWithNoTests" } + - { name: "Browser Tests", cmd: "NODE_ENV=production CI=1 pnpm build && pnpm test:browser" } steps: - name: Checkout @@ -51,6 +52,27 @@ jobs: - name: Install Dependencies run: pnpm install --frozen-lockfile + - name: Setup Python for runtime integration + if: matrix.task.name == 'Unit Tests' || matrix.task.name == 'Browser Tests' + uses: actions/setup-python@v6 + with: + python-version: '3.10' + + - name: Setup uv for runtime integration + if: matrix.task.name == 'Unit Tests' || matrix.task.name == 'Browser Tests' + uses: astral-sh/setup-uv@v4 + with: + enable-cache: true + + - name: Install locked runtime integration dependencies + if: matrix.task.name == 'Unit Tests' || matrix.task.name == 'Browser Tests' + working-directory: modules/piezo-processor + run: uv sync --locked --no-install-project + + - name: Install browser + if: matrix.task.name == 'Browser Tests' + run: pnpm exec playwright install --with-deps chromium + - name: Cache Assets (Vitest) id: asset-cache uses: actions/cache@v5 @@ -68,7 +90,7 @@ jobs: BASE_REF: ${{ github.event.pull_request.base.ref || github.event.merge_group.base_ref }} - name: Upload coverage reports to Codecov - if: matrix.task.name == 'Unit Tests' && !cancelled() + if: github.repository == 'sleepypod/core' && matrix.task.name == 'Unit Tests' && !cancelled() uses: codecov/codecov-action@v5 with: use_oidc: true @@ -79,7 +101,7 @@ jobs: flags: javascript - name: Upload test results to Codecov - if: matrix.task.name == 'Unit Tests' && !cancelled() + if: github.repository == 'sleepypod/core' && matrix.task.name == 'Unit Tests' && !cancelled() uses: codecov/codecov-action@v5 with: report_type: test_results @@ -89,4 +111,12 @@ jobs: fail_ci_if_error: false slug: sleepypod/core files: test-results/junit.xml - flags: javascript \ No newline at end of file + flags: javascript + + - name: Upload browser evidence + if: matrix.task.name == 'Browser Tests' && !cancelled() + uses: actions/upload-artifact@v6 + with: + name: calibration-browser-results + path: test-results/browser + retention-days: 7 diff --git a/app/[lang]/schedule/[curve]/page.tsx b/app/[lang]/schedule/[curve]/page.tsx index 9a740f51d..6027b3864 100644 --- a/app/[lang]/schedule/[curve]/page.tsx +++ b/app/[lang]/schedule/[curve]/page.tsx @@ -8,15 +8,16 @@ const SIDES: SideSelection[] = ['left', 'right', 'both'] export default async function ScheduleCurve({ params, searchParams }: { params: Promise<{ curve: string }> - searchParams: Promise<{ side?: string | string[], from?: string | string[] }> + searchParams: Promise<{ side?: string | string[], from?: string | string[], saved?: string | string[] }> }) { const { curve } = await params - const { side, from } = await searchParams + const { side, from, saved } = await searchParams return ( s === side)} fromBoth={from === 'both'} + savedCurve={typeof saved === 'string' ? saved : undefined} /> ) } diff --git a/app/api/biometrics/watch-upload/route.test.ts b/app/api/biometrics/watch-upload/route.test.ts new file mode 100644 index 000000000..888b8b889 --- /dev/null +++ b/app/api/biometrics/watch-upload/route.test.ts @@ -0,0 +1,157 @@ +// @vitest-environment node +import { createServer, type Server } from 'node:http' +import { mkdtemp, readdir, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterAll, beforeAll, expect, it, vi } from 'vitest' +import { POST } from './route' + +let directory: string +let server: Server +const seen = new Set() +const requests: Array<{ side: string, source: string }> = [] +beforeAll(async () => { + directory = await mkdtemp(join(tmpdir(), 'watch-upload-test-')) + vi.stubEnv('WATCH_UPLOAD_DIR', directory) + server = createServer(async (req, res) => { + let body = '' + for await (const chunk of req) body += chunk + const data = JSON.parse(body) + requests.push(data) + let written = 0 + for (const sample of data.samples ?? []) { + if (!seen.has(sample.timestamp)) { + seen.add(sample.timestamp) + written++ + } + } + res.setHeader('Content-Type', 'application/json') + res.end(JSON.stringify({ written })) + }) + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)) + const address = server.address() + if (!address || typeof address === 'string') throw new Error('missing test port') + vi.stubEnv('PORT', String(address.port)) +}) +afterAll(async () => { + await new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve())) + await rm(directory, { recursive: true, force: true }) + vi.unstubAllEnvs() +}) + +it('streams XML through the real importer, routes to the selected side, dedupes, and removes the file', async () => { + const date = new Date().toISOString().replace('T', ' ').replace(/\.\d+Z$/, ' +0000') + const xml = `` + const request = () => new Request('http://pod/api/biometrics/watch-upload?side=right&days=7', { method: 'POST', body: xml }) + const response = await POST(request()) + expect(response.status).toBe(200) + expect(await response.json()).toMatchObject({ vitals: 1, stages: 0, matchedVitals: 1 }) + expect(requests[0]).toMatchObject({ side: 'right', source: 'apple_watch' }) + expect(await (await POST(request())).json()).toMatchObject({ vitals: 0, matchedVitals: 1 }) + expect(await readdir(directory)).toEqual([]) +}) + +it('rejects malformed files and cleans up after parser failure', async () => { + const response = await POST(new Request('http://pod/api/biometrics/watch-upload?side=left', { method: 'POST', body: ' { + const upload = new AbortController() + const importer = new AbortController() + let staged = false + const timeout = vi.spyOn(AbortSignal, 'timeout') + .mockReturnValueOnce(upload.signal) + .mockImplementationOnce(() => { + expect(staged).toBe(true) + upload.abort(new DOMException('Upload budget expired', 'TimeoutError')) + return importer.signal + }) + try { + const body = new ReadableStream({ + pull(controller) { + controller.enqueue(new TextEncoder().encode('')) + staged = true + controller.close() + }, + }) + const request = new Request('http://pod/api/biometrics/watch-upload?side=left', { method: 'POST', body, duplex: 'half' } as RequestInit) + const response = await POST(request) + expect(response.status).toBe(200) + expect(timeout.mock.calls.map(call => call[0])).toEqual([30 * 60_000, 10 * 60_000]) + expect(await readdir(directory)).toEqual([]) + } + finally { timeout.mockRestore() } +}) + +it('accepts a same-origin browser upload even though Next reports the 0.0.0.0 bind address in request.url', async () => { + const response = await POST(new Request('http://0.0.0.0:3000/api/biometrics/watch-upload?side=left', { + method: 'POST', headers: { origin: 'http://192.168.1.88:3000', host: '192.168.1.88:3000' }, body: '', + })) + expect(response.status).toBe(200) + expect(await readdir(directory)).toEqual([]) +}) + +it('rejects cross-origin requests, missing side, and oversized bodies before import', async () => { + for (const origin of ['http://evil', 'http://192.168.1.88:8080', 'null']) { + const response = await POST(new Request('http://0.0.0.0:3000/api/biometrics/watch-upload?side=left', { method: 'POST', headers: { origin, host: '192.168.1.88:3000' }, body: 'x' })) + expect(response.status).toBe(403) + } + expect((await POST(new Request('http://pod/api/biometrics/watch-upload', { method: 'POST', body: 'x' }))).status).toBe(400) + expect((await POST(new Request('http://pod/api/biometrics/watch-upload?side=left', { method: 'POST', headers: { 'content-length': String(513 * 1024 * 1024) }, body: 'x' }))).status).toBe(413) +}) + +it('still honors a client disconnect when switching to the import budget', async () => { + const client = new AbortController() + const timeout = vi.spyOn(AbortSignal, 'timeout') + .mockReturnValueOnce(new AbortController().signal) + .mockImplementationOnce(() => { + client.abort() + return new AbortController().signal + }) + try { + const response = await POST(new Request('http://pod/api/biometrics/watch-upload?side=left', { + method: 'POST', body: '', signal: client.signal, + })) + expect(response.status).toBe(408) + expect(await readdir(directory)).toEqual([]) + } + finally { timeout.mockRestore() } +}) + +it('rejects missing and empty bodies without retaining staged files', async () => { + for (const body of [undefined, '']) { + const response = await POST(new Request('http://pod/api/biometrics/watch-upload?side=left', { method: 'POST', body })) + expect(response.status).toBe(400) + expect(await readdir(directory)).toEqual([]) + } +}) + +it('enforces the streaming size limit even without a content-length header', async () => { + const chunk = new Uint8Array(1) + Object.defineProperty(chunk, 'byteLength', { value: 513 * 1024 * 1024 }) + const body = new ReadableStream({ + start(controller) { + controller.enqueue(chunk) + controller.close() + }, + }) + const response = await POST(new Request('http://pod/api/biometrics/watch-upload?side=left', { method: 'POST', body, duplex: 'half' } as RequestInit)) + expect(response.status).toBe(413) + expect(await response.json()).toEqual({ error: 'Upload exceeds 512 MiB' }) + expect(await readdir(directory)).toEqual([]) +}) + +it('rejects concurrent uploads and releases the slot after staging is cancelled', async () => { + const client = new AbortController() + const response = POST(new Request('http://pod/api/biometrics/watch-upload?side=left', { + method: 'POST', body: new ReadableStream(), duplex: 'half', signal: client.signal, + } as RequestInit)) + const competing = await POST(new Request('http://pod/api/biometrics/watch-upload?side=right', { method: 'POST', body: '' })) + expect(competing.status).toBe(409) + client.abort() + expect((await response).status).toBe(408) + expect(await readdir(directory)).toEqual([]) + expect((await POST(new Request('http://pod/api/biometrics/watch-upload?side=left', { method: 'POST', body: '' }))).status).toBe(200) +}) diff --git a/app/api/biometrics/watch-upload/route.ts b/app/api/biometrics/watch-upload/route.ts new file mode 100644 index 000000000..c5658c0d2 --- /dev/null +++ b/app/api/biometrics/watch-upload/route.ts @@ -0,0 +1,97 @@ +import { spawn } from 'node:child_process' +import { mkdir, mkdtemp, open, rm } from 'node:fs/promises' +import path from 'node:path' +import { z } from 'zod' +import { WATCH_IMPORT_TIMEOUT_MS, WATCH_UPLOAD_TIMEOUT_MS } from '@/src/utils/watchUploadLimits' + +export const runtime = 'nodejs' +export const dynamic = 'force-dynamic' +const MAX_BYTES = 512 * 1024 * 1024 +const options = z.object({ side: z.enum(['left', 'right']), days: z.coerce.number().int().min(1).max(90).default(30) }) +const resultSchema = z.object({ stages: z.number(), vitals: z.number(), rejectedStageBatches: z.number(), matchedStages: z.number(), matchedVitals: z.number() }) +const shared = globalThis as typeof globalThis & { watchUploadBusy?: boolean } + +/** + * Next builds request.url from the bind address (http://0.0.0.0:3000 on the pod), so a browser's Origin + * never matches it. Compare against the Host the browser actually sent, as /api/mcp does. + */ +function isSameOrigin(request: Request): boolean { + const origin = request.headers.get('origin') + if (origin === null) return true + try { + return new URL(origin).host === request.headers.get('host') + } + catch { + return false + } +} + +/** Raw export.zip/export.xml body, streamed to flash. Uses the same importer as the CLI. */ +export async function POST(request: Request): Promise { + if (!isSameOrigin(request)) return Response.json({ error: 'Cross-origin upload is not allowed' }, { status: 403 }) + const parsed = options.safeParse(Object.fromEntries(new URL(request.url).searchParams)) + if (!parsed.success) return Response.json({ error: 'Specify side=left|right and days=1..90' }, { status: 400 }) + if (!request.body) return Response.json({ error: 'Send export.zip or export.xml as the request body' }, { status: 400 }) + if (Number(request.headers.get('content-length')) > MAX_BYTES) return Response.json({ error: 'Upload exceeds 512 MiB' }, { status: 413 }) + if (shared.watchUploadBusy) return Response.json({ error: 'Another Watch import is running; retry when it finishes' }, { status: 409 }) + shared.watchUploadBusy = true + let signal = AbortSignal.any([request.signal, AbortSignal.timeout(WATCH_UPLOAD_TIMEOUT_MS)]) + let directory: string | undefined + let tooLarge = false + try { + // /tmp is RAM-backed on the pod. Large Health exports belong on persistent storage. + const staging = process.env.WATCH_UPLOAD_DIR ?? '/persistent/sleepypod-data/watch-upload' + await mkdir(staging, { recursive: true }) + directory = await mkdtemp(path.join(staging, 'import-')) + const filename = path.join(directory, 'health-export') + const file = await open(filename, 'wx', 0o600) + let bytes = 0 + try { + await request.body.pipeTo(new WritableStream({ + async write(chunk) { + bytes += chunk.byteLength + if (bytes > MAX_BYTES) { + tooLarge = true + throw new Error('Upload exceeds 512 MiB') + } + await file.writeFile(chunk) + }, + }), { signal }) + } + finally { + await file.close() + } + if (bytes === 0) return Response.json({ error: 'The uploaded file is empty' }, { status: 400 }) + signal = AbortSignal.any([request.signal, AbortSignal.timeout(WATCH_IMPORT_TIMEOUT_MS)]) + const { side, days } = parsed.data + const stdout = await new Promise((resolve, reject) => { + const child = spawn('python3', [path.resolve('scripts/import-watch-health.py'), filename, + '--pod', `http://127.0.0.1:${process.env.PORT ?? '3000'}`, '--side', side, '--days', String(days), '--json'], { signal }) + let output = '' + child.stdout.on('data', (chunk: Buffer) => { + output += chunk.toString() + }) + // Drain errors without disclosing Health contents or internal paths to callers. + child.stderr.resume() + let processError: Error | undefined + child.on('error', (error) => { + processError = error + }) + child.on('close', code => code === 0 ? resolve(output) : reject(processError ?? new Error('Import failed'))) + }) + return Response.json(resultSchema.parse(JSON.parse(stdout))) + } + catch { + return Response.json({ error: tooLarge + ? 'Upload exceeds 512 MiB' + : signal.aborted + ? 'Import interrupted. Some batches may have been saved; retrying the same export is safe.' + : 'Import failed. Use an Apple Health export.zip or export.xml. Some batches may have been saved; retrying is safe.' }, { status: tooLarge ? 413 : signal.aborted ? 408 : 422 }) + } + finally { + try { + if (directory) await rm(directory, { recursive: true, force: true }) + } + finally { shared.watchUploadBusy = false } + } +} diff --git a/app/api/raw/[filename]/route.ts b/app/api/raw/[filename]/route.ts index 0eb1011ed..12b481cf5 100644 --- a/app/api/raw/[filename]/route.ts +++ b/app/api/raw/[filename]/route.ts @@ -20,26 +20,26 @@ export async function GET( } const resolved = path.resolve(RAW_DIR, filename) - if (!resolved.startsWith(path.resolve(RAW_DIR))) { + if (!resolved.startsWith(path.resolve(/* turbopackIgnore: true */ RAW_DIR))) { return NextResponse.json({ error: 'Path traversal detected' }, { status: 400 }) } try { // Reject symlinks to prevent escaping RAW_DIR - const lstats = await lstat(resolved) + const lstats = await lstat(/* turbopackIgnore: true */ resolved) if (lstats.isSymbolicLink()) { return NextResponse.json({ error: 'Path traversal detected' }, { status: 400 }) } // Verify canonical path is still inside RAW_DIR - const canonicalFile = await realpath(resolved) - const canonicalDir = await realpath(RAW_DIR) + const canonicalFile = await realpath(/* turbopackIgnore: true */ resolved) + const canonicalDir = await realpath(/* turbopackIgnore: true */ RAW_DIR) if (!canonicalFile.startsWith(canonicalDir)) { return NextResponse.json({ error: 'Path traversal detected' }, { status: 400 }) } - const s = await stat(resolved) - const stream = createReadStream(resolved) + const s = await stat(/* turbopackIgnore: true */ resolved) + const stream = createReadStream(/* turbopackIgnore: true */ resolved) const webStream = Readable.toWeb(stream) as ReadableStream return new Response(webStream, { diff --git a/app/api/raw/[filename]/tests/route.test.ts b/app/api/raw/[filename]/tests/route.test.ts new file mode 100644 index 000000000..2a57dd7ac --- /dev/null +++ b/app/api/raw/[filename]/tests/route.test.ts @@ -0,0 +1,43 @@ +// @vitest-environment node +import { Readable } from 'node:stream' +import { beforeEach, expect, it, vi } from 'vitest' +import { GET } from '../route' +const fs = vi.hoisted(() => ({ lstat: vi.fn(), realpath: vi.fn(), stat: vi.fn(), createReadStream: vi.fn() })) +vi.mock('node:fs/promises', () => ({ lstat: fs.lstat, realpath: fs.realpath, stat: fs.stat })) +vi.mock('node:fs', () => ({ createReadStream: fs.createReadStream })) +const request = (filename = 'capture.RAW') => GET(new Request('http://localhost/api/raw/' + filename), { params: Promise.resolve({ filename }) }) +beforeEach(() => { + vi.resetAllMocks() + fs.lstat.mockResolvedValue({ isSymbolicLink: () => false }) + fs.realpath.mockImplementation(async (path: string) => path) + fs.stat.mockResolvedValue({ size: 3 }) + fs.createReadStream.mockImplementation(() => Readable.from([Buffer.from([1, 2, 3])])) +}) +it('streams the original bytes with explicit download metadata', async () => { + const response = await request() + expect(response.status).toBe(200) + expect(response.headers.get('Content-Type')).toBe('application/octet-stream') + expect(response.headers.get('Content-Disposition')).toBe('attachment; filename="capture.RAW"') + expect(response.headers.get('Content-Length')).toBe('3') + expect(new Uint8Array(await response.arrayBuffer())).toEqual(new Uint8Array([1, 2, 3])) +}) +it.each(['../private.RAW', 'nested/capture.RAW', 'capture.txt', 'capture.RAW\n'])('rejects invalid filename %s before filesystem access', async (filename) => { + expect((await request(filename)).status).toBe(400) + expect(fs.lstat).not.toHaveBeenCalled() + expect(fs.createReadStream).not.toHaveBeenCalled() +}) +it('refuses symbolic links and canonical paths outside the configured directory', async () => { + fs.lstat.mockResolvedValueOnce({ isSymbolicLink: () => true }) + expect((await request()).status).toBe(400) + expect(fs.realpath).not.toHaveBeenCalled() + fs.realpath.mockResolvedValueOnce('/outside/capture.RAW').mockResolvedValueOnce('/owned') + expect((await request()).status).toBe(400) + expect(fs.createReadStream).not.toHaveBeenCalled() +}) +it.each([['ENOENT', 404, 'File not found'], ['EACCES', 500, 'Failed to read file']] as const)('sanitizes %s filesystem failures', async (code, status, error) => { + fs.lstat.mockRejectedValue(Object.assign(new Error('/private/source'), { code })) + const response = await request() + expect(response.status).toBe(status) + expect(await response.json()).toEqual({ error }) + expect(fs.createReadStream).not.toHaveBeenCalled() +}) diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index f369dec14..eb156a87c 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -227,6 +227,17 @@ sp-logs # tail service logs (journalctl -f) sp-update [branch] # self-update from GitHub (handles iptables) ``` +## Accessing the Pod by Hostname + +The API has no authentication, so the server rejects any `Host` header that isn't a pod address (HTTP 403). This stops DNS-rebinding pages. These work out of the box: IP addresses (`http://192.168.1.88:3000`), `localhost`, single-label names (`eight-pod`), and names ending in `.local`, `.lan`, `.home.arpa`, `.internal` or `.localhost`. To reach the pod through another name, such as a Tailscale MagicDNS name or a reverse proxy, list it in `SLEEPYPOD_ALLOWED_HOSTS`. Entries are comma-separated, and each is an exact host or `*.suffix`: + +```bash +sudo mkdir -p /etc/systemd/system/sleepypod.service.d +printf '[Service]\nEnvironment="SLEEPYPOD_ALLOWED_HOSTS=pod.tail1234.ts.net,*.example.com"\n' \ + | sudo tee /etc/systemd/system/sleepypod.service.d/allowed-hosts.conf +sudo systemctl daemon-reload && sp-restart +``` + ## Network Security (Telemetry Blocking) The pod runs two stock processes that continuously attempt to phone home: @@ -293,3 +304,19 @@ sudo bash /tmp/sleepypod-install --purge-free-sleep It also flushes the running iptables and ip6tables rules; the install then reconfigures iptables itself. Without the flag, run the cleanup block the installer prints, reboot, and re-run the installer. free-sleep's `unblock_internet_access.sh` is not a substitute: it flushes runtime rules only and leaves the persisted files in place. Run `sp-uninstall` if you want to revert (the script flushes iptables and removes sleepypod's units); the purge is not undone, so reinstall free-sleep from its own installer. + + +### Runtime filesystem reads and standalone tracing + +Pod data (RAW captures, saved alarm state, HomeKit identity seeds, archive-push +configuration, procfs and storage-accounting paths) is read on the destination +pod. These dynamic filesystem reads use `turbopackIgnore` call annotations so +Next does not infer that the build machine's whole checkout belongs in the +standalone output. The annotations affect packaging only; existing validation, +file access and error handling still run on the pod. + +The biometrics worker location probes are also excluded from automatic tracing: +`scripts/build/biometrics-worker.mjs` explicitly generates `worker.cjs`, +`coordinator.cjs` and `shadow-host.cjs` in the standalone bundle after Next builds. +Keep that packaging step and its dependency-graph checks when changing tracing. +Do not apply these annotations to shipped assets that need inclusion in a release. diff --git a/docs/adr/0023-autopilot-reactive-automations.md b/docs/adr/0023-autopilot-reactive-automations.md index a38755ddb..c031b63f2 100644 --- a/docs/adr/0023-autopilot-reactive-automations.md +++ b/docs/adr/0023-autopilot-reactive-automations.md @@ -246,10 +246,13 @@ the engine — so shipping it in P0 is premature surface. aggregates (the "P2" capability set). The engine supports them; what gates them firing on live data is the **signal-source wiring**, not the engine. - The engine handles **numeric** signals only. The builder's catalog is - therefore the numeric/backtestable subset; enum/bool signals (sleep stage, - occupancy) in the full catalog read `undefined` → "skip" until their sources - are wired. This is the safe degradation, but it means the shipped builder is - intentionally a subset of the documented catalog. + therefore the numeric/backtestable subset; enum/bool signals (sleep stage) + in the full catalog read `undefined` → "skip" until their sources are + wired. This is the safe degradation, but it means the shipped builder is + intentionally a subset of the documented catalog. Occupancy is wired as the + numeric `{side}.presence` (1 = person, 0 = empty or a light load such as a + pet; `undefined` while unsensable) and `{side}.presence.zones` — see ADR + 0025. Both are live-only: the classified signal has no history table. ### Open questions diff --git a/docs/adr/0025-presence-auto-on.md b/docs/adr/0025-presence-auto-on.md new file mode 100644 index 000000000..3e27ba7c0 --- /dev/null +++ b/docs/adr/0025-presence-auto-on.md @@ -0,0 +1,99 @@ +# ADR 0025: Presence auto-on and person-vs-pet occupancy classification + +**Status:** Accepted, shipped as experimental +**Date:** 2026-10-11 + +## Context + +Auto-off has existed since `autoOffWatcher` (per-side `autoOffEnabled` / +`autoOffMinutes`): once the shared virtual occupancy sensor +(`src/lib/occupancy.ts`) reads a side as reliably empty for the timeout, the +side is powered off. The mirror feature was missing: a sleeper who gets into a +cold bed before their schedule starts has to reach for the app. + +Both directions share one failure mode. The occupancy sensor ORs a movement +window with a calibrated capSense2 level signal, and the level signal is the +sum of three zone deviations (head, torso, legs) over one threshold. A cat on +the pillow deviates one zone hard enough to clear the sum, so it reads as +"in bed": it would hold auto-off open indefinitely today, and would trip any +naive auto-on. + +The question was whether auto-on belongs in Settings (a toggle), in the +automation engine (a rule), or both. + +## Decision + +1. **Classify in the sensor, not in the consumers.** `getOccupancy()` gains + `level.zoneDeviations`, `level.zones` (zones whose deviation exceeds a third + of the calibrated threshold), `vitals` (latest piezo HR/BR within 5 min, + with a resting-human-range verdict) and `classification`: + - `person`: occupied and ≥ 2 zones lit, or human-range vitals, or movement + only with no level signal to say otherwise. + - `light`: occupied, level signal evaluable, single-zone load, no human + vitals. A pet or an object. + - `none`: not occupied. + + Vitals only ever upgrade to `person`. They drop out whenever the sleeper + moves, so a missing or odd reading must never demote a person. + `occupied` and `available` keep their meaning for HomeKit and the API. + +2. **Auto-off treats `light` as empty** when the level signal is evaluable. + Movement-only occupancy with an unsensable level still counts as occupied, + as before. + +3. **Auto-on is a per-side setting, shipped experimental**, implemented in + `src/services/autoOnWatcher.ts` beside the auto-off watcher rather than as + an automation rule. It has to coordinate with away mode, run-once + sessions, the pump-stall guard, the alarm's held power-off and the + controller's baseline selection; the scheduler's power-on paths already + have that plumbing and an automation rule would re-implement it badly. + Guardrails: + - fires only on `classification === 'person'`, sustained for + `autoOnMinutes` (default 5); + - only inside `autoOnStart`..`autoOnEnd` local time (default 20:00–08:00, + wraps midnight, equal times = any time) so a nap does not fire unless + the user widens it; + - a side turned off while a person is still in it (manual, scheduled, or + the global cap) suppresses auto-on until the bed has read non-person for + the dwell, so the bed cannot bounce straight back on; + - unsensable presence resets the dwell and never fires. + + There is no separate "auto-on temperature". `powerOnLocked` resolves the + setpoint from the controller baseline (an active or upcoming schedule), + then the last target the watcher saw while powered, then 75 °F. Auto-on is + an early schedule start, not another preference to keep in sync. + +4. **Expose presence to the automation engine as well.** `{side}.presence` + (1/0, `undefined` while unsensable) and `{side}.presence.zones` join the + signal catalog and the builder, live-only. Power users can build "when + presence changes, set power on at 72 °F" rules; the toggle remains the + default UX. + +## Alternatives considered + +- **A pet preference ("I have a pet").** Rejected: it would still need the + zone/vitals evidence to act on, and the classification is useful without + it (auto-off, the Presence card, HomeKit consumers). +- **Automation rule only.** Rejected for the coordination reasons above, and + because the engine deliberately handles numeric signals with no notion of + suppression after an intentional off. +- **Ask for an auto-on temperature.** Rejected: duplicates the schedule's + setpoint and drifts from it. + +## Consequences + +- Classification thresholds (`PERSON_MIN_ZONES = 2`, zone share = threshold/3, + HR 40–110, BR 8–22) are first guesses tuned against Pod 5 capSense2 frames. + A large dog spanning two zones will read as a person. The Presence card + shows "light load" so users can see what the sensor decided. +- Pod 3 (legacy scalar capSense) has no level signal, so `zones` is null and + movement-only occupancy classifies as `person`; auto-on stands down there + because presence is unsensable. +- Auto-off behaviour changes for pets: a cat left on the bed no longer keeps + it on. + +## References + +- `src/lib/occupancy.ts`, `src/services/autoOnWatcher.ts`, + `src/services/autoOffWatcher.ts`, `src/automation/signals.biometrics.ts` +- ADR 0023 (automation signal catalog) diff --git a/docs/biometrics-backtest-jobs.md b/docs/biometrics-backtest-jobs.md new file mode 100644 index 000000000..4bfd7c57d --- /dev/null +++ b/docs/biometrics-backtest-jobs.md @@ -0,0 +1,262 @@ +# Durable biometric backtest jobs + +`BacktestJobStore` persists experiments, immutable evaluation inputs, leases, +progress, cancellation, results, and holdout exposure in `biometrics.db`. +It is the job repository; execution, API/UI integration, and production worker +resource validation are separate parts of the calibration implementation. + +Enqueue pins the retained dataset hash, protocol hash, exact resolved +configurations, engine manifest hash, and selected split. Duplicate idempotency +keys return the original job, while changed requests fail. Hash or engine +mismatches are rejected. Configuration snapshots are limited to 64 KiB. + +Only one unexpired running or cancellation-pending job can hold the worker +slot. Claim increments both the attempt and fencing token. Every heartbeat, +completion, failure, and cancellation acknowledgment verifies owner, token, +state, and expiry under a short SQLite transaction. An expired worker cannot +publish after replacement. Recovery marks abandoned work interrupted and +allows at most three attempts; abandoned cancellations become cancelled. + +Cancellation of queued work is immediate. Running work exposes a cancellation +request through heartbeat, and cannot publish a successful result afterward. +The worker supervisor must terminate unresponsive execution after its grace +period. The repository itself does not execute code or control processes. + +A result and its terminal job state commit atomically. Summaries are bounded +to 16 MiB each and 64 MiB retained total; job and experiment histories each stop at 1000 +entries. These repository bounds do not replace worker CPU, memory, wall-time, +free-space, or live-ingest safeguards. They require target-pod measurement +before enabling production replay. + +Holdout exposure is recorded on enqueue, conservatively including jobs later +cancelled or failed. It follows physical side and overlapping recorded time +ranges across datasets and experiments. Creating another dataset or changing +its asserted subject name does not erase exposure. Repeated access is labeled +`reused_holdout_is_validation`. Idempotent retries of the same enqueue do not +count twice. Future promotion must recheck current exposure, not rely solely +on the count captured in an older report. + +The current evaluation protocol is explicitly descriptive. This repository +does not award validated improvement; that requires the complete evaluator, +reference support, coverage-aware policy, independent nights, and operational +validation. No backtest state activates a profile. + +## Shipped processed-data worker + +`evaluator.ts` executes comparison and stage replay over the frozen selected +split. Overlapping session rows are deduplicated by original ID; overlapping +windows merge and disconnected sessions are classified separately. It calls +the existing retrospective classifier with the exact resolved rules and the +protocol's explicitly assumed calibration quality. Default/current snapshots +with identical hashes execute once. Fixed clock offsets also apply to stage +scoring. Original unshifted stage traces remain available in the report. + +Numeric comparisons use original stored readings. They do not evaluate a +candidate's waveform-processing parameters. Stage comparisons report +interval-duration agreement, not the chart's older midpoint agreement. +Reports remain descriptive/insufficient-evidence until the promotion policy, +reference support, common-support checks and operational qualification are +complete. Merged replay windows are currently limited to one day, and complete +reports up to 2 MiB remain inline. Larger reports use checksummed files beside +the biometrics database, with a small descriptor in SQLite. The complete report +is available through the same result reader, with both byte and canonical-data +hash verification; it is never truncated to fit the inline limit. + +`pnpm build` now bundles the worker into +`.next/standalone/biometrics-worker/worker.cjs`; `pnpm build:biometrics-worker` +builds it independently. Packaging rejects database, hardware, automation, +scheduler, HomeKit and streaming dependencies. Shared schemas live in the pure +`contracts.ts` module so the worker does not import repositories. Enqueue pins +the SHA-256 of the actual worker bundle, and the runner checks it before launch. + +`executeNextBacktest` runs one leased attempt after an injected host admission +check. It does not start itself or enable production replay. The host must wire +measured resource/ingest admission and lifecycle handling. `runBacktestChild` +uses a separate low-priority Node process, a clean environment without pod +credentials, a 128 MiB JavaScript heap limit, bounded stdin/stdout/stderr, a +wall-time watchdog, and cancellation with SIGTERM followed by SIGKILL. This is +isolation for shipped trusted code; it is not the external sandbox required +for user-proposed algorithm packages. The heap limit is not a total RSS cap. +The child records evaluation runtime and peak RSS for operational inspection. + +The runner heartbeats independently of the child's progress. It rechecks resource +admission every five seconds and before publication; denied or unavailable health +evidence aborts the child and records a resource interruption under the same +bounded retry policy. Checks never overlap. Shutdown leaves +owned unfinished jobs interrupted for bounded retry. Cancellation is +acknowledged only after the child has stopped, and old attempts cannot publish. +Result reads verify their hash and winning fencing token and return the current +holdout exposure count, so later reuse remains visible. + +## Large report storage + +`ReportArtifactStore` stages a uniquely named file, fsyncs it, atomically renames +it and fsyncs the directory before a fenced database transaction publishes the +descriptor and successful job state. File paths derive from the server-owned +database directory, job UUID, attempt token and content hash; request filenames +never select a path. An in-memory database needs an explicitly injected artifact +store for large-report tests. Existing inline results remain readable. + +A failed database commit can leave an unpublished file, but cannot expose a +partial result. Staged files count toward storage checks. The collector removes +only owned files that are unreferenced and older than one hour; it preserves +committed descriptors and conservatively retains future-version named references +during downgrade. A separate SQLite publication lock serializes quota checks, +file publication, and collection across processes. File writes hold only this +sidecar lock; the live database transaction remains limited to fenced publication. +A terminated process releases the lock automatically. Identical staged retries +reuse the verified file without counting its bytes twice. + +`BacktestHost` provides a serialized, recoverable five-second polling loop and +orphan collection every thirty minutes. Shutdown aborts and awaits the current +attempt before database closure. Its global singleton requires an explicit +admission callback. Production startup and measured resource admission remain +pending; merely importing the host does not start replay. + +## Admission evidence + +`admission.ts` defines a deployment-policy contract with a qualification ID and +explicit limits for memory reserve, free disk, one-minute load per available CPU, +current event-loop delay, piezo age, temperature and sample age. It has no shipped +threshold defaults yet. Missing or malformed policy yields `unqualified`; missing +or malformed health evidence yields `health_unavailable`. Fresh evidence must +meet every limit. Future timestamps and stale samples also defer replay, and the +decision returns all applicable pressure reasons for later API/UI exposure. + +The Linux collector uses `MemAvailable`, filesystem `bavail`, normalized load, +thermal zone zero and a twenty-millisecond scheduling probe in the Node host. +The probe measures current scheduling delay rather than using the lifetime +performance histogram. Its supplied timestamp callback must read specifically +`getSensorFrameTimes()['piezo-dual']`, never the newest arbitrary sensor frame. +Unknown piezo freshness, missing thermal data, read failures or unsupported +platforms yield unavailable evidence. Collection makes no hardware commands or +database queries. These helpers are not yet connected to production startup; +measured thresholds and full publication-load qualification remain required. + +## Explicit report release + +Trusted deployment configuration may set `SLEEPYPOD_BIOMETRICS_REPLAY_BUDGET` +to JSON containing `jobWallMs`, `waveformWallMs`, and `waveformCpuSeconds`. +These are operational ceilings, not candidate parameters or RPC inputs. The +coordinator validates the complete budget before spawning: the job wall limit +is at most 48 hours, each waveform process at most one hour, and CPU time at +most 3,600 seconds. CPU time must fit within waveform wall time, which must fit +within job wall time. No production values are inferred from those ceilings. + +Only execution tasks receive the extended job deadline. Read, publication, +inspection, and cleanup deadlines remain unchanged, as does the Node scorer's +separate deadline. Python applies the selected CPU limit before importing the +numerical implementation. Without explicit configuration, existing short +deadlines and the 120-second CPU limit remain. Changing this environment setting +requires restarting the coordinator service; it does not start the worker host, +grant resource admission, or establish hardware qualification. The synthetic +waveform benchmark accepts the same setting for qualification runs. + +The coordinator reserves a separate process slot for read-only report, trace, +dataset-descriptor, and raw-catalog requests. At most one reader and one work +process run at once. Replay, scanning, publication, release, and cleanup share +the serialized work slot. Each lane accepts at most seven active-plus-pending +requests, with a combined limit of eight, reserving capacity for the other lane. +Pending requests expire after 30 seconds; shutdown cancels both active children +and rejects pending work. This prevents a long replay from consuming report-read +capacity. Resource qualification must include the additional reader's memory +and CPU, and execution time budgets remain independently enforced. + +Completed results support an exact-hash, idempotent release. The release +transaction replaces payload bytes with an empty stored object and records +`released_at_ms`, preserving the original content hash and completion/attempt +metadata. Readers return a null payload with explicit unavailability. Report +reads and releases share the publication lock, preventing file deletion between +descriptor lookup and byte verification. After commit, an unreferenced artifact +is unlinked; failed file deletion leaves an orphan counted against disk quota +until collection can remove it. A failed database transaction never deletes the +file. Referenced artifacts are retained conservatively, including future-format +named references. There is no automatic age-based expiry of committed reports; +release is an explicit user action through the shared API. + +## Paired stage support + +Each stage-replay night now includes comparisons of current versus default and, +when supplied, candidate versus default and current. Each window preserves the +configuration hashes and reports agreement, macro F1 and coverage on each +configuration's own support and on the exact intersection of their predicted +intervals. Conflicting or unknown reference intervals remain excluded by the +shared duration scorer. Common-support confusion matrices are retained; absent +paired support yields null metrics, never zero error. Fixed clock offsets apply +to both outputs for scoring without modifying the original displayed traces. + +A separate coverage-reduction flag and duration/coverage differences expose +candidates that omit difficult intervals. For example, omitting the incorrect +half of a prediction may raise its own-support agreement from 50% to 100%, but +cannot produce an agreement gain on the remaining common support when its +labels are unchanged. This is descriptive reporting only. It does not yet +implement the frozen promotion policy, per-night uncertainty aggregation, or +validated activation. Numeric waveform candidates still require raw replay. + +This comparison adds evaluator work and report bytes. Earlier target benchmark +numbers apply only to their recorded worker hash; repeat qualification with the +final worker before enabling production admission defaults. + +## Whole-night stage summaries + +`stageNightSummary` groups comparisons by baseline/candidate labels and exact +configuration hashes. It first sums confusion durations across each cohort's +nonoverlapping windows, then computes nightly agreement and per-stage +precision/recall/F1. It does not average window F1 values. Both full-window +output coverage and eligible-reference coverage remain available per night. +Duplicate cohorts/comparisons and overlapping windows are rejected. + +Each comparison reports scored and unscored night counts, nights with positive, +negative or zero agreement differences, and nights with any coverage reduction. +The equal-night mean agreement difference gives every scored whole-night cohort +one vote. Pooled duration confusion and agreement are reported separately and +can differ substantially when night lengths differ. Missing reference support +remains null and is counted as unscored, not converted to a zero difference. +The observed nightly minimum/maximum is explicitly a descriptive range, not a +confidence interval. The optional seeded interval is reported separately below. The frozen promotion +policy remains required before a report can establish validated improvement. + +## Reproducible exploratory uncertainty + +Experiment protocols may now include `nightUncertainty` with version +`paired-night-percentile-v1`, an explicit seed (1–64 characters), resample count +(200–5,000), and minimum scored nights (3–60). These fields participate in the +immutable protocol hash. Older protocols remain readable and report +`not_requested`; no parameters are silently added to their hashes. + +The statistic is the equal-night mean of paired common-support agreement +differences. Whole-night cohorts are sorted by ID, then resampled with +replacement, preserving the baseline/candidate pairing. A SHA-256 counter +stream over JSON `[version, seed, counter]` supplies big-endian 32-bit draws; +rejection sampling removes modulo bias. The two-sided 95% percentile interval +uses linear interpolation at ranks `(resamples - 1) * 0.025` and +`(resamples - 1) * 0.975`. This is a versioned local implementation, not a call +to SciPy. The general resampling procedure and pairing are described in +[the SciPy bootstrap reference](https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html). + +Missing-reference nights are counted and omitted from the statistic. Too few +scored nights return `insufficient_nights`, and identical observed differences +return `degenerate`, both with a null interval. The requested minimum is a +reporting threshold, not a claim of adequate statistical power. Intervals assume +exchangeable nights; serial dependence and unobserved nights can make the +estimate misleading. Reports retain these limitations and remain +`insufficient_evidence` for promotion. Objective/coverage/operational policy and +held-out qualification are still required for validated activation. + +## Shared support across all submitted configurations + +Saved stage replays compute one intersection across default, current and any +candidate for each evaluation window, after applying the fixed clock offset. +Every pairwise comparison uses that same mask and identifies its +`commonSupportScope` as `all_configurations`. If any configuration has no output, +the shared mask is empty and common-support agreement is null. The configuration +is never silently omitted from the intersection. Each output's own coverage and +agreement remain separate, preserving visibility into lost output and references. + +The interval sweep clips to the evaluation bounds, preserves gaps and labels, +and rejects overlapping or malformed input intervals. The standalone two-output +comparison helper still supports explicitly labelled `pair` scope, but saved +replay reports always supply the all-configuration mask. Nightly summaries and +uncertainty therefore use the same scored intervals for every comparison in a +run. The output-availability policy and validated promotion gate remain separate +requirements; common support alone does not establish improvement. diff --git a/docs/biometrics-calibration-api.md b/docs/biometrics-calibration-api.md new file mode 100644 index 000000000..c6909e213 --- /dev/null +++ b/docs/biometrics-calibration-api.md @@ -0,0 +1,530 @@ +# Biometric calibration API + +The `biometricTuning` tRPC router and `/api/biometric-tuning/*` OpenAPI routes +share the same services. They use the existing LAN trust model. Supplied +requester names are attribution, not authenticated identities. + +## Inspect and create profiles + +- `getManifest` returns the versioned default, parameter schema and current + manifest hash. +- `getProfileOverview` separates desired configuration, the existing stage + view, sensor-calibration freshness, and observed runtime receipts. No receipt + means unreported, not applied. A legacy stage customization awaiting migration + remains visible. +- `resolveProfile` validates explicit leaf/ordered-array overrides and returns + the resolved snapshot, hash and override provenance without saving anything. + Omit a path to restore inheritance; null does not mean remove. +- `createProfileRevision`, `getProfileRevision`, and `listProfileRevisions` + create and inspect immutable candidates. Creating one does not bind it or + write an activation outbox event. Public origins are manual or agent; + migration/system origins are reserved for server workflows. + +Manual activation is available. Desired profile state must not be presented as +actual runtime adoption; availability means requests can be queued, not that a +processor is currently healthy or has adopted a particular revision. + +## Retain data and submit work + +`listSessions` is cursor-paginated and side-specific. `getSessionCapabilities` +reports actual processed timestamp coverage, gaps, reference metric counts, +and unknown processing provenance. Raw waveforms are marked not inventoried; +the presence of processed readings does not imply signal replay capability. +Reference counts include revisions, so dataset creation requires explicit +import IDs and accepts only one revision per family. + +`createDataset` takes side, asserted sleeper ID, session IDs, reference import +IDs, and an idempotency key. The server pins the configured device timezone. +`getDataset` and `listDatasets` return metadata/manifests, not the retained raw +rows. Snapshot integrity is verified on a dataset read. Reference deletion and +observation retention do not change an existing snapshot. + +`createExperiment` pins a dataset and descriptive protocol: evaluation mode, +explicit assumed stage-calibration quality, and fixed alignment tolerance and +clock offset. `enqueueBacktest` takes the experiment ID, exact candidate +revision ID, expected binding generation, split, and idempotency key. Callers +cannot supply default/current snapshots, evaluator hashes, or executable paths. +The server resolves them. The current stage baseline comes from the actual +retrospective view, not a pending desired vitals profile. + +The current evaluation scopes are original-reading comparison and stage rules +on original readings. Waveform parameter changes are not evaluated by these +scopes. The job descriptor states this explicitly. + +`listBacktests`, `getBacktest`, `cancelBacktest`, and `getBacktestResult` expose +bounded job history, progress, cancellation and verified results. Enqueue +persists work; an admitted worker host must execute it. A queued response is +not a completed result or an activation. Result reads include the current +holdout exposure count. Existing successful retries return the original job +when the same public submission key and payload are supplied. + +`getBacktestSummary` (`GET /biometric-tuning/backtests/{id}/summary`) is the +interactive review path. It verifies the complete report in the coordinator, +then omits stage-trace rows before IPC and HTTP serialization. All scores, +per-night comparisons, support, provenance, and operational measurements remain. +Its `projection` is `scores_without_stage_trace_rows_v1`; `contentHashScope` is +`complete_retained_report`. The hash identifies the retained full artifact, +not the summary's serialized bytes. Summaries exceeding 2 MiB fail explicitly +rather than silently truncating nights or scores. `getBacktestResult` continues +to return the full verified report, and `getBacktestTrace` pages exact trace +rows using that same retained hash. The UI and MCP report reader use the summary +path; release and holdout exposure semantics are unchanged. + +The backtest host loop, measured resource admission, richer promotion reports, +and full client UI/MCP integration remain separate implementation steps. The API does not claim that these are enabled merely because job +submission exists. + +Profile diffs report both changed resolved values and changed override intent, +including explicit-default overrides whose numerical output is unchanged. +`validateProfileRevision` checks schema/engine compatibility only; it explicitly +leaves evaluation evidence unchecked. + +Experiment creation requires an idempotency key. `getExperiment` and +`listExperiments` preserve access to its frozen protocol and audit identity. + +`releaseDataset` is explicit and idempotent. It refuses release while queued, +running, cancellation-pending, or interrupted jobs still need the inputs. +After cancellation or completion, release removes the retained input snapshot, +keeps its manifest/hash and holdout audit metadata, and marks it unavailable. +Existing reports remain separate retained results. New runs cannot use a +released dataset; a fresh snapshot requires a fresh creation key. Storage +accounting continues to include the retained manifest. Active configuration +lives in the configuration database and is unaffected. + +Canonical dataset/report hashing uses the same finite-number encoding as +configuration hashing, with a separate bounded budget for realistic multi-night +rows. Configuration limits remain unchanged. Dataset hashes are checked when +reading retained input data, and metadata reads verify available snapshots; +released metadata remains readable without pretending inputs still exist. + +Activation history is available through `listActivationHistory` (side and +optional `beforeGeneration` cursor). Pages contain at most fifty events and +include fallback/default-update evidence. Profile overview separates its +historical receipts from `runtimeHealth`, which reports current numerical +heartbeat freshness and configuration consistency. General and legacy stage +settings use the same versioned intent and acknowledgment protocol. + +## Manual activation and observation + +- `activateProfile` / `POST /biometric-tuning/activate` selects an exact revision. +- `resetProfile` / `POST /biometric-tuning/reset` selects current shipped defaults + without deleting revisions or audit history. Malformed unmigratable legacy + stage settings are retained in the legacy row and reset audit evidence; valid + unmigrated customizations must first be preserved as a revision. +- `rollbackProfile` / `POST /biometric-tuning/rollback` selects the previous + binding, including its exact revision identity (null means the default binding). + +All three require side, expected binding generation, expected resolved hash, +immediate/next-session boundary, requester, idempotency key, and explicit +`allowUnvalidated: true`. These are deliberate manual changes, recorded as +`manual_unvalidated`. They do not assert that a backtest established improvement. +A stale binding, changed idempotent request or mismatched hash is a conflict. +Creating or evaluating a candidate never calls these mutations implicitly. + +`getActivationStatus` / `GET /biometric-tuning/activation-status` accepts a side +and optional event ID. It returns the persisted audit event, observed delivery +and matching-receipt state, receipt freshness and whether the event still +belongs to the desired generation. Polling is read-only and cannot acknowledge +the stage consumer. The persisted event can lag observed receipts until the +five-second reconciliation pass. An applied old event is history, not proof it +is still selected; check `isCurrent` and the overview's current runtime health. + +The cross-runtime integration test uses temporary fixture databases and the +locked piezo environment. Prepare it with +`cd modules/piezo-processor && uv sync --locked --no-install-project`, then run +`pnpm exec vitest run src/biometrics/profiles/tests/runtime-integration.test.ts` +from the repository root. The JS unit-test CI job installs that same environment. +The test runs the shipped Python adapter with a synthetic frame, verifies the +next-session boundary, TypeScript stage receipt, restart restoration and rollback; +it neither starts a sensor follower nor accesses hardware. + +## Releasing completed reports + +`releaseBacktestResult` (`POST /biometric-tuning/backtests/{id}/result/release`) +requires the job ID and `expectedContentHash`. It applies only to successful +jobs and rejects a mismatched hash. Repeating the same release is idempotent. +The result endpoint now includes `available` and nullable `releasedAtMs`. +After release it returns `available: false`, `report: null` and the original +content hash, completion time, winning fencing token and current holdout count. +A released report is distinct from a job that has not completed (which still +returns null as the whole result). + +Release removes inline payload bytes or detaches an artifact descriptor in a +transaction before deleting the file. It does not erase experiments, jobs, +holdout exposure, profile revisions or activation history. A retried enqueue +with the same idempotency key still resolves to the original successful job; +it cannot recreate released bytes or erase previous holdout use. A new replay +requires a new request key and available retained inputs. + +The additive biometrics migration `0027_release_backtest_reports` preserves +existing reports with a null release timestamp. Older readers cannot validate +a released payload against the original hash, so they report an unavailable or +corrupt result rather than silently displaying different data. UI/MCP clients +must use the explicit availability fields when presenting released evidence. + +## Paginated stage traces + +`getBacktestTrace` (`GET /biometric-tuning/backtests/{id}/trace`) requires +`expectedContentHash` and `configuration` (`default`, `current`, or `candidate`). +It accepts an optional `cohortId`, an `offset` (default zero), and `limit` +(default 200, maximum 500). The hash must match the verified retained result. +Unfinished jobs reject trace requests, changed hashes return a conflict, released +reports return not found, and comparison-only runs report that no stage replay +was produced. Missing cohorts/configurations and cursors beyond the selected +trace are errors rather than fabricated empty results. + +The response includes exact intervals, configuration hashes/revisions, cohort +and source-window identities, the total selected interval count, and +`nextCursor: { offset, contentHash }` or null. Reuse that offset and hash as +`offset` and `expectedContentHash`, keeping the selection unchanged. No +intervals are downsampled or interpolated. Trace timestamps precede the fixed +reference-alignment offset, which is returned separately as `clockOffsetMs`. +The provenance is explicitly `versioned_stage_replay`, not original observed +stages. Filtering/pagination never changes evaluation scores. + +This bounds response detail, not artifact-read cost: the service currently +verifies the entire retained report before selecting a page. Parent-process +artifact parsing and hashing remain part of the target performance qualification. + + +## Retained waveform discovery and publication + +`getReplayAdmission` (`GET /biometric-tuning/replay-admission`) reports whether +server policy is configured, its operator-supplied qualification ID, and current +admission reasons. `SLEEPYPOD_BIOMETRICS_ADMISSION_POLICY` is server-owned JSON +matching `admissionPolicySchema`; missing or malformed policy denies admission. +Supplying an ID does not itself qualify hardware. Each check uses fresh process, +storage, thermal, and live piezo receipt evidence; no caller can supply health +samples or override admission through an RPC parameter. + +`inspectRawSource` (`POST /biometric-tuning/raw-sources/inspect`) accepts only +`afterName`, initially null, and inspects at most one server-owned archive. +Persisted evidence is a mutation even though scanning does not modify the +archive. Reuse `nextCursor` until `complete`; `source_unavailable` advances past +an unreadable source, and `archive_unavailable` reports a missing archive root. +`deferred` preserves the cursor and means no new scan was admitted. This API +accepts neither filesystem paths nor executable code. + +`listRawSources` (`GET /biometric-tuning/raw-sources`) accepts `startMs`, `endMs`, +and optional `afterId`, returning at most fifty descriptors and a cursor. Query +all selected cohort windows including five minutes of pre-roll. Catalog event +bounds indicate possible overlap, not continuous support. Descriptors explicitly +say `requires_reverification`; inventory alone has not retained any waveforms. + +`createWaveformDataset` (`POST /biometric-tuning/waveform-datasets`) accepts a +parent dataset ID, its exact `expectedContentHash`, an idempotency key, and +`cohorts: [{ cohortId, sourceIds }]`. The server derives context from that frozen +parent, re-verifies selected archives, and atomically retains inputs and publishes +the immutable dataset. New publication requires ongoing admission; a `deferred` +response contains no dataset. A retry of already committed publication returns +the original descriptor even when current admission is unavailable or original +archives have disappeared. Changed selections under the same key conflict. +Released descriptors remain released; retries never recreate their bytes. + +Create a `signal_replay` experiment using the published dataset, then submit and +poll a durable backtest. Publication does not start a worker host or activate a +profile. Host qualification and scheduling remain separate operational gates. + + +The retained-dataset UI exposes **Select recorded waveforms** on available +processed datasets. It inspects one archive per explicit request, keeps scan +cursors on deferral, and preserves source selections across pages and nights. +Every parent cohort requires a selection, with at most 64 source IDs per cohort. +Publication pins the parent content hash and reuses its request key after a lost +reply or deferral. Released publication retries are shown as unavailable. + +Published waveform datasets offer **Replay retained waveforms** in the protocol +editor. Sleep-stage replay is optional and requires an explicit calibration +quality assumption; numerical-only replay does not. Protocol creation and queued +submission still depend on the separate worker host for execution. The browser +demo contains no RAW archive fixture or qualified pod telemetry, so its waveform +operations remain explicitly deferred. + +### Numeric objective policies + +Signal-replay protocols may pin an `objectivePolicy` with version +`numeric-objective-v1`. Its metric, minimum supported nights and paired references, +maximum lost-observation fraction, minimum MAE improvement, night and subgroup +regression tolerances, required subgroup names, and operational limits are included +in the protocol hash. Processed-reading protocols reject numeric objectives. +Reports retain the thresholds and per-baseline `objectiveAssessments`. + +Caller-supplied thresholds are not a designated promotion policy. The current +production evaluator has no designated policy and does not yet attach measured +operational or subgroup evidence, so its assessments remain insufficient evidence. +The pure assessor checks an independent first-use holdout, supported whole nights, +uncertainty, coverage, subgroup regressions, and measured operational limits before +it can award improvement under a designated policy. Test-only thresholds exercise +boundaries; they are not release defaults or evidence of clinical validation. + +### Replay host startup + +After migrations and hardware initialization, production startup reads +`SLEEPYPOD_BIOMETRICS_ADMISSION_POLICY`. A missing or invalid policy leaves the +background host stopped. A valid policy starts the existing singleton host; +startup does not assert that hardware has been qualified. Its execution and +cleanup operations still obtain fresh admission, including live piezo receipt +age. CI skips this startup with the other hardware-dependent services. Shutdown +aborts and awaits the host before closing the coordinator. + +Deploy only a policy backed by recorded pod measurements. Configure the matching +`SLEEPYPOD_BIOMETRICS_REPLAY_BUDGET` before process startup; the coordinator captures +that budget when its singleton is created. Changing process environment requires +a service restart to start a previously disabled host or replace that budget. + +New waveform dataset manifests advertise retained-input replay capability. +Execution availability remains a separate, current admission query. Older +immutable manifests and their hashes are not rewritten by this change. + +### Exact numeric trace pages + +`GET /api/biometric-tuning/backtests/{id}/numeric-trace` (tRPC +`getBacktestNumericTrace`, MCP `backtest_biometrics` action `numeric_trace`) +returns up to 500 exact recomputed readings per page. Supply the immutable report +`expectedContentHash`, configuration label, optional cohort, and offset/limit. +The next cursor carries the same hash. Released results and stale hashes fail; +paging does not rerun an experiment or change its scores. + +Pages contain only the report dataset's side, within its scoring windows. They +preserve null readings, quality, flags, routing provenance, and the original +replay timestamp; `clockOffsetMs` is metadata for reference alignment and is not +applied to the returned timestamp. Opposite-side readings and internal `hrRaw` +values are excluded. No downsampling is performed. Historical reports without +explicit side provenance cannot supply this projection. Numeric trace UI and +reference overlays remain separate consumers of this API. + +Waveform reports expose these pages through **Inspect numeric traces**. Select a +night, configuration, and metric; the chart shows only that page's individual +readings. The exact-value table includes missing values, quality, flags, source +channel, and limitations. Changing night/configuration resets the cursor. The +processed-only demo does not fabricate waveform readings for this view. + +### Frozen reference pages + +`getBacktestReferenceTrace` / `GET .../backtests/{id}/reference-trace` and MCP +`backtest_biometrics` action `reference_trace` require a report hash, cohort, +metric, and bounded offset/limit. They read the immutable dataset selected by +that report, verify its hash, and expose the frozen import revision, source, +normalized measurement, compatibility reason, and original reference times. +They never resolve the current revision of an import. Original submitted payloads, +annotations, and subject fields are excluded. + +These are display records, not newly matched pairs or independent evidence. +Point/interval and aggregation semantics remain explicit. HRV-family records +with an incompatible definition remain inspectable with their compatibility +reason. Scoring-window boundaries are preserved. A released report or dataset +makes this detail unavailable; it does not silently fall back to live data. + +In a waveform report, open **Inspect numeric traces**, then **Compare frozen +references**. The overlay pages references independently from recomputed readings +(200 records per page), preserves point and interval shapes, and applies the +experiment's fixed clock offset only to recomputed timestamps. The source table +keeps reference timestamps in UTC. Incompatible definitions remain visible in the +table but are not plotted. Display overlap does not establish a new scored pair; +use the report's frozen alignment and common-support scores for comparisons. + +### Numeric subgroup evidence + +Waveform experiments with a numeric objective additionally compute +`reference-defined-numeric-subgroups-v1` for that objective's metric. Fixed IDs +are `reference_points`, `reference_intervals`, `window_first_half`, and +`window_second_half`. Membership uses native reference timestamps and temporal +kind, never candidate values, errors, or missingness. Each disconnected scoring +window has its own midpoint. A point at the midpoint belongs to the second half; +an interval ending there belongs to the first. Intervals spanning the midpoint +are excluded from both half-window groups without splitting or changing their +aggregation; the report retains their excluded count. + +Each group uses the same all-configuration common timestamp support as the main +numeric comparison. References are weighted within a night, then nights receive +equal weight. Objective evidence requires the configured minimum paired records +in every included night; an unsupported night makes that group's mean unavailable +for promotion rather than silently dropping it. A required group with inadequate +night support blocks improvement. Per-window observation coverage counters still +refer to complete scoring windows, not inferred continuous coverage within a +reference subgroup. The report UI shows subgroup means, night counts and boundary +exclusions. Groups overlap and are not independent statistical experiments. +They do not establish motion, pump, empty-bed, or artifact-specific performance; +those require separately qualified context/fixture evidence. Unrecognized required +group names remain unsupported and cannot award improvement. + +### Stage objective evidence + +`stageObjectivePolicy` freezes a `stage-objective-v1` policy alongside the +experiment protocol. It requires `stage_replay`, or `signal_replay` with +`waveformStages: true`; plain comparison cannot evaluate a stage objective. +Numeric and stage policies are separate, and a joint replay may retain both. + +The stage policy sets minimum independent nights and paired reference duration, +minimum candidate output coverage, maximum loss of baseline output from shared +support, minimum equal-night agreement improvement, tolerated night regressions, +macro-F1 regression, and required-stage F1 regression/support. Common-support +confusion durations are combined within each whole night before scoring. The +uncertainty gate uses the lower bound of the frozen whole-night agreement +bootstrap; epochs are not treated as independent nights. Missing required stages +on any evaluated night remain insufficient evidence, never fabricated zero error. + +`stageObjectiveAssessments` preserve the exact policy and evidence for candidate +versus both default and current configurations. The coordinator replaces any +scorer-supplied operational evidence after the final admission check, using the +same explicit sampling-policy rules as numeric objectives. Only a separately +designated qualified policy can award improvement. The UI presents thresholds, +missing evidence and regression reasons. These are agreement results against the +named stage reference, not claims of clinical stage accuracy or live activation. + +### Evidence verification before activation + +The internal profile evidence verifier checks retained report bytes and exact +candidate revision, current configuration, shipped default, dataset/side, frozen +holdout cohorts, protocol, evaluator build, and waveform dependencies. Reused +holdout data or released artifacts cannot pass. These expectations must be +loaded by the service; they are not client assertions. + +The verifier recomputes objective assessments using supervised operational +measurements and separately designated policies. It requires both baseline +comparisons and every frozen holdout night. A changed stage configuration must +improve stage agreement; a numeric change must improve its metric and pass +downstream stage coverage/regression checks, which may report no material change. +An assessment of one numeric metric cannot validate changes to another metric. +A signal replay can freeze `objectivePolicies`, a list of one to three numeric +policies with unique metric identities. It is mutually exclusive with legacy +`objectivePolicy`; legacy snapshots and hashes remain unchanged. Evaluation, +supervisor finalization, and activation verification require candidate comparisons +against both default and current for every policy. Each changed numeric component +must have its own qualified improvement evidence; a passing heart-rate objective +cannot validate breathing or HRV changes. Stage objectives remain separately +required for downstream effects. Report displays group thresholds and reasons by +metric. These policy lists do not themselves designate a production policy. + +The internal validated activation service loads the retained job, experiment, +dataset, report, profile revision and current runtime baseline. It verifies +the report before acquiring write reservations, then rechecks report/dataset +availability, fencing identity, holdout exposure, runtime baseline and binding +generation before committing. A short biometrics write reservation prevents +concurrent exposure, release or runtime writes until the configuration commit +finishes. Full report parsing and hashing happen before this reservation; the +final artifact checks read only metadata. Only the configuration database is +written: binding, evidence audit and outbox intent commit together. + +The audit keeps the evaluation ID, report/configuration/build hashes, policies +and assessment summary even if biometric artifacts are later released. An exact +idempotent retry returns the original activation event without requiring those +artifacts again. Changing the request under the same key is rejected; manual +and validated requests cannot alias one another. A successful request still +returns `pending_delivery`, not proof of processor adoption. + +Stage replay now captures the entire acknowledged current configuration, including +numeric parameters, instead of substituting defaults for those parameters. Pending +or incompatible personalization blocks validated activation. Qualified production +policy designation remains unfinished. The shared +`activateValidatedProfile` procedure (`POST /biometric-tuning/activate-validated`) +accepts the exact revision, resolved hash, binding generation, evaluation ID, +report hash, processing boundary, requester, and idempotency key. Build hashes and +policy designations are resolved on the server. New requests currently fail closed +because no production policy is designated. Exact retries precede build probing +and artifact lookup. MCP `apply_biometric_profile` exposes the same procedure with +`action: "activate_validated"`; it does not accept `allowUnvalidated`. Explicit +manual activation retains its existing unvalidated semantics. + + +The holdout report UI offers application of its exact candidate and report hash. +`reviewValidatedActivation` (`GET /biometric-tuning/validated-activation-review`) +performs a read-only eligibility check using the same stored evidence and installed +build authority. A review creates no activation event or outbox item, and cannot +reserve evidence: application repeats verification and its transactional guards. +The apply control remains disabled when review fails or is refreshing. Once sent, +an uncertain request retains its exact identity and boundary for retry, including +when detailed report data is subsequently released. Processor adoption is polled +separately. The demo explicitly rejects validated activation of synthetic evidence. + + +Promotion designation also pins the complete experiment protocol hash. Matching +objective thresholds alone is insufficient: changes to clock offset, alignment +tolerance, assumed stage calibration quality, uncertainty settings, replay mode, +or objective lists require qualification of the resulting protocol. This server +value cannot be supplied through activation or review requests. The audit already +retains that exact protocol hash. This identity gate does not establish that missing +historical pump/occupancy context is acceptable; replay-context qualification is +still a separate release requirement. + + +Waveform promotion also requires a server-designated list of accepted replay +limitations. The verifier reads both per-run and per-reading diagnostics across +every held-out cohort and distinct configuration, verifies their retained payload +hash and build/configuration identities, and rejects missing or duplicate outputs. +Missing pump/occupancy/sleeper-mode context, short pre-roll, cold starts and event +clock substitution cannot pass merely because numeric scores improved. Unknown +limitation codes cannot be designated until the contract is reviewed. Activation +audit evidence records both the accepted list and the limitations actually observed. +No production replay-context allowance is installed yet; this gate does not by +itself qualify any approximation. + + +The publisher and activation service share a release-installed qualification +registry, `promotion-qualifications.json`, beside the packaged replay worker. +Its version is `biometric-promotion-qualifications-v1`; each entry pins the full +protocol, manifest hash, evaluator build hash, optional Python build hash, +qualification ID, fixture and pod evidence hashes, and replay-context allowance. +Every objective policy must use the entry's qualification ID. Exactly one matching +entry is required. Missing, malformed, oversized, duplicate or mismatched entries +grant no designation. Reads are capped at 256 KiB; no public endpoint writes this +registry. Evidence hashes identify release-reviewed artifacts; loading a registry +is not itself a qualification experiment. The publisher additionally rejects +context outside the registered allowance before awarding metric-level statuses. +No production registry has been generated or installed yet. + + +The supervisor now derives the overall retained report status with the same +assessment logic used for activation. `improved` requires complete qualified +independent evidence for every changed domain and acceptable downstream stages. +Supported failures produce `regressed`; insufficient support remains +`insufficient_evidence`, incompatible metric evidence is `incompatible`, and +supported outcomes below the improvement threshold are `no_material_change`. +An identical candidate/current configuration is also reported as no material +change. The supervisor replaces worker status claims. The UI displays these as +historical evaluation outcomes; an improved report never substitutes for current +activation review, holdout-exposure checks, or processor acknowledgement. + +## Original vitals provenance and CSV compatibility + +`biometrics.getVitals` retains its array and existing numerical fields. It now +adds nullable `processingRunId` and `provenance` metadata for each original +observation: `interpretation: original_observation`, a status, the recorded +configuration hash, and the recorded HRV metric identifier. The run must belong +to the same side and be a live run. Missing associations are `unavailable`; rows +without a run remain `legacy_unknown`. A missing, malformed, or unavailable HRV +definition stays null. The current shipped manifest is never used to infer an +old row's metric identity. This metadata describes the recorded association; +it is not validation evidence or a current processor-health receipt. + +Only bounded scalar metadata is projected from the processing bundle. The raw +bundle, input identity, and unrelated fields are not returned. A future recorded +HRV metric identifier is preserved rather than relabelled as the current window +IBI variability index. Consumers must keep different/unknown definitions +separate when computing comparable baselines. + +Summary and baseline responses now include exact SQL aggregate provenance counts +across the full selected range, independent of chart pagination: distinct recorded +configurations (manifest plus resolved configuration hash), unknown configuration +rows, HRV definitions and unknown HRV rows. Mixed known definitions, or known +and unknown HRV values together, produce null HRV aggregates. Entirely legacy +ranges retain their descriptive values with `legacy_unverified`; no definition +is inferred. All baseline means and deviations are null when multiple recorded +configurations or recorded/unknown configurations coexist. A summary may still +describe original HR/breathing values across configurations, with explicit counts. + +The chart separates recorded processing runs and HRV definitions before smoothing, +omits bands/statistics across those boundaries, and explains original/unknown +provenance. Its raw log exposes run IDs and HRV definitions. Unknown legacy rows +remain unverified: missing history cannot prove that their processing was uniform. +A run boundary is deliberately conservative even when two restarts used the same +configuration. This is distinct from the manifest/configuration counts in the +whole-range API summary. + +The vitals CSV keeps its original five columns first and appends interpretation, +processing-run ID, provenance status, configuration hash, and HRV metric ID. +Unavailable definitions are written as `unknown`, never inferred as clinical +RMSSD. Missing numerical values remain empty. Metadata cells are CSV-escaped +and spreadsheet-formula prefixes are neutralized. Older consumers can continue +using the original named columns; positional consumers must accept appended +columns. Demo/older responses without metadata export unknown provenance. diff --git a/docs/biometrics-calibration-readiness.md b/docs/biometrics-calibration-readiness.md new file mode 100644 index 000000000..60f309ec4 --- /dev/null +++ b/docs/biometrics-calibration-readiness.md @@ -0,0 +1,72 @@ +# Biometric calibration release evidence + +This is the acceptance evidence map for the calibration implementation, including +prospective monitoring and external algorithm experiments. A passing synthetic +benchmark is resource evidence for its exact build and input; it is not evidence +of improved accuracy or authorization to promote an algorithm. + +## Validation recorded on October 13, 2026 + +- Full CI Vitest run at `cb39444b`: 6,856 passed, 18 skipped; 480 files passed, + one skipped. Skips are not counted as verified behavior. +- Locked piezo environment: 203 piezo/shared numerical tests passed; the separate + common-module suite passed 415 tests with five skipped. +- TypeScript and the production build passed after the MCP workflow addition. +- Browser checks at `216dab46`: five passed, including the real database manual + calibration/receipt/rollback flow and English/Spanish provenance views. +- Deployment archive regression at `b1e05db3`: 26 tests passed. Local Python + environments/caches are excluded while source and locked dependencies remain. +- The shadow benchmark harness at `4ee74c25` passes TypeScript/lint and bundles; + those checks do not establish target-pod execution. + +## Acceptance map + +These links locate implementation contracts and behavioral evidence. Target-pod +acceptance is separate from fixture and browser results. + +| Requirement | Implementation and evidence | +| --- | --- | +| Defaults without user configuration | [Registry tests](../src/biometrics/algorithms/tests/registry.test.ts), [numerical golden fixtures](../modules/piezo-processor/test_default_golden.py) | +| Independent compatible side overrides | [Profile activation tests](../src/biometrics/profiles/tests/activation.test.ts), [runtime routing](../modules/common/test_biometric_routing.py) | +| Sensor calibration distinct from personalization | [Runtime ownership and lifecycle](biometrics-runtime-activation.md), [sensor calibration architecture](hardware/calibration-architecture.md) | +| Preserve existing stage overrides | [Runtime integration](../src/biometrics/profiles/tests/runtime-integration.test.ts), [migration/provenance tests](../src/biometrics/profiles/tests/provenance-migration.test.ts) | +| Reference source, metric, interval, timezone and import provenance | [Reference normalization](../src/biometrics/references/tests/normalize.test.ts), [reference storage](../src/biometrics/references/tests/store.test.ts), [import contracts](biometrics-reference-imports.md) | +| HRV definition across storage, reports, UI, exports and prompts | [API/provenance contracts](biometrics-calibration-api.md), [router tests](../src/server/routers/tests/biometrics.provenance.test.ts), [CSV tests](../src/components/biometrics/tests/vitals-csv.test.ts), [MCP contracts](../src/mcp/tests/server.test.ts) | +| Identical retained inputs, whole-night separation, coverage-aware evaluation | [Frozen datasets](biometrics-datasets.md), [dataset tests](../src/biometrics/backtest/tests/datasets.test.ts), [evaluation assessment](../src/biometrics/profiles/tests/evaluation-assessment.test.ts) | +| Missing raw/context limits claims | [Waveform dataset tests](../src/biometrics/backtest/tests/waveform-dataset.test.ts), [retained replay tests](../modules/piezo-processor/test_retained_replay.py) | +| Shared processing, durable jobs, bounded resources, no canonical/hardware effects | [Backtest lifecycle](biometrics-backtest-jobs.md), [waveform publication integration](../src/biometrics/backtest/tests/waveform-publication.integration.test.ts), [shared live/replay tests](../modules/piezo-processor/test_waveform_replay.py) | +| Shared UI/MCP workflow | [Browser workflow](../tests/browser/workflow.e2e.ts), [failed validation](../tests/browser/calibration.e2e.ts), [MCP session tests](../src/mcp/tests/server.test.ts) | +| Exact revision, concurrency and actual acknowledgment | [Runtime integration](../src/biometrics/profiles/tests/runtime-integration.test.ts), [daemon adapter tests](../modules/piezo-processor/test_runtime_adapter.py) | +| Distinct audited reset and rollback | [Activation tests](../src/biometrics/profiles/tests/activation.test.ts), [runtime protocol](biometrics-runtime-activation.md) | +| Original observations distinct from recomputation | [Historical API semantics](biometrics-calibration-api.md), [provenance browser checks](../tests/browser/provenance.e2e.ts) | +| Coherent updates, resets, cleanup, failures and restarts | [Runtime host tests](../src/biometrics/profiles/tests/runtime-host.test.ts), [backtest lifecycle](biometrics-backtest-jobs.md), [deployment tests](../tests/updateDeploy.test.ts) | +| Isolated generated code and reviewed promotion | [External experiment contracts and real-Docker checks](../scripts/algorithm-experiments/README.md), [shadow admission tests](../src/biometrics/shadow/tests/admission.test.ts), [promotion registry tests](../src/biometrics/profiles/tests/qualification-registry.test.ts) | +| Target-pod operational behavior | [Measurements and limitations](biometrics-replay-benchmark.md); deployment and live measurement status below | + +## Delivery and qualification status + +- The current eight-hour replay and 30-minute synthetic shadow supervisor runs + completed successfully; their receipts are linked in the measurement document. + The separate 180-second live source/isolated sink assessment also completed; + its exit-race regression, build identities and limitations are recorded there. + Synthetic output cannot supply held-out accuracy. +- The measured replay policy is installed and the pod reports admitted. + Reviewed promotion/shadow registries still need their own matching fixture and + held-out evidence; absence must remain visible and deny qualified promotion. +- Build `4c3a6a6e` is deployed, including the live-shadow exit-race fix. Migrations succeeded; the calibration API, live + data path and core/numerical services are healthy, and new vitals carry processing + provenance. [Deployment receipt](performance/biometrics-replay-2026-10-13/deployment-verification.json). + Unpersonalized defaults have no requested activation generation, so this check + does not claim a new profile acknowledgment or change the user's profile. +- [PR #824](https://github.com/sleepypod/core/pull/824) is pushed against `dev`. + CodeRabbit full review was requested, but the provider skipped it because + the PR exceeds its 100-file review limit. No completed CodeRabbit review is claimed. +- Build, lint, typecheck, unit, browser and all Python jobs passed at `8909f305`. + The patch-coverage gate passed at 86.47%, within the repository's existing + eight-point margin below its 88.93% target. No coverage threshold was changed. + Direct coordinator and host tests cover admission, cancellation, retention, + error sanitization and resource ownership. + +Legacy reference availability alone does not establish independent nights, metric +compatibility, reference reliability or sufficient held-out support. No accuracy +qualification is inferred from an inventory of existing references. diff --git a/docs/biometrics-datasets.md b/docs/biometrics-datasets.md new file mode 100644 index 000000000..b52e093ae --- /dev/null +++ b/docs/biometrics-datasets.md @@ -0,0 +1,251 @@ +# Frozen biometric datasets + +`DatasetStore` retains bounded processed-data snapshots in `biometrics.db`. +It is the persistence layer for the calibration backtest workflow; it is not +yet a waveform replay service. Creating a dataset does not modify live vitals, +active settings, or hardware. + +A snapshot contains the selected full recorded sessions, original vitals and +movement rows, exact reference import revisions and normalized measurements, +and available processing-run bundles. Unknown legacy processing provenance +stays null. Measured calibration bytes are not inferred from today's profile. +The manifest explicitly records that limitation and disables signal replay. +Stage replay must state its calibration-quality assumption when the original +quality context is unavailable. + +Each snapshot owns its input bytes. Deleting or correcting a reference import, +or pruning original readings, leaves a frozen dataset unchanged. Dataset reads +verify the hash of both manifest and snapshot. Reusing an idempotency key +returns the original dataset; a different selection under that key fails. +Only one revision from a reference family may appear in a dataset. Both side +and asserted sleeper mapping must match its references. + +The initial processed-data limits are 60 selected sessions, 20 reference +imports, 20,000 retained rows, 8 MiB per snapshot including its manifest, and +64 MiB across these snapshots. These are conservative service bounds, not a +claim about target-pod replay performance. Raw artifacts need separate retention +and free-space safeguards before signal replay can be enabled. + +Whole-night grouping uses the selected timezone's local noon as the night +boundary. Overnight sessions and naps touching the same night stay together; +a session spanning noon connects adjacent night cohorts. Cohorts are sorted +chronologically before assigning tuning, validation, and holdout. Fewer than +three cohorts yields descriptive tuning only. More cohorts establish a split, +not validated improvement or adequate statistical power. ICU/timezone runtime +versions and the actual assignment are retained; later timezone updates cannot +silently change an existing dataset. + +Coverage includes actual timestamp counts, duplicates, and leading/internal/ +trailing gaps under an explicit nominal cadence. This estimates recording +availability, not measured signal support duration. Overlapping selected +sessions retain original row IDs; evaluators must deduplicate shared rows and +score one independent night cohort rather than treating sessions as independent +nights. Retrospective classification remains retrospective. + +## Shared waveform replay driver + +`modules/common/biometrics/replay.py` provides the internal streaming driver for +retained waveform experiments. Callers supply immutable per-logical-side +configurations, integer Unix-millisecond record times, recorded sleeper mode +and occupancy snapshots, pump-context availability, and an experiment output +sink. The driver does not read current bindings, occupancy files, or databases. +It processes both physical channels in left-before-right order using the same +sample repair, pump/presence gates, stateful estimator, and single-sleeper +arbitration as the live adapter. + +Records must be ordered within the retained half-open time window. Each channel +is capped at 5,000 int32 samples per record; processor buffers retain only their +fixed windows. Pre-roll is processed, but outputs are scored only inside the +requested half-open scoring window. The output records preserve computed +logical side, physical source side, configuration hash, event timestamp, +quality, and flags, including when a held candidate is flushed after a mode +change. Context and outputs are immutable; outputs stream to the supplied sink. + +Cold-start state and replacing live arrival time with recorded event time are +explicit limitations. Missing mode, occupancy, or pump context adds separate +limitations; unknown mode uses a disclosed two-side fallback. Less than five +minutes of pre-roll is also flagged. These labels do not claim exact historical +parity. The shared driver does not itself discover or retain RAW files, capture +historical context, or authorize a public waveform job. Those integration and +resource checks are required before dataset capabilities may advertise raw +signal replay. + +`modules/common/biometrics/raw_artifacts.py` supplies retained-byte storage for +that integration. It accepts a selected `.RAW.gz` filename under a server-owned +archive directory, rejects path traversal, symlinks and non-regular files, and +copies into a private temporary file. It detects source changes, verifies gzip +expansion within a byte limit, hashes the exact compressed bytes, fsyncs the +copy, and atomically publishes a content-addressed artifact. Repeated retention +of identical bytes reuses the verified artifact. Deleting the rolling archive +thereafter does not remove the retained input. + +A process lock serializes publication/quota checks and verified reads. Crash +staging files are reclaimed only while holding that lock. Published artifacts +are immutable and private (0600 files, 0700 directory); readers check their +size and SHA-256 before exposing a bounded decompression stream. A changed or +missing artifact fails rather than substituting an archive file with the same +original name. Gzip validation is not CBOR/frame or timestamp validation: the +record inventory must still establish those properties before advertising +waveform coverage. + +The injectable retention limits default to 256 MiB compressed per artifact, +512 MiB expanded, 1 GiB and 1,024 published artifacts total, and a 512 MiB free-space +reserve. One additional compressed staging copy can exist while publication +is serialized, including during an idempotent retry at the published quota. +These are provisional storage limits, not a measured production replay policy. +Dataset references, release/garbage collection, record inventory, and job +admission must be integrated before this storage helper is exposed publicly. + +`raw_records.py` adds a bounded firmware-CBOR decoder and retained-file inventory. +It supports the definite and indefinite maps/arrays/strings used by captured +firmware, with explicit record-byte, string-byte, node-count and nesting limits. +Container lengths are checked before allocation; duplicate/non-text map keys, +semantic tags, malformed UTF-8 and trailing values are rejected. The retained +iterator distinguishes clean end-of-file from a truncated outer RAW record and +counts empty sequence placeholders toward its record limit. + +The inventory reports exact retained artifact identity, event-time bounds, +relevant record counts, usable dual-channel frames, lost-marker counts, +truncated sample tails, timestamp reversals and oversized channels. Its +`ordered_waveform_available` flag means the retained records meet this driver's +ordering and frame bounds; it does not establish continuous recording coverage, +complete pump context, or availability of occupancy/calibration snapshots. +Equal event timestamps are reported and preserved. Other sensor records are +counted without being misrepresented as waveform support. Numerical sensor +values retain CBOR float semantics; nonfinite or invalid event timestamps are +reported as unavailable for ordered replay. + +`raw_sources.py` inspects server-owned cold archives without retaining them. +Discovery is bounded and excludes symlinks; inspection streams checksum, +expanded-size validation and record inventory. Its result explicitly states +`retained: false`. Selection passes the inspected descriptor as `expected` to +`RawArtifactStore.retain`; changed compressed bytes or sizes fail before +publication. A source name or successful inspection alone is never evidence +that a dataset has frozen input bytes. Persisted inventory caching and dataset +references remain responsible for tracking inspected versus retained state. + +Migration `0028_retained_raw_artifacts` adds separate tables for inspected +sources, content-addressed retained artifacts, and ordered dataset references. +`RawSourceCatalog` validates the Python inspection contract, including matching +hashes, event bounds, record counts and ordered-waveform eligibility. Repeating +an inspection refreshes its timestamp; different evidence for identical bytes +and inventory version fails. Replacing a filename preserves the old identity +and marks it unavailable. A late failure for an old identity cannot mark newer +bytes unavailable. + +Catalog queries return at most 50 entries per page for potential event-time +overlap. Unexpired inspection evidence is labelled `requires_reverification`, +not available or retained: selected copies still require the expected-descriptor +check. Source scans do not create datasets or retained-artifact rows. Shared +artifact references have unique dataset positions and cannot dangle; release +and file garbage collection must coordinate those references before deletion. +The catalog and schema do not yet enable public waveform jobs. + +Dataset release now atomically tombstones its RAW references with the dataset +snapshot. Queued, running, cancellation-pending and interrupted jobs prevent +release. An artifact becomes eligible for collection only when its last live +reference is released; releasing another dataset never changes that timestamp. +This eligibility timestamp does not itself mean the file has been deleted. + +`RawArtifactStore.collect_released` holds the same process lock as verified +readers and publication, then takes a SQLite immediate transaction while it +checks eligibility and unlinks files. The default one-hour grace period is +injectable. Directory traversal is bounded before any deletion; the collector +preserves unknown files/metadata, rejects symlinks and mismatched identities, +and keeps database tombstones. It scans present files, so previously collected +rows cannot starve later eligible files. Unregistered crash orphans are not yet +collected automatically. + +The internal `retain(..., expected=descriptor, publish=callback)` hook allows a +trusted publisher to commit references before releasing the file lock, including +when reusing existing bytes. Publishers must lock files before beginning their +SQLite transaction, commit before returning, and never reacquire the file lock +inside the callback. Publication failure leaves complete bytes for recovery. +The public waveform dataset creation path and scheduled collector still need +to be wired through this protocol before waveform replay is enabled. + +`retained_replay.py` connects immutable artifact descriptors to the shared +waveform engine. Its versioned `retained-waveform-input-v1` contract contains a +retained/pre-roll/scoring window, ordered artifact descriptors with inspected +event bounds, and frozen context intervals. It rejects duplicate or overlapping +artifact windows instead of guessing which boundary frames to discard. Missing +context intervals resolve to unknown; it never reads the current sleeper mode, +occupancy or calibration to fill historical gaps. + +The adapter verifies retained bytes and consumes every selected file to EOF, +including events outside the scoring window. It checks actual event bounds and +ordering against the frozen input, then passes selected frames to the same +numerical driver as direct replay. Initial bounds are 64 artifacts, 1 GiB +compressed input, one million RAW records across all artifacts (including +empty placeholders), 20,000 context intervals and 20,000 output readings. +These are explicit implementation bounds, not a qualified pod resource policy. +The caller must also enforce a process deadline; cooperative cancellation does +not interrupt a file checksum operation already in progress. + +Output sink rows are provisional until the adapter returns successfully. A +late archive integrity failure must fail the job and discard its staged result. +The returned run metadata includes consumed/decoded counts, exact artifact and +configuration hashes, processing counters and limitations. The public durable +worker still needs to capture this input contract, retain its references and +publish the completed result through its fenced job lifecycle. + +The fixed Python entry point is `python -B -m common.biometrics.replay_worker +--retained-directory `. Its input is one bounded JSON +`retained-waveform-work-v1` envelope containing `input` and both logical-side +`configs`. It opens existing retained storage in read-only mode: missing storage +or a missing lock fails, and it neither creates directories nor changes their +permissions. RAW retention/collection methods reject calls on that reader. + +The worker emits bounded JSON-lines `reading` events and exactly one final +`result` on success. Readings explicitly identify HRV as +`window_ibi_variability_v1`; nonfinite output fails serialization. Input rejects +duplicate JSON fields and nonfinite constants. Errors emit one fixed diagnostic +without exporting inputs, paths or stack traces. CPU limits are 120 seconds +(121 hard), file descriptors are capped at 64, core dumps are disabled, numerical +thread counts are one, and process priority is reduced. These provisional caps +are not a qualified memory or pod admission policy and do not sandbox uploaded +code. + +`runWaveformChild` provides the coordinator-side transport with a clean child +environment, 8 MiB input, 16 MiB output, 64 KiB event and 16 KiB diagnostic +limits, a 20,000-reading bound, and timeout/cancellation with TERM/KILL escalation. +It verifies event order, schema, final row counts and per-side configuration +hashes, returning readings only after a final result and zero exit. A missing or +failed worker never returns its provisional rows. Executable, module and retained +storage locations are internal server configuration and must not be accepted +from public requests. The transport is tested against the actual Python worker; +the coordinator still needs to invoke the frozen dataset publisher and durable +waveform jobs before public waveform capability can be enabled. + +Waveform dataset publication now has an internal complete path. The TypeScript +preparer verifies a processed parent dataset, detaches its historical context, +and creates a new `waveform-dataset-v1` snapshot with one frozen input per +whole-night cohort. It preserves original observations and references, requires +matching scoring windows, normalizes cohort order, and hashes the complete new +snapshot. Shared artifacts must have identical descriptors and inspected event +bounds across cohorts. The parent dataset remains independently readable. + +The fixed `common.biometrics.publication_worker` accepts a server-prepared row +and inspected source selections, with database/archive/retained paths supplied +only by server configuration. It opens an existing database, enables foreign +keys, and uses `retain_batch` to hold the file lifecycle lock through all copies +and reference publication. Copies and checksum/decompression verification finish +before the SQLite immediate transaction. That transaction inserts the dataset, +artifact metadata, and all ordered references together, enforcing the aggregate +snapshot quota. A failed transaction rolls back every database write. Complete +unreferenced files can remain after failure and still require orphan recovery. + +Identical retries return the original dataset, including a released tombstone; +they do not resurrect it or require the rolling archives. A new dataset may +reuse exact retained bytes after rechecking compressed identity and expanded +size, even when the original archive directory has expired. Reusing an artifact +clears its collection eligibility only in the transaction adding its new live +references. Existing reference history and artifact creation time remain intact. + +The shared reader understands both processed and waveform snapshot versions and +verifies their full content hashes. A committed integration test covers actual +archive inspection → TypeScript preparation → Python publication → shared +reader verification → Python numerical replay → release → collection, while +checking that canonical observations and the parent dataset remain unchanged. +Public `signalReplay` remains false until coordinator/API source selection and +fenced waveform-job/report integration are wired to this publisher. diff --git a/docs/biometrics-reference-imports.md b/docs/biometrics-reference-imports.md new file mode 100644 index 000000000..5413a8c94 --- /dev/null +++ b/docs/biometrics-reference-imports.md @@ -0,0 +1,175 @@ +# Biometric reference import contract + +Reference imports are comparison data. They never write the pod's `vitals`, +change a profile, or control hardware. Agreement with a named reference source +is not clinical accuracy. APIs use the existing LAN trust model. + +## Preview and commit + +Call `biometricTuning.previewImport` with an explicit physical `side`, a +`subjectId` identifying the sleeper, a named `source`, and bounded `rows`. +Neither sleeper nor side is inferred from a filename. The preview returns +accepted/rejected/duplicate/conflict counts, bounded rejection examples, +metric identities, time interpretation, coverage endpoints and a content hash. +It writes nothing. + +```json +{ + "side": "left", + "subjectId": "sleeper-A", + "source": "wearable-export", + "timePolicy": { "timezone": "America/Los_Angeles", "disambiguation": "reject" }, + "rows": [ + { + "recordKey": "sample-123", + "metric": "heart_rate", + "value": 62, + "unit": "bpm", + "timestamp": "2026-10-10T23:00:00-07:00" + } + ] +} +``` + +Commit with `biometricTuning.commitImport`, passing the same payload as `import` +and the returned hash as `previewHash`. Changes to data, mapping, units, source +or time interpretation require another preview. When some rows were rejected, +commit additionally requires `acceptRejectedRows: true`. An import with no +accepted rows cannot be committed. Repeating the same content is idempotent. + +For a corrected full export, include `supersedesImportId` naming the latest +revision in the family. Corrections preserve source, sleeper and side; they +create a new immutable revision rather than editing old measurements. A stale +parent is rejected. Selecting an import ID always selects that exact revision. + +`listImports` pages at 50 imports using both `beforeCreatedAtMs` and `beforeId`. +`getImportMeasurements` pages at up to 500 rows using `afterId`. Follow +`nextCursor` to read the complete import. Preview summaries never contain the +full export. `deleteImport` explicitly removes its measurement rows and any +linked legacy chart rows, leaving a compact unavailable-import tombstone; +retrying the deletion is a no-op. A deleted import ID cannot supply readings. + +REST equivalents are under `/api/biometric-tuning/references`, with POST +`/preview`, POST `/commit`, GET `/`, GET `/{id}`, GET `/{id}/measurements` and +DELETE `/{id}`. All use the same service as tRPC. + +## Records + +Required fields: `recordKey`, `metric`, `value`, `unit`, and a time representation. +`recordKey` is the source's stable record identity. Repeated identical keys in +one batch are deduplicated; different measurements with the same key are all +rejected. Optional row `subjectId` must agree with the batch mapping. + +Use exactly one time representation: + +- Point: `timestampMs` (integer Unix milliseconds) or `timestamp` (ISO text). +- Interval: `startMs` plus `endMs`, or ISO `start` plus `end`. + +Intervals are half-open `[start, end)` and may span at most 24 hours. Unix +milliseconds must fall between 2001 and 2100; Unix seconds in millisecond fields +are rejected. ISO text includes seconds, optional milliseconds, and preferably +an explicit `Z` or `±HH:MM` offset. Invalid calendar dates are rejected. + +Naive local ISO times require a `timePolicy` with an IANA `timezone` or explicit +`utcOffsetMinutes`. Repeated DST times are rejected unless `disambiguation` is +explicitly `earlier` or `later`, or a valid offset is supplied. Nonexistent local +times are rejected rather than shifted. An explicit offset must agree with a +supplied timezone. An absent source timezone stays absent in provenance; it is +never inferred from the pod's current timezone. + +| Metric identifier | Units accepted | Meaning | +| --- | --- | --- | +| `heart_rate` | `bpm`, `beats/min`, `Hz` | Reference heart rate; Hz converts to bpm | +| `breathing_rate` | `breaths/min`, `br/min`, `Hz` | Reference breathing rate | +| `window_ibi_variability_v1` | `ms`, `s` | Pod window-level IBI variability index, not clinical RMSSD | +| `hrv_rmssd` | `ms`, `s` | Reference RMSSD, kept distinct from the pod index | +| `hrv_sdnn` | `ms`, `s` | Reference SDNN, kept distinct from RMSSD and the pod index | +| `hrv_unknown` | `ms`, `s` | Unspecified HRV definition; never directly scored | +| `sleep_stages` | `stage` | Interval label: `wake`, `light`, `deep`, `rem`, `asleep`, `in_bed`, `unknown` | + +Numeric values must be finite, nonnegative and within JavaScript's safe numeric +range after conversion. These are data representation constraints, not medical +normal ranges. Original submitted values and units are retained. + +Optional fields are `aggregation` (`instant`, `mean`, `median`, `window_index`, +`stage`), `windowMs`, `quality` (0–1), and `annotation` (up to 1000 characters). +The pod index requires `aggregation: "window_index"` and `windowMs: 300000` for +direct comparison. Matching HRV definitions with different or unknown windows +are not directly comparable. RMSSD/SDNN must never be scored as the pod index. + +Unknown, generic asleep and in-bed stages remain separate labels and are +unscored. Conflicting overlapping stage labels are preserved, but both entire +intervals are marked unscorable under the persisted `exclude_conflicts` policy. +Back-to-back intervals do not overlap. An annotation is untrusted data, never +an instruction, executable code or a command. + +## Text adapters and limits + +The shared `parseReferenceText` adapter accepts a JSON array of records or CSV +with the field names above as the header. CSV supports quoted commas, escaped +quotes and multiline annotations. Duplicate/unknown headers, malformed quotes +and mismatched column counts are rejected. CSV timestamps must retain their +explicit field units; there is no seconds/milliseconds guessing. + +Each upload is limited to 2 MiB, 2000 records, 12 JSON nesting levels and a +bounded node count. Rejection examples stop at 25, while aggregate counts are +complete. No server path, URL, shell command, archive or automatic cloud upload +is accepted. Hosts may parse a local export and send bounded normalized batches. +Do not log complete health exports. + +## Alignment semantics + +`reference-alignment-v1` uses chronological nearest-unused point matching with +a fixed tolerance (default 30 seconds). Each pod observation and reference is +used in at most one pair. Interval comparisons aggregate actual observations +inside the interval using the declared mean or median. Sparse references are +never forward-filled. Equivalent duplicate references do not count twice. + +Clock offset is explicit and fixed for the comparison; a training-derived +offset must remain frozen for validation/holdout. Reports include MAE, median +and 95th-percentile absolute error, signed bias, matched/unmatched counts, +reference and output coverage, exclusions and limitations. Point samples +contribute counts, not inferred duration. Interval duration counts only known +observation support and does not fill gaps. Incompatible HRV has no error score. +This alignment primitive does not by itself establish validated improvement. + +## Legacy Apple Watch clients + +The existing `biometrics.importReferenceStages` and `importReferenceVitals` +contracts still accept Unix seconds and return `{ written }`. They retain +first-write-wins heart-rate behavior and whole-request stage-overlap rejection. +Newly accepted rows and normalized provenance commit atomically; if provenance +cannot be saved, the legacy insert rolls back too. + +Old clients provide no subject identity or source timezone. Such imports are +visibly mapped to `legacy_unknown:left` or `legacy_unknown:right`, with that +limitation in the original record annotation. Older unlinked rows remain +legacy/unknown instead of receiving invented provenance. New clients should +use explicit sleeper mapping and the preview/commit workflow. + +### Stored-reading comparison + +`biometricTuning.compareReadings` (`POST /api/biometric-tuning/compare`) accepts an +exact `importId`, `side`, `subjectId`, metric, and a half-open `startMs`/`endMs` +interval of at most 24 hours. Optional `pointToleranceMs` (default 30 seconds) +and `clockOffsetMs` (default zero) are fixed, reported settings; this endpoint +does not fit an offset. It reads all stored observations in that interval, +without chart downsampling, and returns bounded summary scores, coverage, +source revision, and processing-run counts. A null processing-run ID means +`legacy_unknown`. `requestedMetric` identifies the export's metric; `metric` +identifies the pod metric actually available. Clinical HRV cannot be scored +against the pod's window IBI variability index. + +This is a read-only comparison of original readings. Its input hash identifies +the read snapshot but does not retain bytes or constitute a saved evaluation. +Only reference intervals fully inside the requested range are used, preserving +the original aggregation window. Stored readings do not record their support +duration, so comparisons report point counts and unknown duration (zero), +without filling gaps. A changed reference revision must be selected explicitly. + +The shared stage scorer uses intersecting interval duration, reports a confusion +matrix (rows reference, columns predicted), per-stage precision/recall/F1, +macro F1, coverage, and stage-duration/transition differences on common support. +Absent classes have null scores. Transitions across unobserved gaps are not +inferred. Conflicting intervals from separate sources remain unscored. This +scorer is a primitive for replay reports; it is not a new live stage classifier. diff --git a/docs/biometrics-replay-benchmark.md b/docs/biometrics-replay-benchmark.md new file mode 100644 index 000000000..d6684c9c8 --- /dev/null +++ b/docs/biometrics-replay-benchmark.md @@ -0,0 +1,666 @@ +# Initial synthetic replay measurement + +Measured on the requested pod (`192.168.1.88`, SSH port 8822) on October 12, +2026. This is an initial worker-cost measurement, not the final admission-policy +or end-to-end live-system qualification. + +| Property | Observation | +| --- | --- | +| Runtime | ARM64, four CPUs, Node 24.16.0 | +| Physical memory | 2,014,796 KiB | +| Available memory before / after | 1,440,260 / 1,425,340 KiB | +| `/persistent` available before | 3,741,108 KiB | +| One-minute load before / after | 0.54 / 0.36 | +| Input | 27 separate 12-hour nights, 720 minute readings each; 19,440 readings total | +| Input bytes | 2,374,215 | +| Configurations | Identical default/current; one unique numerical evaluation after deduplication | +| Child settings | nice 19, 128 MiB Node heap limit, 30-second external timeout | +| Evaluator elapsed time | 2,519.355 ms | +| Child peak RSS | 132,436 KiB (about 129 MiB) | +| Complete report size | 2,450,542 bytes | +| Core / piezo service status | Both active before and after | + +The shipped worker SHA-256 was +`16bab5135fcf0ba4d4a1624660f8565ba0b073cf39e7d13760df17347cd24d6a`. +Input was generated from the repository's processed-backtest fixture, expanded +to daily-separated nights with synthetic `60 + sin(minute / 10) * 8` BPM, +HRV index 30 and breathing rate 14. No personal recordings were transferred. +The worker and synthetic input/output existed only in a unique temporary pod +directory, which was removed after measurement. No live database, service +configuration or hardware controls were changed. + +The report exceeded the old 2 MiB inline limit. That concrete failure motivated +the large-report artifact path rather than reducing dataset size or dropping +traces. The worker was invoked directly for measurement; the old repository +would have rejected publication of this report. + +Still required before choosing final admission limits: three distinct candidate +configurations, repeated and worst-case inputs, wall-clock startup/serialization, +artifact I/O, CPU utilization, SQLite contention, ingest age, temperatures and +control/API responsiveness while replay runs. Service `active` status alone does +not prove those properties. This measurement does not cover waveform replay or +untrusted algorithm execution, and does not justify a universal speed claim. + +## Three-configuration follow-up + +On October 12, the same pod and worker hash completed three consecutive runs +with distinct default/current/candidate configuration hashes. Input remained +27 synthetic twelve-hour nights (19,440 readings). The deep-stage HR-ratio +thresholds were 0.92, 0.90, and 0.94 respectively. Other input and protocol +settings were unchanged, including assumed calibration quality zero. These +are distinct evaluations, but this fixture does not exercise every stage branch +or establish a worst-case runtime. + +| Run | Wall time, including child startup/output | Evaluator time | Peak worker RSS | JSONL output bytes | +| --- | --- | --- | --- | --- | +| 1 | 4,295.804 ms | 3,397.305 ms | 156,036 KiB | 3,675,631 | +| 2 | 4,163.752 ms | 3,252.410 ms | 141,956 KiB | 3,675,638 | +| 3 | 4,247.537 ms | 3,330.915 ms | 145,188 KiB | 3,675,631 | + +A Python harness sampled local read-only performance, sensor-source, and device +status HTTP endpoints approximately every half second (requests are sequential, +so request time extends the interval). It recorded only latency, frame age, +resource counters and thermal-zone-zero temperature. It used four seconds of +baseline and three seconds afterward. There were seven baseline samples, +twenty-three replay samples and four afterward; all requests succeeded. + +| Measurement | Baseline | During replay | After | +| --- | --- | --- | --- | +| Minimum available memory | 1,358,520 KiB | 1,215,752 KiB | 1,328,848 KiB | +| Median aggregate CPU busy | 23.78% | 56.65% | 22.22% | +| Maximum aggregate CPU busy | 28.76% | 76.68% | 29.92% | +| Maximum zone-zero temperature | 33.503 °C | 37.356 °C | 30.934 °C | +| Maximum performance GET latency | 34.92 ms | 328.04 ms | 26.11 ms | +| Maximum sensor-source GET latency | 103.29 ms | 45.32 ms | 27.32 ms | +| Maximum device-status GET latency | 51.58 ms | 108.24 ms | 74.00 ms | +| Maximum reported live-frame age | 457 ms | 514 ms | 418 ms | + +CPU busy is the change in `/proc/stat` non-idle counters across adjacent samples +in the same phase, aggregated across all four CPUs. It includes the running +services and measurement harness. The server's lifetime event-loop p95 remained +21.14 ms; that lifetime histogram cannot establish the replay interval's p95. +The sensor endpoint reports the newest frame of any sensor type, so this is not +proof of piezo-specific freshness. Temperature zones one through three were +unreadable; only zone zero was sampled. Status GET latency is not proof of +hardware-write or closed-loop control responsiveness. + +The harness read the worker's complete output file and extracted operational +counters; it did not run the production Node result parser, artifact publisher +or database transactions. Thus parent-process event-loop cost, persistent +artifact I/O, SQLite contention, larger/reference-heavy fixtures, waveform replay +and sustained-load qualification remain outstanding before final admission limits +are selected. Node ran at nice 19 with a 128 MiB heap and a 45-second external +timeout. Both services remained active afterward. Temporary synthetic input, +worker, harness and output were removed from the pod; no live configuration, +observations or hardware settings were changed. + +## Production runner and reference-heavy qualification + +A follow-up on October 12 used the production `DatasetStore`, `BacktestJobStore`, +child driver, result publication, and integrity-checked result reader against a +new synthetic SQLite database on the pod's persistent storage. The application +service and its databases were left untouched. The reusable harness is +`scripts/benchmarks/biometrics-runner.ts`; bundle it with esbuild for Node, leaving +`better-sqlite3` external, and pass a **new** directory, copied migration +folder, packaged worker path, and `mixed` or `dense` fixture name. Directory +creation is exclusive so an existing database cannot be selected accidentally. + +The tested worker SHA-256 was +`45547553cd456cba5051fe6e60e8562b5510ae32bbf8b92cfeb62c59cdeb59d1`. +The coordinator used Node's 384 MiB heap limit; its evaluator child retained +nice 19 and the 128 MiB heap limit. These are measurement settings, not yet an +approved deployment policy. + +The mixed fixture retained 20 twelve-hour nights, 14,400 vital readings, +3,600 movement rows, and 1,600 heart-rate/stage reference records. It includes +periodic missing HR readings, motion, and varying HR/HRV/breathing. Chronological +splitting evaluated 12 tuning nights. The three configurations have distinct +deep-stage HR-ratio thresholds; assumed calibration quality is explicitly one. +This synthetic fixture measures execution and does not validate biometrics. + +| Measurement | Run 1 | Run 2 | Run 3 | +| --- | --- | --- | --- | +| Coordinator execution through publication | 12,007.73 ms | 11,543.85 ms | 11,673.61 ms | +| Evaluator runtime | 5,202.12 ms | 4,962.23 ms | 5,048.52 ms | +| Evaluator peak RSS | 168,392 KiB | 167,180 KiB | 174,336 KiB | +| Integrity-checked result read | 2,286.37 ms | 2,256.85 ms | 2,267.35 ms | +| Coordinator event-loop maximum | 2,386.56 ms | 2,300.58 ms | 2,365.59 ms | +| Coordinator event-loop p95 | 22.64 ms | 22.95 ms | 20.92 ms | +| Coordinator process peak RSS (cumulative) | 257,564 KiB | 261,296 KiB | 262,292 KiB | + +All three attempts succeeded, produced 1,813,618-byte reports, and left the +synthetic canonical vitals unchanged. Dataset creation took 4,322.13 ms. These +reports remain below the 2 MiB inline threshold, so this workload does not +qualify the artifact path. Minimum available system memory during the replay +phase was 1,140,300 KiB and maximum zone-zero temperature was 39.457 °C. + +**Admission is not qualified by these results.** The coordinator's multi-second +synchronous work would block the application if wired into its main event loop. +Dataset construction and result reads have the same issue. Move these operations +into a bounded coordinator worker before enabling production execution, then +repeat the benchmark. A favorable p95 conceals the long individual stalls. + +The harness's own HTTP requests and piezo receipt callbacks run on that same +coordinator event loop. Their delays therefore cannot be attributed solely to +the application server or sensor ingestion. The separate standard-library probe +`scripts/benchmarks/biometrics-probe.py` measures loopback API latency from another +process. Neither probe retains API bodies or raw sensor samples. The Node +probe retains only sensor timing and resource measurements. This remains +synthetic, non-production-database testing: it does not establish live SQLite +contention, hardware-write responsiveness, waveform replay, or untrusted-code +isolation. + +### Large artifacts and independent API probe + +The dense fixture used the same worker with 19,000 synthetic vitals, no movement +rows, and 800 reference records across 20 nights. Twelve tuning nights produced +reports of 2,301,040, 2,301,041, and 2,301,034 bytes. All three stored result +descriptors were verified as `biometric-report-artifact-v1`, all result reads +passed integrity checks, and canonical synthetic vitals were unchanged. + +| Measurement | Run 1 | Run 2 | Run 3 | +| --- | --- | --- | --- | +| Coordinator execution through publication | 13,236.50 ms | 12,901.88 ms | 13,064.22 ms | +| Evaluator runtime | 5,501.95 ms | 5,455.62 ms | 5,499.04 ms | +| Integrity-checked artifact result read | 2,803.76 ms | 2,933.55 ms | 2,973.02 ms | +| Coordinator event-loop maximum | 2,961.18 ms | 2,780.82 ms | 2,885.68 ms | +| Evaluator peak RSS | 166,692 KiB | 161,240 KiB | 162,472 KiB | + +Dataset creation took 4,477.81 ms. Coordinator peak RSS reached 262,456 KiB; +minimum available system memory during replay was 1,105,904 KiB; zone-zero +maximum temperature was 40.274 °C. Piezo source-timestamp age seen by the +coordinator peaked at 4,728 ms during replay versus 2,584 ms in its baseline. +Because this callback shares the blocked coordinator loop, that measurement +still combines actual frame age with callback delay. + +The independent Python probe made 12 baseline samples, 251 samples spanning +fixture setup and all three attempts, and 9 afterward. Every GET returned 200. + +| Maximum HTTP latency | Baseline | Entire benchmark | After | +| --- | --- | --- | --- | +| Performance | 36.88 ms | 57.52 ms | 22.48 ms | +| Sensor source | 93.53 ms | 72.37 ms | 22.72 ms | +| Device status | 66.74 ms | 80.61 ms | 52.32 ms | + +The production service remained active after the benchmark. These observations +support keeping heavy coordination separate from the application event loop; +they do not approve running it on that loop. The harness's configuration labels +are synthetic comparison roles (thresholds 0.90, 0.92, 0.94), not a promotion +comparison against an asserted shipped-default profile. Repeat qualification +after coordinator isolation and again for the final deployed worker. The +measurements do not yet select universal admission thresholds or qualify live +SQLite contention and closed-loop control responsiveness. + +## Isolated coordinator follow-up + +The coordinator implementation at `0e5c87c0` was measured on the same pod on +October 12 using the dense fixture above. The optional final `COORDINATOR` +argument to `scripts/benchmarks/biometrics-runner.ts` routes dataset creation, +experiment creation, execution, and report reads through the shipped coordinator. +The caller uses the same 384 MiB heap cap as the preceding benchmark; each +coordinator has a 384 MiB cap and nice 19, and its evaluator retains 128 MiB/nice +19. Each operation gets a fresh process. The worker hash remains +`45547553cd456cba5051fe6e60e8562b5510ae32bbf8b92cfeb62c59cdeb59d1`. + +An initial isolated run succeeded three times. The repeat below adds a timer +turn after report serialization, so its event-loop maximum includes final IPC +deserialization and JSON serialization as well as execution/publication. This +is a calling-process measurement, not the production application's event-loop +histogram. Both runs used exclusively created synthetic databases and retained +no personal recordings or API response bodies. + +| Measurement | Run 1 | Run 2 | Run 3 | +| --- | --- | --- | --- | +| Execution through publication | 14,955.19 ms | 15,390.62 ms | 15,141.36 ms | +| Integrity-checked full report read | 4,178.71 ms | 4,418.67 ms | 4,237.17 ms | +| Calling-process event-loop maximum | 144.83 ms | 187.83 ms | 174.59 ms | +| Calling-process event-loop p95 | 21.22 ms | 21.04 ms | 21.12 ms | +| Calling-process cumulative peak RSS | 153,428 KiB | 156,940 KiB | 156,940 KiB | +| Evaluator peak RSS | 164,460 KiB | 171,136 KiB | 162,032 KiB | + +All three attempts succeeded; full reports remained approximately 2.30 MB and +canonical synthetic vitals were unchanged. Dataset construction took 5,693.98 ms +with a calling-process event-loop maximum of 56.30 ms. During replay, minimum +available system memory was 1,046,688 KiB, maximum zone-zero temperature was +41.909 °C, and maximum piezo source timestamp age was 2,756 ms. This does not +measure the coordinator's peak RSS; the memory values above must not be summed +as a complete process-tree maximum. + +The independent probe recorded 12 baseline samples, 291 spanning the complete +benchmark, and nine afterward. Every request returned HTTP 200. + +| Maximum HTTP latency | Baseline | Entire benchmark | After | +| --- | --- | --- | --- | +| Performance | 30.48 ms | 160.12 ms | 38.15 ms | +| Sensor source | 27.47 ms | 109.92 ms | 24.23 ms | +| Device status | 65.74 ms | 159.46 ms | 76.02 ms | + +Isolation removed the multi-second calling-loop stalls observed previously, +but full-report IPC/serialization still transfers large trace arrays. In one +retained synthetic report, `nights` occupied about 2.21 MB; the first evaluated +window's `blocks` alone occupied 56,786 bytes. A bounded summary for interactive +review, with exact traces fetched through the existing paginated endpoint, is +an appropriate next optimization. Keep the complete immutable report and its +hash; a transport projection must not silently redefine the retained artifact. + +These observations still do not qualify live-database write contention, +closed-loop hardware responsiveness, waveform processing, or arbitrary-code +isolation. No production resource policy or automatic background execution was +enabled by this benchmark. Temporary benchmark databases and binaries were +removed after measurements were copied locally; the production service was +left running without a deployment or restart. + +## Retained waveform pipeline on the pod + +The October 12 follow-up used the coordinator and Node evaluator built from +`fe25e71a`, plus the Python process-limit compatibility fix accompanying this +measurement. The pod's Python 3.10.4 installation omits the optional `resource` +extension. Workers now use `/usr/bin/prlimit` on Linux when that extension is +missing, failing startup if the utility fails. A separate child on the pod +confirmed CPU soft/hard limits of 120/121 seconds, core limits of zero, and +open-file limits of 64/64 in `/proc/self/limits`. + +The new `scripts/benchmarks/biometrics-waveform-fixture.py` creates deterministic +500 Hz dual-channel sine-plus-noise CBOR archives in an exclusively created +directory. The runner `biometrics-waveform-runner.ts` creates a separate database, +retains those archives through the production publisher, enqueues three unique +configuration hashes, executes the leased waveform pipeline, and reads its +summary. The first five minutes are retained pre-roll. Configuration labels +are synthetic roles; this fixture does not measure biological accuracy. + +The 30-minute fixture contained 5,976,451 compressed bytes. Dataset creation +took 1.309 seconds and retention took 15.809 seconds. Execution reached its +180-second coordinator limit before publishing a result. Canonical synthetic +observations were unchanged. **This workload is not qualified for automatic +execution.** Full-night execution needs a different scheduling/budget design; +simply enabling the existing host would not make these jobs complete. + +The independent probe sampled the benchmark process and its descendants through +`/proc`, in addition to the existing loopback HTTP measurements. RSS is a sampled +sum, not an exact peak or unique physical-memory accounting. The failed +30-minute run produced 12 baseline, 448 benchmark, and nine post-run samples. +All sampled HTTP responses returned 200. + +| Measurement | During the complete 30-minute-fixture benchmark | +| --- | --- | +| Maximum sampled process-tree RSS | 313,332 KiB | +| Minimum available system memory | 1,181,144 KiB | +| Maximum zone-zero temperature | 40.041 °C | +| Maximum performance GET latency | 49.01 ms | +| Maximum sensor-source GET latency | 79.46 ms | +| Maximum device-status GET latency | 115.39 ms | +| Maximum age of piezo WebSocket receipt | 1,446 ms | + +The receipt age is measured at the benchmark client; it is not the age of the +original sensor timestamp. The calling process's largest sampled scheduling +delay was 179.88 ms, so favorable HTTP timings alone do not establish its event +loop budget. Measurements use synthetic databases and do not qualify live +SQLite write contention or hardware-command latency. No production admission +policy is selected by these results. + +A 10-minute follow-up (five minutes pre-roll and five minutes scored) completed +the entire pipeline with the same three configurations. Its 1,992,541 compressed +bytes took 15.599 seconds to retain. Leased replay and publication took 83.402 +seconds, and integrity-checked summary retrieval took 1.289 seconds. The summary +was 28,798 bytes; synthetic canonical observations again remained unchanged. +The replay caller's event-loop p95/max were 22.00/63.90 ms, including a timer +turn after the coordinator reply. This is a successful short fixture, not a +qualified whole-night workload. + +The independent probe recorded 12 baseline, 247 benchmark, and nine post-run +samples, all with HTTP 200 responses. During the benchmark, process-tree RSS +reached 314,852 KiB, available memory stayed at or above 1,184,736 KiB, and +zone-zero temperature reached 39.924 °C. Maximum performance/sensor-source/device +GET times were 155.93/96.78/75.31 ms. Maximum piezo receipt age was 1,626 ms. + +The successful run pinned compound build +`b91e8552bc6d973922bd38808d04490683e3e05d8c67c9af5ad37d40a64bfe45` +and Python build +`fad13cfff0b361fbec950e0b6ddd892fe3d97f42f3ef9c5031af2e700c579aca`. +Its complete report hash was +`8446b1a8fd4c0f1530c25ed186ac5231c8020c60e4dffc42b1ec248aa2b46291`. +Python used the pod's installed NumPy 2.2.6 and SciPy 1.15.3. The staged +benchmark used fixed production worker bundles, not the temporary diagnostic +coordinator used to investigate the missing Python extension. + +After both runs, no benchmark descendants remained. Synthetic telemetry and +logs were copied locally, and the staging directory was removed. Both installed +services remained active on the original `homekit-0b53800` release. No deployment, +restart, live-database modification, or hardware command was part of this test. + +### Where waveform execution time goes + +A subsequent synthetic 30-minute profile on the same pod separated startup +from processing. The Yocto Python installation also lacks `_lsprof`, so the +diagnostic used `time.monotonic()` and `time.process_time()` around the shipped +functions. These timings include small wrapper overhead and are not a complete +function profiler. It retained the same noisy dual-channel fixture in a new +directory and discarded derived readings after counting them. + +| Operation | Wall time | CPU time | +| --- | --- | --- | +| Import replay and numerical dependencies | 12.726 s | Not measured | +| Retain the synthetic archive | 0.369 s | Not measured | +| Complete one-configuration retained replay | 45.184 s | 44.751 s | +| Inner CBOR decoding (included above) | 0.596 s | 0.583 s | +| Per-side ingestion (included above) | 41.623 s | 41.238 s | +| Numerical estimates (included in ingestion) | 30.365 s | 30.082 s | + +The run consumed 1,860 records including 1,800 piezo frames and produced 50 +scored readings plus ten pre-roll readings. These overlapping timings must not +be summed. Both startup and numerical processing matter; archive decoding was +not the dominant cost. A full-night run still needs direct measurement rather +than extrapolating this short profile into a production runtime guarantee. + +### Interrupted eight-hour qualification (2026-10-13 UTC) + +The three-configuration, eight-hour synthetic run on `192.168.1.88:8822` +was cancelled during its third configuration after systemd explicitly stopped +`sleepypod` at 01:09:44 UTC. Core and piezo services restarted at 01:10:17 UTC; +`NRestarts=0` and the journal showed the stop request and SIGTERM. This task did +not issue that stop. The installed release remained `homekit-0b53800`. +The admission monitor withdrew permission when HTTP health disappeared. This +run is **not a completed qualification**, regardless of the operation timer's +`succeeded` field (which only records that the execution RPC returned). + +Before cancellation, the benchmark recorded 293 concurrent descriptor reads, +zero read failures, read p95 1520 ms / maximum 2077 ms, 5962 HTTP probes with six +failures at shutdown, HTTP p95 42 ms / maximum 937 ms, at least 1,045,213,184 bytes +available memory, maximum temperature 44.594 C, maximum piezo receipt age 1768 ms, +and peak benchmark process-tree RSS 396,940 KiB. Canonical observations remained +unchanged. These observations support cancellation behavior but do not establish +a production resource policy or a successful full-night replay. + +Synthetic measurements and probe metadata were copied locally for analysis. +After confirming all benchmark descendants had exited, its exact staging directory +was removed; both installed services were active. Repeat full qualification in an +uninterrupted window before enabling the production replay host. + +Retained RAW cleanup runs through the coordinator's serialized work lane when +its host performs startup/periodic collection. It uses a fixed Python module, +server-derived database/storage paths, a two-minute deadline, bounded diagnostics, +and a count-only response. Admission is checked before execution and every five +seconds; host shutdown cancels the child. Released artifacts and unpublished +orphans retain their one-hour grace periods and cross-process publication locks. +An absent retained directory is a no-op. Enabling the production host still +requires deployment qualification; adding cleanup does not enable replay. + +### Per-run process measurements + +New retained waveform runs include `resources` with version +`waveform-process-resources-v1`. Monotonic wall time and process CPU time cover +input reading, build verification, replay, and the final build check through +result generation. They exclude interpreter startup before `run`, final result +serialization, and teardown. Peak RSS is the kernel process-lifetime high water +mark: Linux `/proc/self/status` `VmHWM` converted from KiB to bytes (including +Yocto without Python's `resource` extension), or Darwin `ru_maxrss` in bytes. +Unavailable telemetry is null; current RSS is never substituted for a peak. + +Waveform reports summarize the sequential processes as total measured wall/CPU +time and the largest individual process memory peak. Shared configuration hashes +are executed once and are counted once. Missing legacy measurements make the +corresponding aggregate unavailable. Compact report projections retain these +measurements, and the review UI labels their units and scope. They exclude Node +coordination/scoring and do not measure live ingestion health, so they cannot +alone satisfy a whole-job operational promotion policy. Execution measurements +are part of the retained report hash; replay determinism is assessed on derived +readings rather than equality of timing counters between executions. + +Successful jobs also retain `jobOperations` (`backtest-job-operations-v1`), +written by the coordinator after its final admission check rather than supplied +by the scoring child or experiment. Its monotonic duration runs from initial +admission through publication preparation. The supervisor sends ingestion age +from the same fresh health sample used to admit work over the internal process +channel. Only count, maximum observed age, largest monotonic observation gap, +and qualification identity are retained; raw sensor frames are not included. + +`allChecksMeasured` means every admission check supplied evidence under one +qualification identity, with at least initial and final checks. It does not mean +continuous monitoring: the nominal check interval is five seconds, delays are +reported, and intervening health is unobserved. Boolean-only admission callbacks +used by older tools/tests remain supported but produce unavailable health +evidence. A failed or invalid health callback cannot fall back to a permissive +boolean callback. Admission withdrawal still interrupts work without publishing +a result. The waveform report UI shows these observations separately from Python +process costs. Whole-job memory evidence and designated promotion policy remain +separate requirements. + +The supervisor now also samples summed RSS for the owned replay process group +at a nominal one-second interval, plus admission boundaries. Linux procfs scans +are bounded to 4,096 process IDs and 64 group members with at most 16 concurrent +reads. Only group IDs and RSS are read; arguments, environments and sensor data +are excluded. Missing procfs, unsupported platforms, malformed data, or exceeded +bounds yield unavailable samples. Exited processes may disappear during a scan. + +The final admission receipt carries this accumulator into `jobOperations.memory` +(`process-group-rss-observations-v1`): attempts, valid sample count, largest +observed summed RSS and largest observation gap. The coordinator and its worker +children share the owned group; the main application process is outside it. +The sampler stops when the coordinator closes, with no detached polling loop. +This is not an exact kernel peak: short-lived processes or peaks between scans +may be missed, scans are not atomic, and shared pages may be counted more than +once. Reports retain these scope limitations rather than substituting this value +for the per-process lifetime high-water marks. + +### Completed eight-hour synthetic qualification (2026-10-13 UTC) + +The uninterrupted run on `192.168.1.88` completed all three distinct +configurations over the 28,800-second dual-channel 500 Hz synthetic fixture +(95,621,299 compressed bytes). Staged production code was `e1ab3942`, with the +completion-validating benchmark runner from `2ff9bf58`. Completion required a +successful job, retained waveform report, matching dataset/Python identities, +three unique expected configuration hashes, and positive scored outputs for each. +Canonical synthetic observations remained unchanged. + +- Replay: 2,625,664 ms (43.8 minutes); retention: 22,064 ms; summary: 2,849 ms. +- 366 concurrent dataset reads, zero failures; p95 1,516 ms, maximum 1,867 ms. +- Internal probe: 7,456 API requests, zero failures; p95 41.5 ms, maximum 311 ms. +- External probe: 16,905 API requests, zero failures; p95 38.6 ms, maximum 1,429 ms. +- Peak observed benchmark process-tree RSS: 413,163,520 bytes (394 MiB). +- Minimum sampled available memory: 1,132,576,768 bytes; maximum temperature 43.777°C. +- Maximum sampled ingestion age: 1,250 ms. In 68 samples a newer frame arrived + after the collector captured its start timestamp, producing a negative age; + those are collection-timing artifacts, not measured negative latency. +- Replay parent event-loop p95 23.1 ms, maximum 96.7 ms. + +Retained report hash: +`bc1ac53d9168a5f2e6fd521212c69f6ae53939911e7ba17e4f1ccb0fb8e7cdcc`. +Compound build: +`375b2ae0e0e81545b234f1ee3357a2b0cd4cde222cd413591c71c88766b75607`. +Python build: +`23670017f01a37b0281fdea016440261af6f9e6519bff5f294b9056d6f5caf29`. + +Synthetic measurements, probe output, and compact summary were retained locally +under `/tmp/biometrics-waveform-qualification-results/fullnight-success/`. +The owned pod staging directory was removed only after successful termination +and a descendant-process check. Core (`sleepypod.service`) and piezo services +remained active and the health API returned 200 afterward. These results qualify +the stated staged build and fixture; later telemetry and objective changes need +additional target verification. Synthetic signal recovery does not establish +agreement with real reference exports or clinical accuracy. + +A subsequent target smoke test exercised the new process-group sampler against +an isolated Node child holding a touched 32 MiB buffer. Four of four observations +were available; peak observed RSS was 83,582,976 bytes, maximum observation gap +999.8 ms. The child exited and the temporary sampler directory was removed; +both application and piezo services remained active. This verifies Linux +collection and cleanup, not a repeat full-night qualification of newer code. + +### Frozen sampled operational objectives + +Numeric objective policies may explicitly opt into +`operational.observations.version = sampled-process-group-v1`, with maximum +memory and ingestion observation gaps. Without that opt-in, sampled RSS cannot +satisfy the existing peak-memory field. After the final admission check, the +coordinator replaces the scorer's operational evidence with its own measured +job duration, group RSS, and ingestion observations. Missing/partial samples +remain unavailable; excessive observation gaps or a qualification identity +mismatch prevent improvement. Threshold values and sampling semantics remain +part of the frozen experiment protocol. This supplies operational evidence; +a designated qualified policy, sufficient independent reference support, and +subgroup checks are still required for validated promotion. + +### Current-build 30-minute smoke (2026-10-13 UTC) + +An isolated run of `4453c0a5` on the same aarch64 pod (Node 24.16.0, locked +Python 3.10.4 environment) completed three distinct configurations over an +1,800-second dual-channel 500 Hz synthetic fixture, 5,976,451 compressed bytes. +It used the current packaged coordinator/evaluator and common Python sources, +including numerical flock, shared application admission and process-group cleanup +confirmation. Production services continued running; this was not a deployment. + +- Replay 202,760 ms; retention 16,172 ms; summary 1,542 ms. +- 27 concurrent dataset reads, zero failures; p95 1,860 ms. +- 638 internal API requests and 1,545 independent probe requests, zero failures; + respective p95 latencies 65.7 ms and 41.6 ms. +- Peak sampled benchmark process-tree RSS 433,369,088 bytes (413 MiB). +- Minimum available memory 992,690,176 bytes; minimum free disk 3,450,945,536 bytes. +- Maximum temperature 46.696°C; maximum measured ingest age 992 ms. Nine negative + ages reflect collector/frame timing, as in the earlier full-night run. +- Replay parent event-loop p95 28.0 ms, maximum 186.4 ms. + +Completion verifies a retained report, exact dataset/Python/configuration identities, +and positive scored outputs for each configuration. Synthetic canonical rows +remained unchanged. After termination, no process referenced the owned staging +directory; it was removed. Core and piezo services remained active and the health +API returned 200. No production databases, bindings or service configuration changed. + +The [compact measurement summary](performance/biometrics-replay-2026-10-13/smoke-4453c0a5.json) +records exact result/build hashes, explicit test budgets and SHA256 identities of +the inputs and telemetry. Full synthetic telemetry is retained locally at +`/tmp/biometrics-qualification-4453c0a5/`. These are sampled observations, not exact +kernel peaks or continuous guarantees. They verify this short replay workload; +full-night current-build, representative-reference and prospective shadow +qualification remain outstanding. The measured event-loop maximum also prevents +interpreting the admission samples as a guarantee that all delays stayed below +the benchmark's 100 ms admission threshold. No promotion registry or production +budget defaults were derived from this smoke alone. + +### Synthetic shadow supervisor measurement harness + +`scripts/benchmarks/biometrics-shadow-runner.ts` exercises the shipped Node +supervisor and kernel-limited Python shadow child with generated 500 Hz dual +channels. It retains bounded resource receipts, partial failure observations, +output identity, exact Python/runner identities and parent event-loop measurements. +It uses no live waveform subscription, database, credentials or hardware API. + +Bundle with esbuild (`--bundle --platform=node --target=node22 --format=cjs`), +then run on the target Linux pod with arguments `OUTPUT_JSON PYTHON_PATH +MODULES_PATH [SECONDS]`. The default is 1,800 synthetic seconds, processed as +fast as the worker permits; accepted durations are 150–28,800 seconds. Use the +locked installed Python environment and the source modules matching the build. +Wrap it in `biometrics-probe.py` for independent live-service telemetry. Run only +after other replay/shadow measurements finish: the child uses their same kernel +execution lock. The harness's explicit ceilings are measurement stop conditions, +not installed qualification or recommended deployment limits. + +This measures synthetic processing and supervision only. It does not establish +live capture/sink performance, representative artifact robustness, held-out +accuracy or permission to install a shadow qualification registry. + +### Current worker eight-hour run (2026-10-13) + +The current worker/coordinator bytes completed three configurations over 28,800 +synthetic seconds at 500 Hz on the same four-core ARM64, 2 GiB pod. The staged +source was `db52f1da`; subsequent API/UI/prompt changes leave the measured worker +and coordinator hashes unchanged. Exact hashes and artifact identities are in +[the compact evidence](performance/biometrics-replay-2026-10-13/fullnight-db52f1da.json). + +- Replay: 2,756.7 seconds (45.94 minutes); retention: 21.74 seconds; summary: + 2.99 seconds. All operations completed and fixture canonical rows were unchanged. +- 363 concurrent report reads: no failures, p95 1,943.7 ms, maximum 2,183.3 ms. +- 7,390 internal HTTP observations: no failures, p95 57.1 ms, maximum 300.8 ms. + The independent probe recorded 17,568 observations, also without failures, + p95 39.6 ms and maximum 500.1 ms. +- Peak sampled process-tree RSS: 443.1 MiB; minimum available memory: 902.6 MiB; + minimum filesystem space: 3.02 GiB; maximum temperature: 49.965 °C. +- Aggregate CPU busy averaged 68.1%, peaking at 99.0% over a sampling interval. + Maximum one-minute load per CPU was 1.0575. These include live services and + the probe, not just the replay child. +- Maximum observed piezo arrival age: 1,560 ms. There were 71 negative age + differences because the collector records its start before asynchronous reads + and can observe a newer frame afterward; those are not negative ingest latency. +- Parent event-loop p95 during replay: 27.28 ms, maximum 154.01 ms. A 100 ms + admission check is therefore not a continuous event-loop latency guarantee. + +The original process handle exited successfully and no owned worker descendants +remained before the separate shadow measurement started. Core and piezo services +were active afterward. The read-only HTTP probes do not prove actuator-write +latency or clinical performance. Raw measurements remain under +`/tmp/biometrics-qualification-db52f1da/` on the measurement workstation; the +committed evidence records their SHA-256 hashes. No live database or profile was +changed by this benchmark. + +### Shadow supervisor measurement (2026-10-13) + +After the full-night replay released its worker slot, the staged shadow harness +processed 1,800 synthetic seconds at 500 Hz through the shipped Linux process +wrapper and Node supervisor. It completed in 61.34 seconds, producing 30 outputs +and reporting 29.17 seconds of numerical CPU time. The child process-group RSS +peaked at a sampled 113.15 MiB; the independent probe observed 256.5 MiB for the +whole benchmark tree. The completion receipt retains cold-start and insufficient +preroll limitations. + +The probe recorded 456 read-only HTTP observations with no failures (p95 45.41 ms, +maximum 130.39 ms). Minimum available memory was 1,019.5 MiB and maximum temperature +47.513 °C. Parent event-loop p95 was 26.10 ms and maximum 68.62 ms. RSS observation +gaps reached 333.42 ms; this is sampled memory evidence, not an exact kernel peak. + +[The compact receipt](performance/biometrics-replay-2026-10-13/shadow-4ee74c25.json) +retains build identities, completion/cleanup observations, limits and raw-artifact +hashes. The original SSH handle exited zero, no owned descendants remained, and +both live services were active before the owned staging directory was removed. +This run does not exercise the live source/database sink, establish held-out +accuracy or authorize a shadow qualification registry. + +### Target-specific replay deployment policy + +The measured pod uses the [recorded policy](performance/biometrics-replay-2026-10-13/deployment-policy.json). +Its admission floor is 768 MiB available memory and 2 GiB free disk; it refuses +load per CPU above 1.1, temperature above 55 °C, piezo age above five seconds, +stale health samples above two seconds, or a scheduling probe above 100 ms. +These are conservative operating cutoffs around the observed workload, not +universal hardware defaults or guarantees that every transient stays below them. +A temporary refusal is expected when current health exceeds a cutoff. + +The whole-job deadline is two hours; each numerical waveform child has a +30-minute wall deadline and 1,200 CPU-second limit, matching the successful +full-night measurement. Larger jobs can still exceed these ceilings and fail +explicitly. Existing worker serialization, process limits, quotas and health +checks remain enforced. This policy authorizes bounded replay on this measured +pod only; it provides no accuracy designation or shadow qualification. + +Deployment stores the two environment values in +`/etc/sleepypod/biometrics-replay.env`, loaded by +`/etc/systemd/system/sleepypod.service.d/biometrics-replay.conf`. A service restart +is required after changing them. Remove that drop-in and restart to return to +unconfigured admission. No user profile or hardware temperature setting changes +when installing this policy. + +### Live source and isolated sink measurement (2026-10-13) + +The [live operational receipt](performance/biometrics-replay-2026-10-13/live-shadow-operational.json) +records a successful 180-second observation through the loopback sensor stream, +read-only active observations, shared numerical worker and fenced isolated sink. +It retained six comparison points (5,396 bytes), with zero canonical rows in the +isolated database. The worker used 4.95 CPU seconds, sampled peak group RSS of +108.61 MiB and a maximum RSS observation gap of 497.24 ms. The independent probe +made 1,587 read-only HTTP requests with no failures (p95 40.21 ms, maximum +123.67 ms); minimum available memory was 1,082.67 MiB and maximum temperature +44.828 °C. No raw frames or numerical readings were exported in this receipt. + +Two earlier attempts failed: one refused health admission; another reported +unavailable RSS after its worker completed. The latter exposed a process-exit +race fixed in `4c3a6a6e`: disappearance after a validated terminal response now +requires bounded, successful pipe closure. Missing RSS during processing, bad +exit status and a worker that never exits still fail. Regression tests reproduce +all three outcomes. The successful repeat used the fixed supervisor without +relaxing resource limits. Its SSH process exited zero, owned processes were +absent, and all three owned staging directories were removed after saving +metadata receipts locally. + +This trusted measurement harness uses read-only production handles and a fresh +isolated database. It does not install a qualification registry, alter profile +bindings or exercise production registry admission. Cold start, insufficient +preroll, missing occupancy context and missing pump context remain recorded; +this operational check cannot establish accuracy or authorize promotion. diff --git a/docs/biometrics-runtime-activation.md b/docs/biometrics-runtime-activation.md new file mode 100644 index 000000000..77d11fcbb --- /dev/null +++ b/docs/biometrics-runtime-activation.md @@ -0,0 +1,142 @@ +# Biometric runtime activation protocol + +`requestManualActivation` is the configuration transaction for an explicitly +unvalidated manual change. It checks side, exact revision/hash, expected binding +generation and idempotency key before committing the binding, audit event and +immutable outbox payload together. A missing legacy migration prevents a new +binding from discarding existing stage personalization. Reset selects the +current shipped default; rollback selects the previous binding and remains a +distinct audited action. Neither operation deletes revision history. + +An explicit reset can recover malformed legacy stage data that cannot pass the +stage-classifier schema. The original legacy row remains untouched, and the +reset event preserves its value as `evidence.preservedLegacyClassifier` with +`recovery: "invalid_legacy_reset"`. This is audit data, never an executable +profile revision. The binding, evidence and default outbox payload commit +atomically; a publication failure leaves the original state intact. Valid +legacy customizations still require migration before any new binding, including +reset. Recovery does not claim processor adoption. + +For a bound revision whose engine is unavailable, the web recovery view shows +its preserved revision ID and the current default recovery hash. It cannot +produce a valid before/after diff of the unavailable engine and explicitly says +so. The user can request an acknowledged, generation-checked reset to defaults; +the old revision remains the previous binding. Rollback to that revision still +requires its engine to be compatible and never silently substitutes defaults. + +`projectRuntimeConfig` validates the manifest/configuration identity and writes +an immutable side/generation projection into the biometrics database. Duplicate +identical delivery succeeds; a different payload for an existing generation +fails. `deliverRuntimeOutbox` acknowledges delivery only after projection. +Transient failures retry with capped exponential delay; invalid envelopes fail +visibly. These are separate database transactions, so a crash after projection +is handled by idempotent redelivery rather than assumed cross-database atomicity. + +`repairCurrentRuntimeProjections` restores the latest authoritative delivered +payload after biometric history has been cleared. It preserves configuration +ownership and does not manufacture runtime receipts. + +`reconcileActivation` considers only receipts matching the desired generation +and exact bundle hash. Heart-rate, breathing-rate, HRV-index and sleep-stage +components must all report applied with an effective timestamp before the +activation is marked applied. Old-generation or wrong-hash receipts cannot +acknowledge a newer binding. A final generation check fences concurrent intent +changes while reconciling. + +The Node startup hook starts one `globalThis` coordinator after migrations. +Every five seconds it migrates legacy stage personalization, repairs delivered +projections, delivers pending intents and reconciles receipts. Shutdown stops +its timer. Public manual activation, reset and rollback submit exact intent; +status polling remains read-only. Availability does not assert processor health +or adoption. The cross-runtime fixture verifies the public request → Python +boundary → stage receipt → restart → rollback handoff. +There are no hardware-command paths in these services. + +The Python `common.biometrics.runtime` reader independently validates the latest +projected payload, its envelope, shipped manifest and resolved hash. Reads are +bounded before materializing SQLite text. Missing tables, corrupt newest +generations and incompatible engines surface errors; the reader neither creates +schema nor silently selects an older generation. Snapshots retain immutable +configuration bytes and return fresh decoded values. + +Its two-phase boundary selector allows immediate changes at the caller's next +sample boundary and next-session changes only when occupancy is explicitly +false. Unknown occupancy, including an uninitialized restart, does not count as +an empty bed. Newer requests supersede pending ones, stale generations fail, and +a late commit cannot acknowledge a superseded selection. The caller must install +the selected configuration before committing selection and writing receipts. +The live piezo daemon now uses these primitives through `runtime_adapter.py`. +It polls projections every five seconds, installs configurations before ingesting +an accepted dual-channel frame, and writes receipts only after both channel +buffers have accepted that frame. Heart-rate, breathing-rate and HRV-index +receipts are separate from the stage-view consumer's acknowledgment. Heartbeats are +bounded to once per thirty seconds unless the applied configuration changes. + +In single-sleeper mode both physical channels use the home side's profile. +Each computed candidate carries its immutable processing context through +quality arbitration and write retries, including its original logical side and +physical source channel. The observation, quality row and processing bundle +commit in one SQLite transaction. Retained contexts recreate their processing +bundle after history reset. This does not imply raw input retention: provenance +explicitly labels raw identity unavailable and timestamps as processing-clock +based. Measured sensor calibration is not consumed by this numerical path and +its calibration hash remains null. + +Numerical changes reset sample buffers, tracking and presence state; stage-only +changes preserve numerical state. Routing changes reset state to prevent mixing +sleepers. A restart restores only a compatible projection previously acknowledged +by all three numerical components, with fresh estimator warmup. Pending +next-session requests still require known empty occupancy. Invalid projections +retain the actual compatible configuration and report degraded module health; +an incompatible restored configuration falls back to defaults. Missing schema +prevents custom adoption and preserves legacy writes. The coordinator records +fallback intent separately from the consumer receipts proving its adoption. + +The stage reader selects the newest projection for which all three numerical +components acknowledged the exact bundle hash and an effective boundary. Partial +adoption of a newer generation keeps the prior view. Reading the classifier for +use in the client or server records the stage-consumer receipt; merely inspecting +profile overview or enqueueing a backtest does not. No stage receipt is generated +by the background coordinator. Until a consumer requests the new view, the +activation can truthfully remain pending. + +`getClassifier` exposes the applied classifier and provenance separately from a +pending desired classifier. The editor retains pending edits and polls for +adoption. `getSleepStages` includes the selected view with classified results; +its interpretation is explicitly `current_classifier_view`, since the rules +rescore original readings rather than replace historical observations. Backtest +current-stage baselines use this same applied view. + +Legacy `setClassifier` writes now use the profile service and durable outbox. +They retain HR, breathing and HRV overrides, and null clears only stage overrides. +The response retains classifier/isDefault fields and adds pending activation +identity. Existing legacy columns remain migration baselines, never a second +writable authority. These changes are manual/unvalidated. Demo responses label +simulated adoption explicitly. + +`recoverRuntimeIntent` handles obsolete defaults, missing intent and unavailable +pinned engines. It publishes a new generation and audit event atomically with +its outbox payload. Fallback preserves the desired pinned revision, immutable +revision history and the previous rollback target. Repeated recovery passes are +idempotent. If that same pinned engine becomes compatible again, restoration +waits for the next session. Repair of a lost custom boundary also defaults to the +next session. The fallback explanation survives acknowledgment and remains +visible in the stage view and desired-profile warning. + +A conflicting biometric projection is retained for diagnosis and its delivery +marked failed. The next recovery pass creates a new generation; it does not +rewrite the conflicting immutable snapshot. Missing projections after history +reset are simply redelivered with their original identity. + +Profile overview includes numerical processor health separately from historical +adoption. All three components must report the same generation/hash and a +heartbeat no more than 120 seconds old to be current. Missing components, +incomplete adoption, failures, future timestamps and mixed generations are +reported explicitly. The retrospective stage consumer is on demand and is not +subject to that continuous heartbeat test. This reports health; it does not +claim an old activation was never applied or automatically switch algorithms. + +`biometricTuning.listActivationHistory` / `GET /biometric-tuning/activations` +returns side-scoped audit events, fifty per page using a generation cursor. +Results include exact hashes, revision identities, evidence and fallback reason; +internal request hashes and idempotency keys are omitted. diff --git a/docs/performance/biometrics-replay-2026-10-13/deployment-policy.json b/docs/performance/biometrics-replay-2026-10-13/deployment-policy.json new file mode 100644 index 000000000..c19bec11f --- /dev/null +++ b/docs/performance/biometrics-replay-2026-10-13/deployment-policy.json @@ -0,0 +1,17 @@ +{ + "admission": { + "qualificationId": "pod-192-168-1-88-replay-20261013-bda8d03d", + "minimumAvailableMemoryBytes": 805306368, + "minimumFreeDiskBytes": 2147483648, + "maximumLoadPerCpu": 1.1, + "maximumEventLoopDelayMs": 100, + "maximumPiezoAgeMs": 5000, + "maximumTemperatureC": 55, + "maximumSampleAgeMs": 2000 + }, + "budget": { + "jobWallMs": 7200000, + "waveformWallMs": 1800000, + "waveformCpuSeconds": 1200 + } +} diff --git a/docs/performance/biometrics-replay-2026-10-13/deployment-verification.json b/docs/performance/biometrics-replay-2026-10-13/deployment-verification.json new file mode 100644 index 000000000..61be2d857 --- /dev/null +++ b/docs/performance/biometrics-replay-2026-10-13/deployment-verification.json @@ -0,0 +1,100 @@ +{ + "verifiedAt": "2026-10-13T14:14:45.388115+00:00", + "build": { + "branch": "feat/biometrics-calibration", + "commitHash": "4c3a6a6e", + "commitTitle": "fix(biometrics): handle shadow RSS disappearance during clean exit", + "buildDate": "2026-10-13T14:12:14.467Z", + "version": null + }, + "admission": { + "httpStatus": 200, + "result": { + "configured": true, + "qualificationId": "pod-192-168-1-88-replay-20261013-bda8d03d", + "admitted": true, + "reasons": [] + } + }, + "manifest": { + "httpStatus": 200, + "result": { + "manifestHash": "96c2f01a329b4346b4a3f2801ba973650d7235a5c4bd00bd0ded96f1a5768b9b", + "activationAvailable": true + } + }, + "dataPath": { + "httpStatus": 200, + "result": { + "nodes": [ + { + "id": "sensor-piezo", + "status": "ok" + }, + { + "id": "sensor-cap", + "status": "ok" + }, + { + "id": "sensor-temp", + "status": "ok" + }, + { + "id": "frames", + "status": "ok" + }, + { + "id": "dac", + "status": "ok" + }, + { + "id": "piezo-processor", + "status": "ok" + }, + { + "id": "sleep-detector", + "status": "ok" + }, + { + "id": "environment-monitor", + "status": "ok" + }, + { + "id": "dac-monitor", + "status": "ok" + }, + { + "id": "database", + "status": "ok" + }, + { + "id": "scheduler", + "status": "ok" + }, + { + "id": "out-vitals", + "status": "ok" + }, + { + "id": "out-sleep", + "status": "ok" + }, + { + "id": "out-temp", + "status": "ok" + }, + { + "id": "out-live", + "status": "idle" + } + ], + "verdict": "ok" + } + }, + "services": { + "sleepypod.service": "active", + "sleepypod-piezo-processor.service": "active", + "sleepypod-sleep-detector.service": "active", + "sleepypod-environment-monitor.service": "active" + } +} diff --git a/docs/performance/biometrics-replay-2026-10-13/fullnight-db52f1da.json b/docs/performance/biometrics-replay-2026-10-13/fullnight-db52f1da.json new file mode 100644 index 000000000..13851cfe2 --- /dev/null +++ b/docs/performance/biometrics-replay-2026-10-13/fullnight-db52f1da.json @@ -0,0 +1,93 @@ +{ + "scope": "Eight-hour synthetic replay resource measurement; not shadow, clinical or activation qualification", + "sourceCommit": "db52f1da", + "architecture": "aarch64", + "nodeVersion": "24.16.0", + "pythonVersion": "3.10.4", + "completed": true, + "fixtureSeconds": 28800, + "compressedBytes": 95621299, + "canonicalObservationsUnchanged": true, + "readFailures": 0, + "replayBudget": { + "jobWallMs": 7200000, + "waveformWallMs": 1800000, + "waveformCpuSeconds": 1200 + }, + "operations": [ + { + "phase": "snapshot", + "succeeded": true, + "elapsedMs": 1512.0559429999994, + "eventLoopP95Ms": 23.134207, + "eventLoopMaxMs": 39.550975, + "parentPeakRssKiB": 130780 + }, + { + "phase": "retain", + "succeeded": true, + "elapsedMs": 21739.24188, + "eventLoopP95Ms": 22.036479, + "eventLoopMaxMs": 51.937279, + "parentPeakRssKiB": 130780 + }, + { + "phase": "waveform-replay", + "succeeded": true, + "elapsedMs": 2756681.940805, + "eventLoopP95Ms": 27.279359, + "eventLoopMaxMs": 154.009599, + "parentPeakRssKiB": 153608 + }, + { + "phase": "summary", + "succeeded": true, + "elapsedMs": 2994.1249830001034, + "eventLoopP95Ms": 22.020095, + "eventLoopMaxMs": 34.865151, + "parentPeakRssKiB": 153752 + }, + { + "summaryBytes": 59919, + "resultHash": "bda8d03d158b04bb3669ff613e0936ed46822c0083c4d37f9f30e5338bc95f97", + "compoundBuildHash": "2b6514629ca87fc82753d6b0940b1c94d754f834585ef115387eaf536ef58df1", + "pythonBuildHash": "212c5643dd24cae1cbf9c346b0729d7ea144c453cd56370ec810223c508a5bc3" + } + ], + "concurrentReadMs": { + "count": 363, + "p95": 1943.6675560000003, + "max": 2183.3170790000004 + }, + "internalApiMs": { + "count": 7390, + "p95": 57.09343400015496, + "max": 300.81892600000015 + }, + "internalApiFailures": 0, + "externalApiMs": { + "count": 17568, + "p95": 39.62092199799372, + "max": 500.0740819959901 + }, + "externalApiFailures": 0, + "minimumAvailableMemoryBytes": 946417664, + "minimumFreeDiskBytes": 3243700224, + "maximumTemperatureC": 49.965, + "maximumIngestionAgeMs": 1560, + "negativeSampleAgeCount": 71, + "peakObservedBenchmarkTreeRssBytes": 464650240, + "maximumBenchmarkProcessCount": 4, + "artifactSha256": { + "benchmark.cjs": "bc1dc66c597b020480830548c2de7368a7bdbe2e68a440779c7de79b8254145f", + "payload.tgz": "9cf50a0a8488a1ef8f79ccc41bdd7d086c9dd68bcbba61156d11d8f768813a36", + "measurements.json": "696bdd6462504844c90936afcfe1c8cac7429d141f16893270292cdcc59c29bc", + "probe-28800.json": "e9865a8019e8e3360b022daab587ff5397d317bb9e17c362231ebc3288747af3", + "fixture-28800.json": "165071ec9b67e9dd2143575e8c2758b99090d47c1fb6412291b778d4be59dce0" + }, + "workerSha256": "c635a135ad0cfc2a327292968d08ab043ca8b523dd4c0149be6b793c8a192a5e", + "coordinatorSha256": "c31f3e6dc614cc688d06d25158318cea9c48f3f9ca3d157b576f1269e7672c05", + "maximumLoadPerCpu": 1.0575, + "meanAggregateCpuBusyPercent": 68.06810450124667, + "maximumAggregateCpuBusyPercent": 99.00497512437812 +} diff --git a/docs/performance/biometrics-replay-2026-10-13/live-shadow-operational.json b/docs/performance/biometrics-replay-2026-10-13/live-shadow-operational.json new file mode 100644 index 000000000..91e9c6862 --- /dev/null +++ b/docs/performance/biometrics-replay-2026-10-13/live-shadow-operational.json @@ -0,0 +1,81 @@ +{ + "version": "live-shadow-operational-measurement-v1", + "date": "2026-10-13", + "runtimeFix": "shadow RSS disappearance during clean exit", + "scope": "loopback live input and read-only active observations with isolated sink; not production registry admission or accuracy qualification", + "windowSeconds": 180, + "state": "completed", + "reason": "observation_window_complete", + "isolatedSink": { + "pointCount": 6, + "byteCount": 5396, + "canonicalRows": 0 + }, + "build": { + "runnerHash": "6a2966046edffea389604b28b89a7d9b4417dd3dcda41e64f7adab2103b11d03", + "pythonBuildHash": "b16390a3796398903b80db34a1e758d6e8ee3fb30dcfc4034e4b30bbdb13d0e9", + "manifestHash": "96c2f01a329b4346b4a3f2801ba973650d7235a5c4bd00bd0ded96f1a5768b9b" + }, + "worker": { + "inputBytes": 1013420, + "outputBytes": 118657, + "peakObservedRssBytes": 113881088, + "rssSamples": 1192, + "maximumObservationGapMs": 497.2393699999993, + "budgets": { + "cpuSeconds": 120, + "addressSpaceBytes": 805306368, + "maximumRssBytes": 402653184, + "wallMs": 270000, + "acknowledgmentMs": 30000, + "idleMs": 10000, + "rssIntervalMs": 100, + "rssTimeoutMs": 1000 + }, + "memoryScope": "sum_rss_of_shadow_process_group", + "exactKernelPeak": false, + "cpuMs": 4953.220225 + }, + "limitations": [ + "cold_start_without_serialized_state", + "less_than_five_minutes_preroll", + "missing_occupancy_context", + "missing_pump_context" + ], + "probe": { + "samples": 529, + "httpRequests": 1587, + "httpFailures": 0, + "httpP95Ms": 40.213834996393416, + "httpMaxMs": 123.67384999379283, + "peakProcessTreeRssBytes": 295608320, + "minimumAvailableMemoryBytes": 1135259648, + "maximumTemperatureC": 44.828, + "meanHostCpuPercent": 56.69213951033886, + "maximumHostCpuPercent": 96.59574468085107 + }, + "cleanup": { + "launchExitCode": 0, + "ownedProcessesAbsentAfterExit": true, + "remoteStageRemoved": true + }, + "qualificationInstalled": false, + "profileBindingsChanged": false, + "priorAttempts": [ + { + "state": "failed", + "reason": "Measurement health admission refused", + "cleanupConfirmed": true + }, + { + "state": "failed", + "reason": "Shadow RSS observation unavailable", + "cleanupConfirmed": true + } + ], + "runtimeFixCommit": "4c3a6a6e", + "rawArtifactSha256": { + "biometrics-live-shadow-third-summary.json": "7e5dca3d1c4a6b3e0360d256a203054ac694b42481eee3ea3b7a333dedc858a9", + "biometrics-live-shadow-third-probe.json": "1b86d9a5c1f2fa1f045b19cd310f7dafb09673f1017c7b43e376884888f01d10" + } +} diff --git a/docs/performance/biometrics-replay-2026-10-13/shadow-4ee74c25.json b/docs/performance/biometrics-replay-2026-10-13/shadow-4ee74c25.json new file mode 100644 index 000000000..f1c02fe11 --- /dev/null +++ b/docs/performance/biometrics-replay-2026-10-13/shadow-4ee74c25.json @@ -0,0 +1,117 @@ +{ + "version": "synthetic-shadow-supervisor-measurement-v1", + "scope": "accelerated synthetic frames through shipped supervisor and kernel-limited child; no live capture, database sink, held-out accuracy or installation qualification", + "runnerHash": "aaf032040970edf5758d0dbb358afe7044ce0274ee82492b5fd6a62925ef930d", + "pythonBuildHash": "b16390a3796398903b80db34a1e758d6e8ee3fb30dcfc4034e4b30bbdb13d0e9", + "node": { + "version": "v24.16.0", + "platform": "linux", + "arch": "arm64" + }, + "manifestHash": "96c2f01a329b4346b4a3f2801ba973650d7235a5c4bd00bd0ded96f1a5768b9b", + "configHash": "933fb4329b8c24091565b43388576d641f2806e0fda52bad44f2a2968b997aba", + "seconds": 1800, + "sampleRateHz": 500, + "channels": 2, + "wallMs": 61340.47504500001, + "outputCount": 30, + "outputHash": "e75ead5601fc3c5da660c9778dea0f6a2c999900a4d345754a0ed07f0b4970cf", + "receipt": { + "health": { + "state": "completed", + "reason": "window_complete", + "records": 1800, + "outputs": 30, + "cpu_ms": 29170.628042, + "input_hash": "a523ceeec2382b220c9f95874c22d74922c21db93dcb313d10824b60e8c27ad2", + "counts": { + "records": 1800, + "accepted_frames": 1800, + "empty_channel_frames": 0, + "pump_gated_frames": 0, + "preroll_outputs": 0, + "scored_outputs": 30 + }, + "limitations": [ + "cold_start_without_serialized_state", + "less_than_five_minutes_preroll" + ] + }, + "inputBytes": 9997366, + "outputBytes": 998314, + "peakObservedRssBytes": 118648832, + "rssSamples": 347, + "maximumObservationGapMs": 333.41997099999935, + "budgets": { + "cpuSeconds": 1200, + "addressSpaceBytes": 805306368, + "maximumRssBytes": 402653184, + "wallMs": 1800000, + "acknowledgmentMs": 30000, + "idleMs": 30000, + "rssIntervalMs": 100, + "rssTimeoutMs": 1000 + }, + "memoryScope": "sum_rss_of_shadow_process_group", + "exactKernelPeak": false + }, + "partial": { + "kind": "partial_supervisor_observation", + "health": { + "state": "completed", + "reason": "window_complete", + "records": 1800, + "outputs": 30, + "cpu_ms": 29170.628042, + "input_hash": "a523ceeec2382b220c9f95874c22d74922c21db93dcb313d10824b60e8c27ad2", + "counts": { + "records": 1800, + "accepted_frames": 1800, + "empty_channel_frames": 0, + "pump_gated_frames": 0, + "preroll_outputs": 0, + "scored_outputs": 30 + }, + "limitations": [ + "cold_start_without_serialized_state", + "less_than_five_minutes_preroll" + ] + }, + "inputBytes": 9997366, + "outputBytes": 998314, + "diagnosticBytes": 0, + "peakObservedRssBytes": 118648832, + "rssSamples": 347, + "maximumObservationGapMs": 333.41997099999935, + "budgets": { + "cpuSeconds": 1200, + "addressSpaceBytes": 805306368, + "maximumRssBytes": 402653184, + "wallMs": 1800000, + "acknowledgmentMs": 30000, + "idleMs": 30000, + "rssIntervalMs": 100, + "rssTimeoutMs": 1000 + }, + "cleanupConfirmed": true, + "memoryScope": "sum_rss_of_shadow_process_group", + "exactKernelPeak": false + }, + "failure": null, + "parentEventLoopP95Ms": 26.099711, + "parentEventLoopMaxMs": 68.616191, + "parentPeakRssKiB": 150836, + "probe": { + "requests": 456, + "failures": 0, + "p95Ms": 45.41382499883184, + "maximumMs": 130.38527100434294, + "maximumTemperatureC": 47.513, + "minimumAvailableMemoryBytes": 1069072384, + "peakBenchmarkTreeRssBytes": 268959744 + }, + "artifactSha256": { + "shadow-measurements.json": "e57bf3bd10374b31059f60e39f8645d2be158ab88d2eb9a1abf1baa9b9a786bd", + "shadow-probe.json": "13fd0f3723b6ebc5650cc52868d8c02f5f4bee6cb0629303b53998f09ee883cd" + } +} diff --git a/docs/performance/biometrics-replay-2026-10-13/smoke-4453c0a5.json b/docs/performance/biometrics-replay-2026-10-13/smoke-4453c0a5.json new file mode 100644 index 000000000..c5c3d9078 --- /dev/null +++ b/docs/performance/biometrics-replay-2026-10-13/smoke-4453c0a5.json @@ -0,0 +1,88 @@ +{ + "scope": "30-minute synthetic worker resource smoke; not full-night, shadow, clinical or activation qualification", + "sourceCommit": "4453c0a5", + "architecture": "aarch64", + "nodeVersion": "24.16.0", + "pythonVersion": "3.10.4", + "completed": true, + "fixtureSeconds": 1800, + "compressedBytes": 5976451, + "canonicalObservationsUnchanged": true, + "readFailures": 0, + "replayBudget": { + "jobWallMs": 900000, + "waveformWallMs": 240000, + "waveformCpuSeconds": 180 + }, + "operations": [ + { + "phase": "snapshot", + "succeeded": true, + "elapsedMs": 1560.2145219999984, + "eventLoopP95Ms": 21.987327, + "eventLoopMaxMs": 29.081599, + "parentPeakRssKiB": 129868 + }, + { + "phase": "retain", + "succeeded": true, + "elapsedMs": 16172.227184, + "eventLoopP95Ms": 22.233087, + "eventLoopMaxMs": 42.532863, + "parentPeakRssKiB": 129868 + }, + { + "phase": "waveform-replay", + "succeeded": true, + "elapsedMs": 202759.751582, + "eventLoopP95Ms": 28.049407, + "eventLoopMaxMs": 186.384383, + "parentPeakRssKiB": 129868 + }, + { + "phase": "summary", + "succeeded": true, + "elapsedMs": 1541.8592840000056, + "eventLoopP95Ms": 21.790719, + "eventLoopMaxMs": 24.936447, + "parentPeakRssKiB": 129868 + }, + { + "summaryBytes": 58915, + "resultHash": "ed2505dfb5cc91be94ebeea9f26bddc5ee214c95e9ca730eebb459dbff9c989d", + "compoundBuildHash": "3f9ee531c01c501f543278dfb5641eb57eea9cf2906d7ce2f94f01d6ab1a2e34", + "pythonBuildHash": "0eb411804b1b4ba608a1a8db11406dcf983f0ff9bea0f2d8c5eb027b6a187254" + } + ], + "concurrentReadMs": { + "count": 27, + "p95": 1859.888511000012, + "max": 1860.4550230000168 + }, + "internalApiMs": { + "count": 638, + "p95": 65.67037400000845, + "max": 305.6198179999999 + }, + "internalApiFailures": 0, + "externalApiMs": { + "count": 1545, + "p95": 41.55511000135448, + "max": 129.56501200096682 + }, + "externalApiFailures": 0, + "minimumAvailableMemoryBytes": 992690176, + "minimumFreeDiskBytes": 3450945536, + "maximumTemperatureC": 46.696, + "maximumIngestionAgeMs": 992, + "negativeSampleAgeCount": 9, + "peakObservedBenchmarkTreeRssBytes": 433369088, + "maximumBenchmarkProcessCount": 4, + "artifactSha256": { + "benchmark.cjs": "bcfd8e5024a606aa1ede8f33481a884d553cc4344eeeb4fcd7c610a943e69984", + "payload.tgz": "82ace2b631e042d8c0993c21d5923580df614357bd00fc598692588b6e485543", + "measurements.json": "d0eb400d98dd78c12921b2bc2f42c1471c3d6a4e529c4b01be863a3f64c95f7e", + "probe.json": "134ded3108c2f678cfa8aafec1bee782ab502820b632214c2ae280c9e581f330", + "fixture-1800.json": "861e719fb0c8bad1ddf89c625e7a19388e32dc35984ff90504c00b3c3e06e4a6" + } +} diff --git a/docs/research/sleep-stage-calibration/PLAN.md b/docs/research/sleep-stage-calibration/PLAN.md new file mode 100644 index 000000000..ccc9013eb --- /dev/null +++ b/docs/research/sleep-stage-calibration/PLAN.md @@ -0,0 +1,100 @@ +# Sleep-stage calibration against Apple Watch — plan + +Status: draft 2026-10-10. Owner: Jonathan. Each workstream below is scoped so one +agent can own it end to end. Workstreams A–D are independent; E depends on C. + +## 1. Recorded findings + +Source: Health export `/Users/ng/Desktop/apple_health_export/export.xml` joined per +minute with `vitals` + `movement` from the Pod 5 `biometrics.db` +(192.168.1.88:8822, `/persistent/sleepypod-data/biometrics.db`, timestamps in +seconds). `scripts/sleep-stage-dataset.py` does the join and reproduces the +first three table rows within 1 point with `--summary` (it labels by exact +Watch segment times, so 10-08 prints 52/32/68); `feat2.py` here runs the windowed +features and leave-one-night-out tree on its CSV. See `README.md`. The +720-combo grid search lived in the original `eval.py` (removed; in git history). + +Usable nights: 2026-10-08/09 and 2026-10-09/10, left side, 587 Watch-labeled vitals samples (about one per minute). +(2026-10-01/02 has Watch only against the right side, HR MAD 12 bpm — excluded.) + +| Variant | 10-08 | 10-09 | +|---|---|---| +| Server as deployed (movement-only, `calibrationQuality` defaults to 0) | 53% | 55% | +| iOS rule set (offline simulation, `calibrationQuality`=1) | 33% | 31% | +| Always "light" | 69% | 57% | +| Best of 720 threshold combos (pooled) | 55% | 57% | + +Why the rules fail: +- Per-minute pod HR ratio for Watch-deep is 0.95–1.04, never < 0.92. The "deep = low HR" rule fires on noise. +- Pod HRV relative to night median: Watch-deep 0.56–0.78×, Watch-REM 1.1–1.4×. The rule "REM = low HRV" is inverted; it labels deep as REM. +- `movement.total_movement` is exactly 0 for 97% of epochs on 10-09 (39% nonzero on 10-08). Wake rules cannot fire. +- Both callers omit `calibrationQuality`, so the server never emits deep/REM: `src/server/routers/biometrics.ts:1217`, `src/components/Sleep/sleepData.ts:255`. +- 126 of 782 `sleep_records` have `sleep_duration_seconds` = 57600 (the `MAX_SESSION_S` 16 h cap) with 0 exits. Vitals stop when the bed empties (0 rows 11:05→18:04 on 10-10), so the cap pads the stage timeline, not the vitals averages. +- Pod HRV is an index of successive differences of 10 s-window IBIs (docs/wiki/topics/piezo-processing.md), not SDNN. Watch SDNN is 3 spot readings/night. No scalar correction is valid. + +Stages are **not stored**. `classifySleepStages` runs at query time over `vitals` + `movement`. A classifier change therefore applies to every past night automatically. + +## 2. Backfill policy (what can and cannot be repaired) + +| Layer | Stored? | Backfill | +|---|---|---| +| Sleep stages | No (derived at read) | Automatic once classifier changes. Nothing to migrate. | +| `sleep_records` 16 h cap | Yes | One-off repair, `scripts/repair-capped-sessions.py` (PR #804), for records with `sleep_duration_seconds`=57600. Occupancy evidence is the side's `vitals` only; `movement` is written every minute while a session is open, so it runs to the cap. Vitals are split into runs at gaps > 5 min; occupancy ends at the end of the last run lasting ≥ 30 min. `left_bed_at` = that end + `ABSENCE_TIMEOUT_S` (120 s), never later than the original; duration recomputed, intervals clipped, exits recounted. Records with no qualifying run are left unchanged and listed. Dry run by default; `--apply` first writes `.bak.` with the SQLite backup API. | +| `movement` zeros | Yes | Not recoverable: raw `.RAW` is a rolling single file on tmpfs (Pod 5), nothing archived (`archive-push-staging` empty). `cap_sense_frames` holds ~2 days of per-window zone sums (69k rows, 10-08→10-10) — only those days could be re-derived, not worth it. Forward-only fix. | +| `vitals` HR/HRV/BR | Yes | Not recomputable (raw gone). Keep as is; treat HRV as an index. | +| HealthKit samples written by the iOS app ("sleepypod" source) | In HealthKit | iOS app must delete its own `HKCategoryTypeIdentifierSleepAnalysis` samples for affected nights and rewrite after the server returns new stages. HealthKit permits deleting samples your app wrote. Out of this repo; track in sleepypod-ios. | +| Watch labels | In HealthKit | iOS app can read Watch `SleepAnalysis` and POST to the pod (see D) so the dataset accrues without manual exports. | + +## 3. Workstreams + +### A. Sleep detector: stuck presence / 16 h cap (modules/sleep-detector, python) +Why: 126/782 sessions hit `MAX_SESSION_S`; the stage timeline runs to 17:30 on an empty bed. +What: find why presence stays true after the bed empties on Pod 5 (adaptive baseline, exit fraction, or capSense2 sentinel handling); fix; add the repair script from §2 as `scripts/repair-capped-sessions.py` (dry-run default). +Acceptance: +- Replay test with recorded capSense2 frames ending in an empty bed closes the session within `ABSENCE_TIMEOUT_S`. +- Repair script on a copy of the pod DB rewrites every capped record that has a qualifying vitals run (52 of 126 on the 2026-10-10 copy) and lists the rest unchanged; new `left_bed_at` ≤ end of the last qualifying run + 120 s and ≤ the original; writes `.bak` first; a re-run finds nothing. +- `docs/sleep-detector.md` updated. + +### B. Sleep detector: movement all-zero (modules/sleep-detector, python) +Why: 97% zero epochs on 10-09 makes wake undetectable. +What: determine which stage of the PIM pipeline (pump gate, P5 baseline subtraction, median filter, scale factor) collapses the signal on Pod 5; fix so a quiet night yields the documented distribution (70–80% epochs 0–50, not 97% exactly 0). +Acceptance: +- Unit test with a synthetic still-then-roll sequence produces nonzero scores for the roll and a nonzero floor during stillness. +- One live night on Pod 5 shows ≥ 30% nonzero epochs and ≥ 1 epoch > 200 per bed exit. + +### C. Dataset builder (scripts/, python) — shipped in PR #802 +Why: calibration needs ≥ 20 labeled nights; the join originally lived in three scratch scripts (now removed). +What (done): `scripts/sleep-stage-dataset.py --side left --out dataset.csv` emitting one row per pod vitals sample (about one per minute): `night, side, ts, watch_stage, hr, hrv, br, movement, hr_quality` plus `watch_hr` when within 60 s. Also `--summary` printing per-night agreement for the current classifier and the always-light baseline. +Acceptance: +- Reproduces the first three rows of the table in §1 for the two nights (±1 point). +- Rejects nights where pod-vs-Watch HR MAD > 8 bpm with a warning (side mismatch guard). +- README in this folder documents how to pull a consistent copy of the DB (SQLite backup API on the pod, or `scp -P 8822 root@:/persistent/sleepypod-data/biometrics.db{,-wal} .` while the writers are quiet). +- Python tests run in CI (Python Modules workflow). +Remaining: accrue ≥ 20 Watch-labeled nights; confirm `--from-db` parity against the export for 10-09 once the app has posted that night (D is merged). + +### D. Comparison-data ingestion from the app (src/server, tRPC + iOS stub) +Why: manual Health exports do not scale to 20 nights. +What: tRPC mutation `biometrics.importReferenceStages({side, source:'apple_watch', segments:[{start,end,stage}]})` writing a new `reference_stages` table (migration in `src/db/biometrics-migrations`), idempotent on (side, source, start). The iOS app posts the Watch's `SleepAnalysis` samples each morning. Dataset builder (C) reads this table when no export is given. +Acceptance: +- Migration + Drizzle schema; mutation validates input at the boundary; duplicate posts are no-ops. +- `sleep-stage-dataset.py --from-db` yields the same rows as the export input (the default; there is no `--from-export` flag) for 10-09, once the iOS app has posted that night. + +### E. Classifier v2 (src/lib/sleep-stages.ts) — blocked on C, needs ≥ 20 nights +Why: the current rule family cannot beat always-light. +What: replace `classifyEpoch` with windowed features: 15-min HR std, 15-min mean HRV / night median HRV, breathing-rate std, fraction of night elapsed, movement (once B lands). Start with a small, inspectable model (depth-3 tree or hand thresholds derived from it), same post-processing (smoothing, transition constraints). Pass real `calibrationQuality` from `calibration_profiles` at both call sites; fall back to light/wake-only when below 0.3. +Acceptance: +- Leave-one-night-out agreement ≥ always-light + 10 points on every held-out night, reported in `docs/research/sleep-stage-calibration/RESULTS.md`. +- Cohen's κ reported alongside accuracy. +- Existing `sleep-stages` tests updated; quality score unchanged in shape. +- UI copy: HRV card labels the pod value "HRV index", not ms-SDNN. + +### F. iOS app follow-ups (sleepypod-ios, not this repo) +- Align the comparison window to the intersection of Watch coverage and pod session; show "compared" over that intersection only. +- Delete and rewrite the app's own HealthKit stage samples after E ships. +- Post Watch stages to D each morning. + +## 4. Delegation notes +- A and B share `modules/sleep-detector`; acquire `ygg lock acquire modules/sleep-detector` and sequence B after A, or give both to one agent. +- C and D can run in parallel; D must not change C's CSV schema. +- Do not touch E until C reports ≥ 20 nights with HR MAD ≤ 8 bpm. +- Pod access: ssh port 8822, no `sqlite3` binary on the pod — copy the DB and query locally. diff --git a/docs/research/sleep-stage-calibration/README.md b/docs/research/sleep-stage-calibration/README.md new file mode 100644 index 000000000..7e199d92e --- /dev/null +++ b/docs/research/sleep-stage-calibration/README.md @@ -0,0 +1,154 @@ +# Sleep-stage calibration dataset + +Builds a per-minute dataset that pairs Apple Watch sleep stages with the pod's +vitals and movement, so the stage classifier in `src/lib/sleep-stages.ts` can be +measured and recalibrated. Background and findings: [PLAN.md](PLAN.md). + +## 1. Pull the pod database + +The pod has no `sqlite3` binary, so copy the database off the pod and query +locally. sshd listens on port 8822. + +The pod's services write to the database all the time, so copying +`biometrics.db` and its `-wal` file one after the other is not an atomic +snapshot. A checkpoint between the two reads can leave the copy inconsistent. +Take the snapshot on the pod with SQLite's backup API through the pod's +`python3`, then copy that one file: + +```sh +ssh -p 8822 root@ python3 - <<'EOF' +import sqlite3 +src = sqlite3.connect('file:/persistent/sleepypod-data/biometrics.db?mode=ro', uri=True) +dst = sqlite3.connect('/persistent/sleepypod-data/biometrics.snapshot.db') +src.backup(dst) +dst.close() +EOF +mkdir -p poddb +scp -P 8822 root@:/persistent/sleepypod-data/biometrics.snapshot.db poddb/biometrics.db +ssh -p 8822 root@ rm -f /persistent/sleepypod-data/biometrics.snapshot.db +``` + +The snapshot includes the rows still in the WAL. If you copy the live files +instead (`scp -P 8822 root@:/persistent/sleepypod-data/biometrics.db{,-wal} .`), +do it while the writers are quiet, such as when no one is in bed, and copy +both files in one command. Without the `-wal` file the last few hours are +missing. The script opens the DB read-only. SQLite still reads a WAL if one is +present and may create a `-shm` file beside it. + +## 2. Export Apple Health + +On the iPhone: Health → profile picture → Export All Health Data. AirDrop the +zip to the Mac and unzip it. The script needs `apple_health_export/export.xml` +(or the directory). The file is parsed incrementally with constant memory, so +a 100+ MB export takes a few seconds. + +Only samples whose source name matches `--source` (default `Watch`) are used: +`SleepAnalysis` stages (Core→light, Deep, REM, Awake; InBed and Unspecified are +ignored) and `HeartRate`. + +## 2b. Import the Watch data into the pod + +The export can also be pushed into the pod itself, so the Biometrics page +draws Watch heart rate as a dashed line over the pod's, and later dataset +builds can use `--from-db` without an export: + +```sh +scripts/import-watch-health.py ~/Desktop/apple_health_export --pod --side left +``` + +It posts `SleepAnalysis` segments to `POST /api/biometrics/reference-stages` +and `HeartRate` samples to `POST /api/biometrics/reference-vitals`, in batches +of 2000, for the last 30 days (`--days 0` for everything; `--dry-run` to count +only). Heart rate is first-write-wins on side + time + source, so re-running +on a newer export only adds what is new. Stages are stricter: the pod rejects +a batch whose segment overlaps a stored one with different boundaries (say a +night the iOS app already posted); the script skips that batch with a warning, +still imports heart rate, and exits 1. Watch heart rate is kept +in its own `reference_vitals` table and never mixed into the pod's `vitals`, +which feed occupancy, sleep staging and the chart baselines. + +## 3. Build the dataset + +```sh +scripts/sleep-stage-dataset.py ~/Desktop/apple_health_export poddb/biometrics.db \ + --side left --out dataset.csv --summary +``` + +The shebang runs under `uv`; plain `python3 scripts/sleep-stage-dataset.py …` +works too (stdlib only, Python ≥ 3.9). + +One row per pod vitals sample (about one per minute) inside each night's +window. A window is the Watch's stage coverage padded by `--pad-min` (default +10) on each side. Windows never overlap: sleep that runs past the noon +rollover stays with the night it started in. + +| column | meaning | +|---|---| +| `night` | local date the night started (sample time minus 12 h) | +| `side` | `left` or `right` | +| `ts` | pod vitals timestamp, unix seconds | +| `watch_stage` | `wake`/`light`/`deep`/`rem` of the Watch segment containing `ts`, empty in the padding | +| `hr`, `hrv`, `br` | raw pod vitals, not outlier-filtered. HRV is a pod index, not SDNN | +| `movement` | nearest `movement.total_movement` within 60 s | +| `hr_quality` | `vitals_quality.quality_score` for that vitals row | +| `watch_hr` | nearest Watch heart-rate sample within 60 s | + +Options: + +- `--nights 2026-10-08,2026-10-09` keeps only those nights. +- `--max-hr-mad 8` is the side-mismatch guard. A night where the mean absolute + difference between pod HR and Watch HR exceeds it is skipped with a warning, + because it means the Watch wearer slept on the other side. Nights with fewer + than 10 HR matches are kept with a warning that the guard could not run. +- `--from-db` reads Watch stages from the `reference_stages` table and Watch + heart rate from `reference_vitals` (both filled by the import script in + section 2b, or by the iOS app), instead of an export: + `scripts/sleep-stage-dataset.py --from-db poddb/biometrics.db --side left --out dataset.csv`. + On a pod that has never had Watch heart rate imported, `watch_hr` is empty + and the guard cannot run. Night labels use the machine's local time zone; + set `TZ` if it differs from the pod's. + +## 4. Summary + +`--summary` prints, per night, the agreement with the Watch over Watch-labeled +vitals samples for the ported classifier as deployed (`calibrationQuality` 0, +movement-only), with the iOS rule set (`calibrationQuality` 1), and for an +always-light baseline. Samples arrive about once a minute, so agreement is +weighted by sample, and the per-stage distribution below the table counts +samples, not minutes. The classifier runs over the night window, so its +average HR differs slightly from the server's, which uses the whole sleep +record. + +Output for the two usable nights on 2026-10-10. It is within 1 point of +PLAN.md section 1, which labeled whole minutes instead of exact segment times: + +``` +night side rows labeled HR-MAD deployed(cq=0) iOS(cq=1) always-light +2026-10-08 left 212 202 2.3 52% 32% 68% +2026-10-09 left 405 385 5.4 55% 31% 57% +``` + +The 2026-10-01 night is rejected by the guard on the right side +(HR MAD 11.8 bpm) and has no left-side vitals. + +## 5. Exploration + +`feat2.py` reads the CSV and prints windowed features per Watch stage and a +leave-one-night-out tree and forest: + +```sh +docs/research/sleep-stage-calibration/feat2.py dataset.csv +``` + +It needs numpy and scikit-learn; its shebang pulls them through `uv`. + +## Tests + +```sh +python3 -m unittest scripts/tests/test_sleep_stage_dataset.py +``` + +The Python Modules CI workflow runs them on every PR that touches +`scripts/**/*.py`. The tests use small synthetic fixtures, not a real export. When +`src/lib/sleep-stages.ts` changes, update the port in the script and the +classifier cases in the test, which mirror `src/lib/tests/sleep-stages.test.ts`. diff --git a/docs/research/sleep-stage-calibration/feat2.py b/docs/research/sleep-stage-calibration/feat2.py new file mode 100755 index 000000000..96a8fb1c5 --- /dev/null +++ b/docs/research/sleep-stage-calibration/feat2.py @@ -0,0 +1,109 @@ +#!/usr/bin/env -S uv run --with numpy --with scikit-learn --quiet --no-project +"""Windowed features per Watch stage + leave-one-night-out tree, from dataset.csv. + +Input is the CSV written by scripts/sleep-stage-dataset.py. Exploratory only; +classifier v2 (PLAN.md workstream E) builds on this once >= 20 nights exist. + +Usage: docs/research/sleep-stage-calibration/feat2.py dataset.csv +""" +import csv +import sys +from collections import Counter, defaultdict + +import numpy as np + +STAGES = ('wake', 'light', 'deep', 'rem') +KEYS = ['hr_std', 'hr_rel', 'hrv_rel', 'br_std', 'tso'] + + +def num(v): + return float(v) if v != '' else np.nan + + +def load(path): + nights = defaultdict(list) + with open(path, newline='') as f: + for r in csv.DictReader(f): + nights[(r['night'], r['side'])].append(r) + for rows in nights.values(): + rows.sort(key=lambda r: int(r['ts'])) + return dict(sorted(nights.items())) + + +def windowed(rows): + """15-min centered window features for every row of one night. + + Windows and elapsed fraction use timestamps, not row indices, so gaps in + the vitals do not stretch a window across hours.""" + ts = np.array([int(r['ts']) for r in rows]) + hr = np.array([num(r['hr']) for r in rows]) + hrv = np.array([num(r['hrv']) for r in rows]) + br = np.array([num(r['br']) for r in rows]) + # A nonpositive median would make the ratios infinite; treat it as missing. + medhr, medhrv = (m if m > 0 else np.nan for m in (np.nanmedian(hr), np.nanmedian(hrv))) + lo = np.searchsorted(ts, ts - 450, side='left') + hi = np.searchsorted(ts, ts + 450, side='right') + span = max(ts[-1] - ts[0], 1) + out = [] + for i, r in enumerate(rows): + s = slice(lo[i], hi[i]) + out.append({'w': r['watch_stage'], 'hr_std': np.nanstd(hr[s]), 'hr_rel': np.nanmean(hr[s]) / medhr, + 'hrv_rel': np.nanmean(hrv[s]) / medhrv, 'br_std': np.nanstd(br[s]), + 'tso': (ts[i] - ts[0]) / span}) + return out + + +def main(path): + nights = load(path) + print('=== movement health per night (nonzero fraction, p50/p90/p99)') + for (n, side), rows in nights.items(): + mv = [float(r['movement']) for r in rows if r['movement'] != ''] + if mv: + print(f' {n} {side}: n={len(mv)} nonzero={sum(1 for x in mv if x > 0) / len(mv):.0%} ' + f'p50={np.percentile(mv, 50):.0f} p90={np.percentile(mv, 90):.0f} p99={np.percentile(mv, 99):.0f}') + + print('\n=== windowed features per Watch stage (15-min centered)') + groups = defaultdict(list) + for (n, side), rows in nights.items(): + f = [x for x in windowed(rows) if x['w']] + if f: + groups[n].extend(f) + print(f' night {n} {side}') + for s in STAGES: + g = [x for x in f if x['w'] == s] + if g: + m = {k: np.nanmedian([x[k] for x in g]) for k in KEYS} + print(f" {s:5} n={len(g):3} hr_std={m['hr_std']:.2f} hr_rel={m['hr_rel']:.3f} " + f"hrv_rel={m['hrv_rel']:.2f} br_std={m['br_std']:.2f} tso={m['tso']:.2f}") + + if len(groups) < 2: + print('\nleave-one-night-out needs >= 2 Watch-labeled nights') + return + try: + from sklearn.ensemble import RandomForestClassifier + from sklearn.tree import DecisionTreeClassifier + except ImportError as e: + print('no sklearn', e) + return + + def X(g): + return np.nan_to_num(np.array([[x[k] for k in KEYS] for x in g]), nan=0.0, posinf=0.0, neginf=0.0) + + def y(g): + return [x['w'] for x in g] + + print('\n=== leave-one-night-out, windowed features, small tree / forest') + for night, te in groups.items(): + tr = [x for n, g in groups.items() if n != night for x in g] + for name, clf in (('tree d3', DecisionTreeClassifier(max_depth=3, class_weight='balanced', random_state=0)), + ('forest', RandomForestClassifier(200, max_depth=4, class_weight='balanced', random_state=0))): + clf.fit(X(tr), y(tr)) + pr = clf.predict(X(te)) + acc = np.mean(pr == np.array(y(te))) + print(f' test night {night} {name:8} acc={acc:.0%} pred={dict(Counter(str(p) for p in pr))} truth={dict(Counter(y(te)))}') + + +if __name__ == '__main__': + if len(sys.argv) != 2: + sys.exit(__doc__) + main(sys.argv[1]) diff --git a/docs/sleep-detector.md b/docs/sleep-detector.md index 8cd9db0b0..e0628dc61 100644 --- a/docs/sleep-detector.md +++ b/docs/sleep-detector.md @@ -14,7 +14,7 @@ flowchart TD TypeFilter -->|"capSense/capSense2"| Extract["Extract channel values\n(sentinel filter, ref compensation)"] TypeFilter -->|Other| Skip[Skip] Extract --> Presence["Presence detection\n(raw-unit rise over baseline or fallback)"] - Extract --> PumpGate{"Pump gate active?\n(RPM > 0 OR guard OR ref anomaly)"} + Extract --> PumpGate{"Pump gate active?\n(pump changed state < 3s ago OR ref anomaly)"} PumpGate -->|Yes| ZeroDelta["delta = 0\n(suppress artifact)"] PumpGate -->|No| Delta["Movement delta\n(|current - previous| per channel)"] ZeroDelta --> Session["SessionTracker\n(per side)"] @@ -22,8 +22,7 @@ flowchart TD Presence --> Session Session -->|"60s epoch"| RawScore["Raw score = min(1000, sum * scale)"] RawScore --> Baseline["Baseline subtraction\n(score - P5 of trailing 30 epochs)"] - Baseline --> MedianFilt["3-epoch median filter"] - MedianFilt --> Clamp["Clamp to 0-1000"] + Baseline --> Clamp["Clamp to 0-1000"] Clamp --> MovDB[("movement table\n(total_movement 0-1000)")] Session -->|"absence > 120s"| SleepDB[("sleep_records table")] ``` @@ -36,7 +35,7 @@ Movement is measured as the sum of absolute sample-to-sample deltas across the 3 ```mermaid flowchart LR - Raw["capSense2 record\n[A1,A2,B1,B2,C1,C2,ref1,ref2]"] --> Sentinel{"Any sensing value\n(indices 0-5) == -1.0?"} + Raw["capSense2 record\n[A1,A2,B1,B2,C1,C2,ref1,ref2]"] --> Sentinel{"Sentinel (-1.0) frame,\nor the frame after one?"} Sentinel -->|Yes| Hold["Zero-order hold\n(keep previous, delta=0)"] Sentinel -->|No| Avg["Average pairs\nA=(A1+A2)/2\nB=(B1+B2)/2\nC=(C1+C2)/2"] Avg --> Ref["Reference compensation\nsubtract ref drift from nominal 1.16"] @@ -47,8 +46,7 @@ flowchart LR Delta --> Buf Buf -->|"Every 60s"| Score["raw_score = min(1000, sum * 10)"] Score --> Baseline["score -= P5(trailing 30 epochs)\n(skip during 10-min cold start)"] - Baseline --> Median["3-epoch median filter"] - Median --> Clamp["clamp(0, 1000)"] + Baseline --> Clamp["clamp(0, 1000)"] Clamp --> DB[("movement table")] ``` @@ -76,13 +74,13 @@ Score is capped at 1000. capSense2 (Pod 5) sums its float channel deltas `×10`. capSense (Pod 3/4) integer channels change by several counts per sample with nobody moving, so a plain delta sum saturates every minute. Instead each channel's delta counts `×1`, only by its excess over `CAPSENSE_NOISE_K` (6) times that channel's typical delta (median over the last ~10 minutes, at least 1), and only when it lands on a level the channel hasn't held in the last `CAPSENSE_LEVEL_MEMORY` samples (~3 min). The level rule is for breathing flicker: in some lying positions one channel flips between two fixed levels (~200 counts apart) with each breath while the body is still, which otherwise scored 1000 for half an hour at a time. Turning over moves the channels to new levels and still counts; the return leg of an out-and-back motion (a tap, a twitch) doesn't. -After summing, each epoch has the 5th percentile of the trailing epochs subtracted. capSense2 epochs then pass through a 3-epoch median filter. capSense epochs don't: a turn-over is usually one busy minute between still ones, which the median zeroed, and the noise margin and level memory already reject that sensor's artifacts. +After summing, each epoch has the 5th percentile of the trailing epochs subtracted. There is no median filter: a turn-over is usually one busy minute between still ones, which a 3-epoch median zeroed. On capSense the noise margin and level memory reject that sensor's artifacts; on capSense2 the periodic glitch the median used to hide (the frame after a sentinel) is skipped at the source. Normal healthy sleep averages ~10 major position changes per night (De Koninck et al. 1992). ### Sentinel filtering -capSense2 firmware occasionally emits `-1.0` as a sentinel value on read errors in the sensing channels (indices 0-5). These are filtered via zero-order hold (carry forward the last valid reading, emit delta=0 for that sample). The next valid sample computes its delta against the last valid reading, which may span the sentinel gap — this produces a slightly larger delta but avoids the massive spike that a raw sentinel-to-valid transition would cause. Reference channel sentinels (indices 6-7) disable reference compensation for that sample but do not affect the sensing channels. +capSense2 firmware emits `-1.0` as a sentinel value on read errors in the sensing channels (indices 0-5); Pod 5 sends an all `-1.0` frame every 5 minutes. These are filtered via zero-order hold (carry forward the last valid reading, emit delta=0 for that sample). The frame right after a sentinel frame is still settling — every channel above 28.5 reads exactly 28.5 — so it is held too; its down-and-back-up delta added ~110 to one epoch in five on an occupied side. The next valid sample computes its delta against the last valid reading, which may span the gap. Reference channel sentinels (indices 6-7) disable reference compensation for that sample but do not affect the sensing channels. ### Reference channel compensation @@ -90,32 +88,30 @@ The capSense2 record includes a reference channel pair (indices 6,7) that reads ### Pump artifact gating (#230) -**Problem.** The pod's air pump runs periodically to maintain mattress pressure. Pump vibrations couple mechanically through the mattress into the capacitive sensor electrodes, producing small but consistent delta spikes (~0.05-0.2 per channel per sample). Over a 60-second epoch with ~120 samples, these accumulate to raw scores of 60-200 per pump-active epoch. Over a full night with frequent pump cycles, movement scores escalate from a true ~50 to 960-990 by the early morning hours. +**Problem.** Pump transients couple mechanically through the mattress into the capacitive electrodes. The gate was first built for a pump that cycles, and gated every sample while the pump ran. Pod 5 circulates continuously (~1940 RPM, boosting to ~3100) for as long as a side is on, so that gate zeroed movement for whole nights: 97% of epochs on pod 88's 2026-10-09 night were exactly 0. A steady pump does not raise the delta floor: on pod 88 an empty side scored ~25 per epoch with the pump off, at 1940 RPM and at 3100 RPM. -**Three-signal detection.** The `PumpGateCapSense` class uses three independent signals: +**Two-signal detection.** The `PumpGateCapSense` class gates a side's delta: -1. **Primary: frzHealth pump RPM.** The `frzHealth` record (Pod 5 only, ~0.06 Hz) reports pump RPM per side. Any RPM > 0 means the pump is running. This is the most reliable signal but has low temporal resolution (~16s between updates). +1. **Pump state change.** For `PUMP_GUARD_S` (3 s, ~6 samples at ~2 Hz) after that side's pump starts, stops, or steps its RPM by more than `PUMP_RPM_CHANGE_FRACTION` (10%). The RPM comes from `frzHealth` (`side.pump.rpm` on Pod 5, ~0.1 Hz). Once a side's `frzHealth` reports an RPM, its `frzTherm` `power` is ignored: that is the TEC drive, which drops to 0 while the pump keeps running. Pods without `frzHealth` RPM still take pump state from `frzTherm`. -2. **Secondary: Reference channel anomaly.** The capSense2 reference channel pair (indices 6,7) is mechanically coupled to the sensor PCB but does not respond to body presence. When `|ref_delta| > 0.02` AND at least 2 of 3 active channel deltas correlate (both spike together with magnitude > 0.5x the ref delta), the sample is flagged as mechanical coupling rather than body movement. +2. **Reference channel anomaly.** The capSense2 reference channel pair (indices 6,7) is mechanically coupled to the sensor PCB but does not respond to body presence. When `|ref_delta| > 0.02` AND at least 2 of 3 active channel deltas correlate (both spike together with magnitude > 0.5x the ref delta), the sample is flagged as mechanical coupling rather than body movement. -3. **Guard period: 3 seconds.** After pump-off is detected (RPM transitions from >0 to 0), a 3-second guard period (~6 samples at ~2 Hz) suppresses deltas while residual vibrations decay. This is shorter than the piezo processor's 5-second guard because capacitive sensors have lower sensitivity to mechanical vibration than piezoelectric sensors. - -When any signal is active, the movement delta for that sample is forced to 0. +When either signal is active, the movement delta for that sample is forced to 0. Any steady pump floor that remains is removed by the baseline subtraction below. **Pipeline position.** Pump gating is applied after reference compensation and before delta computation: ```text -1. Sentinel filter (-1.0 values) +1. Sentinel filter (-1.0 frames and the frame after each) 2. Pair averaging 3. Reference compensation -4. PUMP GATE — if pumpActive OR inGuardPeriod OR refAnomaly: delta = 0 +4. PUMP GATE — if pump changed state < PUMP_GUARD_S ago OR refAnomaly: delta = 0 5. Per-channel delta 6. 60s epoch accumulation ``` ### Baseline subtraction -After computing the raw epoch score (`min(1000, sum * scale)`), the 5th percentile of the trailing 30 epochs is subtracted. This removes slow-building noise floors (residual pump artifacts that leak through the gate, thermal drift in the capacitive sensor). +After computing the raw epoch score (`min(1000, sum * scale)`), the 5th percentile of the trailing 30 epochs is subtracted. This removes slow-building noise floors (steady pump vibration, thermal drift in the capacitive sensor). A still sleeper's floor varies by a few points from minute to minute, so still epochs land at small nonzero scores rather than exactly 0. - **Trailing window:** 30 epochs (30 minutes at 60s epochs) - **Percentile:** 5th (robust to outliers; represents the quietest ~1.5 epochs in the window) @@ -123,11 +119,9 @@ After computing the raw epoch score (`min(1000, sum * scale)`), the 5th percenti The subtracted score is clamped to a minimum of 0. -### 3-epoch median filter - -A 3-epoch running median is applied as the final smoothing step after baseline subtraction. This suppresses isolated spike artifacts (single-epoch transients from sensor glitches, brief vibration events) without attenuating sustained movement events. +### No median filter -The median filter output is clamped to [0, 1000] before writing to the database. +A 3-epoch median filter used to follow baseline subtraction. It erased a lone busy minute between still ones, which is what a turn-over looks like, so it was dropped for capSense and then for capSense2. The subtracted score is clamped to [0, 1000] before writing to the database. ## Presence Detection @@ -141,12 +135,13 @@ The baseline is maintained by the detector itself (`AdaptiveBaseline`), not by s - **Drift:** while the bed is empty and the reading is within the exit threshold, the baseline follows it with time constant `BASELINE_UP_TAU_S` (30 min). A load between the exit and enter thresholds is never learned as empty. - **Contamination:** any reading below baseline pulls it down with `BASELINE_DOWN_TAU_S` (2 min), so a baseline captured with someone in bed recovers minutes after they get up. +- **Exit to a new empty level:** the empty level itself moves with the bedding. On pod 88 the covers pulled back before bed lowered it by 7.7 (summed), the contamination rule learned that, and the made bed then read +7.5 over the baseline, above the exit threshold, for the rest of the day: 126 of 782 sessions ran to the 16 h cap. So while a side reads occupied, the detector tracks its occupied load (the peak of a 5-minute average of the rise, `OCCUPIED_LOAD_TAU_S`). If the rise falls below `EXIT_COLLAPSE_FRACTION` (25%) of that load and stays there for `EXIT_SETTLE_S` (10 min), the bed is empty at a new level: the baseline is reseeded to the reading and the absence is dated from where the collapse began, so `left_bed_at` lands on the real exit. Sleepers on pod 88 held 50-100% of their load through the night; the empty bed read 10-20%. The load is a plain mean until 5 minutes have been observed and sets no peak before then, so a transient while getting in, or right after a restart, can't raise the bar for the night. Only observed time counts toward the 10 minutes: a sensor gap or service downtime adds nothing. In single-sleeper mode the exit is dated no earlier than the other side last read occupied. The load, collapse start and observed collapse time are saved with the baseline, so a restart mid-collapse doesn't lose them; a saved value that isn't finite (or a load that isn't positive) drops the collapse, not the baseline. A collapse confirmed right after a restart dates the exit at its own start, not at the last presence before the restart. A collapse still under 10 minutes old when the 16 h cap fires is unconfirmed, so the cap closes the session as usual and logs the discarded collapse at INFO. The rule runs on capSense2 (Pod 5) only. capSense (Pod 3/4) has no recorded nights to set the fraction, and a sleeper there can hold a dip from +600 to +70 per channel. It is also off while the occupied load is at or under 4x the exit threshold (12 summed units at the default 6.0 threshold): no reading there is both under 25% of the load and above the exit threshold. Such a low-gain side has only the ordinary exit path (see Known Limitations). - **Stuck load:** after a session force-closed at `MAX_SESSION_S`, the current level becomes the new empty level. -- **Seeding:** the saved state file, else the active calibration profile — including this detector's own published baseline when the state file is lost (legacy capSense profiles with a z-score threshold get the raw-unit default) — else the first frame. A calibration profile newer than any seen — a manual recalibration — is adopted as a reseed. +- **Seeding:** the saved state file, else the active calibration profile — including this detector's own published baseline when the state file is lost (legacy capSense profiles with a z-score threshold get the raw-unit default) — else the first frame. A calibration profile newer than any seen — a manual recalibration — is adopted as a reseed; adopted mid-collapse, it drops the collapse and the occupied load is learned again (logged at INFO). - **Publishing:** every `BASELINE_PUBLISH_S` (15 min) the baseline is upserted to `calibration_profiles` with `source: "adaptive"`, in the calibrators' shape, so Node's occupancy check and the UI see the same level. Profiles marked adaptive never replace a live baseline; they only seed one when there is none. - Per-sample tracking time is capped at `BASELINE_MAX_STEP_S`, so a gap or restart can't move the baseline in one step. Frames with a missing side or capSense2 sentinels are ignored. -Replaying recorded capSense data starting from a baseline captured with the sleeper in bed, the baseline recovers within minutes of them getting up, sessions end at the real wake time rather than at the next recalibration, and a bedding shift of a few tens of units per channel produces no session. +Replaying pod 88's recorded capSense2 frames for 2026-10-09, the session ends at 15:28, the real exit, instead of at the 16 h cap the next morning. Replaying recorded capSense data starting from a baseline captured with the sleeper in bed, the baseline recovers within minutes of them getting up, sessions end at the real wake time rather than at the next recalibration, and a bedding shift of a few tens of units per channel produces no session. ## Single-Sleeper Mode @@ -171,7 +166,7 @@ A session starts on the first present sample and ends after `ABSENCE_TIMEOUT_S` An open session lives in memory, so it is checkpointed to `sleep-detector-state.json` next to `biometrics.db` (override with `SLEEP_DETECTOR_STATE_PATH`) every `STATE_SAVE_INTERVAL_S` and immediately on session start, bed-exit, and close. The write is tmp + fsync + rename. On startup the detector: -- **resumes** the session when the last saved sample is recent. If the occupant is still in bed the downtime counts as sleep; if absence is committed before any presence is seen, the exit is dated at the last pre-restart presence (they left while the detector was down). +- **resumes** the session when the last saved sample is recent. If the occupant is still in bed the downtime counts as sleep; if absence is committed before any presence is seen, the exit is dated at the last pre-restart presence (they left while the detector was down), unless an occupied-load collapse dates it. - **closes** it at the last presence when the gap exceeds `STATE_MAX_GAP_S` (30 min), since the downtime can't be attributed to sleep. - **skips replayed samples**: the RAW follower re-reads the current file from offset 0, so samples at or before the saved `last_ts` are ignored — for every side, with or without an open session. The presence hysteresis latch is restored too, so a reading between the exit and enter thresholds keeps its state. @@ -229,20 +224,23 @@ This filters phantom-session flicker (1-3 scattered non-still epochs per bucket) | `BASELINE_UP_TAU_S` | 30 min | Empty-bed drift tracking time constant | | `BASELINE_DOWN_TAU_S` | 2 min | Recovery when the reading falls below baseline | | `BASELINE_PUBLISH_S` | 15 min | Baseline written back to calibration_profiles | +| `EXIT_COLLAPSE_FRACTION` | 0.25 | Rise below this fraction of the occupied load counts as a collapse | +| `EXIT_SETTLE_S` | 10 min | A collapse held this long rebases the baseline and ends the stay | +| `OCCUPIED_LOAD_TAU_S` | 5 min | Averaging time for the occupied load | | `CALIBRATION_RELOAD_S` | 60 s | Poll calibration_profiles for updates | | `STATE_SAVE_INTERVAL_S` | 60 s | Checkpoint an open session to the state file | | `STATE_MAX_GAP_S` | 30 min | Longer downtime closes a restored session instead of resuming it | | Movement scale (capSense2) | 10x | Pod 5 float channels, deltas ~0.05-5.0 | -| Movement scale (capSense) | 0.5x | Pod 3 int ADC channels, deltas ~1-50 | +| Movement scale (capSense) | 1x | Pod 3/4 int channels; only each delta's excess over `CAPSENSE_NOISE_K` (6) times the channel's typical delta counts | | Movement cap | 1000 | Prevents outlier scores from sensor glitches | | Sentinel value | -1.0 | capSense2 firmware error indicator | | Reference nominal | 1.16 | Expected reference channel value | -| `PUMP_GUARD_S` | 3.0 s | Guard period after pump-off; 6 samples at ~2 Hz (#230) | +| `PUMP_GUARD_S` | 3.0 s | Guard period after a pump state change; 6 samples at ~2 Hz (#230) | +| `PUMP_RPM_CHANGE_FRACTION` | 0.1 | RPM step that counts as a pump state change | | `REF_ANOMALY_THRESHOLD` | 0.02 | Reference channel deviation for secondary pump detection | | `BASELINE_TRAILING_EPOCHS` | 30 | 30-minute trailing window for baseline subtraction | | `BASELINE_COLD_START_EPOCHS` | 10 | 10-minute minimum before baseline subtraction activates | | `BASELINE_PERCENTILE` | 5 | 5th percentile; represents quietest epoch in trailing window | -| `MEDIAN_FILTER_WINDOW` | 3 | 3-epoch median filter; suppresses isolated spikes | | `MIN_BUCKET_NONSTILL_FLOOR` | 2 | Chart density gate; minimum non-still epochs per rendered bucket | ## Literature References @@ -264,12 +262,14 @@ This filters phantom-session flicker (1-3 scattered non-still epochs per bucket) 4. **Scale calibration.** The `* 10` scale factor and 1000 cap were empirically tuned on one Pod 5. Different pod generations or mattress configurations may need adjustment. -5. **Pump gate frzHealth dependency.** The primary pump signal comes from `frzHealth` records which are Pod 5 only and arrive at ~0.06 Hz (~16s between updates). There is a detection latency window where pump vibrations may leak through before the first frzHealth record confirms pump-on. The reference channel anomaly detector (signal 2) partially covers this gap but is less reliable than direct RPM monitoring. +5. **Pump gate frzHealth dependency.** The pump state signal comes from `frzHealth` records, which are Pod 5 only and arrive every ~10 s. A pump start can leak a few seconds of transient before the first record confirms it. The reference channel anomaly detector partially covers this gap. 6. **Pump gate field name uncertainty.** The exact field names in frzHealth records for pump RPM (`pumpRpm`, `pump_rpm`, etc.) have not been confirmed on live hardware. The implementation checks multiple candidate names for robustness, but if the firmware uses an unexpected name, the primary signal will be inactive and only the reference anomaly detector will provide gating. 7. **Baseline subtraction cold start.** Movement scores during the first 10 minutes of a session are not baseline-subtracted, which may produce slightly elevated readings compared to later in the night. This is acceptable because the baseline requires sufficient history to be meaningful. -8. **Median filter smoothing behavior.** The 3-epoch median filter is causal (trailing window), so it does not depend on future epochs. It may still soften abrupt transitions, which is acceptable since movement data is not used for real-time alerting. +8. **Isolated glitches are not smoothed.** Without a median filter, a one-off sensor glitch that survives the sentinel and pump gates scores as one busy minute. The chart's density gate (two non-still epochs per bucket) keeps single minutes from rendering on their own. 9. **Calibrator RAW path coupling (Pod 5).** The calibrator reads RAW files from `RAW_DATA_DIR`, which must match the tmpfs path created by `sleepypod-tmpfs-prep` (`/persistent/biometrics`, per ADR-0018). A mismatch makes every reader see an empty directory, so the detector (and piezo/temperature calibration) receives no frames at all. The calibrator unit file declares `RequiresMountsFor=/persistent/biometrics` to surface this as a startup failure rather than a silent runtime degradation. + +10. **Low-gain sides have no exit to a new empty level.** The occupied-load collapse rule needs a reading under 25% of the load and above the exit threshold, which is impossible for loads at or under 4x the exit threshold. The rule is disabled there, so a low-gain side whose empty level moved can still hold a session open until the 16 h cap, as before the rule existed. diff --git a/docs/wiki/topics/sleep-detection.md b/docs/wiki/topics/sleep-detection.md index b290a57b1..b7b56de0c 100644 --- a/docs/wiki/topics/sleep-detection.md +++ b/docs/wiki/topics/sleep-detection.md @@ -26,23 +26,23 @@ The previous approach measured "how present" someone was, not "how much they mov 1. **Sentinel filter** — capSense2 firmware occasionally emits -1.0; handled via zero-order hold 2. **Pair averaging** — average redundant channel pairs (A1+A2)/2, etc. 3. **Reference compensation** — subtract ref channel drift from nominal 1.16 (common-mode rejection) -4. **Pump gate** — suppress deltas during pump activity (see below) +4. **Pump gate** — suppress deltas for 3 s after a pump state change (see below) 5. **Per-channel delta** — |current - previous| per channel 6. **60s epoch accumulation** → `raw_score = min(1000, sum × scale)` 7. **Baseline subtraction** — subtract P5 of trailing 30 epochs (removes slow-building noise) -8. **3-epoch median filter** — suppress isolated spike artifacts -9. **Clamp** to [0, 1000] +8. **Clamp** to [0, 1000] + +No median filter: it zeroed a turn-over, usually one busy minute between still ones. The capSense2 frame after each firmware sentinel is skipped instead. Scale factor is sensor-dependent: capSense2 (Pod 5) uses ×10, capSense (Pod 3) uses ×0.5 to normalize different ADC ranges. ## Pump Artifact Gating -Without gating, pump vibrations accumulate to raw scores of 60-200 per pump-active epoch, escalating from ~50 to 960-990 by early morning. +Pod 5 runs its pump continuously while a side is on, and a steady pump doesn't raise the delta floor. Gating the running pump zeroed 97% of a night's epochs, so only changes gate. -Three-signal detection: -1. **frzHealth pump RPM** (Pod 5 only, ~0.06 Hz) — any RPM > 0 means pump running +Two-signal detection: +1. **Pump state change** — 3-second guard after a side's frzHealth RPM starts, stops, or steps by >10% (shorter than [[piezo-processing|piezo's 5s]] — capacitive sensors are less sensitive to mechanical vibration) 2. **Reference channel anomaly** — |ref_delta| > 0.02 AND 2+ active channels correlate -3. **3-second guard period** after pump-off (shorter than [[piezo-processing|piezo's 5s]] — capacitive sensors are less sensitive to mechanical vibration) ## Presence Detection diff --git a/drizzle.package-experiments.config.ts b/drizzle.package-experiments.config.ts new file mode 100644 index 000000000..6a1af3e37 --- /dev/null +++ b/drizzle.package-experiments.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from 'drizzle-kit' + +export default defineConfig({ + schema: './src/db/package-experiment-schema.ts', + out: './src/db/package-experiment-migrations', + dialect: 'sqlite', +}) diff --git a/eslint.config.ts b/eslint.config.ts index e9c4580a1..5c7857037 100644 --- a/eslint.config.ts +++ b/eslint.config.ts @@ -25,6 +25,13 @@ export default defineConfig([ tseslint.configs.strict, pluginReact.configs.flat.recommended, ...nextVitals, + // Next selects its Babel parser for JS and for .mts/.cts config files. + // The strict TypeScript rules above need TypeScript scope references; + // otherwise used runtime variables are reported as type-only usages. + { + files: ['**/*.js', '**/*.mjs', '**/*.cjs', '**/*.mts', '**/*.cts'], + languageOptions: { parser: tseslint.parser }, + }, globalIgnores([ '.claude/**', diff --git a/instrumentation.ts b/instrumentation.ts index 0d23c6258..20e323ef6 100644 --- a/instrumentation.ts +++ b/instrumentation.ts @@ -1,484 +1,20 @@ -import { shutdownBaseController } from '@/src/hardware/base/instance' -/** - * Server startup and process lifecycle management. - * - * Handles: - * - Job scheduler initialization with exponential-backoff retry - * - DAC hardware monitor initialization (non-blocking) - * - Hardware daemon pre-flight validation - * - Centralized SIGTERM/SIGINT signal handling - * - Global unhandled rejection/exception handlers - * - Graceful shutdown sequencing with 10 s force-exit watchdog - * - * Entry points: - * - `register()` — Next.js instrumentation hook, called automatically on server start. - * Runs only in the Node.js runtime; skipped on the Edge runtime. - * - `initializeScheduler()` — idempotent, safe to call from app code as a fallback. - */ - -import { getJobManager, shutdownJobManager } from '@/src/scheduler' -import { getAutomationEngine, shutdownAutomationEngine } from '@/src/automation' -import { startTemperatureController, stopTemperatureController } from '@/src/temperature/instance' -import { closeDatabase, closeBiometricsDatabase } from '@/src/db' -import { startBiometricsRetention, stopBiometricsRetention } from '@/src/db/retention' -import { startAutomationRunsRetention, stopAutomationRunsRetention } from '@/src/db/automationRunsRetention' -import { startHealthSampler, stopHealthSampler } from '@/src/lib/healthSampler' -import { getDacMonitor, shutdownDacMonitor } from '@/src/hardware/dacMonitor.instance' -import { startPiezoStreamServer, shutdownPiezoStreamServer } from '@/src/streaming/piezoStream' -import { startBonjourAnnouncement, stopBonjourAnnouncement } from '@/src/streaming/bonjourAnnounce' -import { startMqttBridge, shutdownMqttBridge } from '@/src/streaming/mqttBridge' -import { initializeKeepalives, shutdownKeepalives } from '@/src/services/temperatureKeepalive' -import { startAutoOffWatcher, stopAutoOffWatcher } from '@/src/services/autoOffWatcher' -import { startDatabaseIntegrityChecks, stopDatabaseIntegrityChecks } from '@/src/db/integrity' -import { startPerformanceMonitoring, stopPerformanceMonitoring, recordStartupPhase } from '@/src/lib/serverPerformance' -import { shutdownHomeKit, startHomeKitIfEnabled } from '@/src/homekit' - -let isInitialized = false -let isShuttingDown = false -let handlersRegistered = false -let initializationPromise: Promise | null = null -let hardwareReady = false -let hardwarePromise: Promise | null = null - -/** - * Centralized graceful shutdown coordinator. - * Sequences: wait for in-flight jobs → shutdown scheduler → close database → exit - */ -async function gracefulShutdown(signal: string): Promise { - if (isShuttingDown) return - isShuttingDown = true - - console.log(`Received ${signal}, starting graceful shutdown...`) - - // Force exit after 10s if graceful shutdown hangs - const forceExitTimer = setTimeout(() => { - console.error('Graceful shutdown timed out after 10s, forcing exit') - process.exit(1) - }, 10_000) - forceExitTimer.unref() - - stopPerformanceMonitoring() - await stopDatabaseIntegrityChecks() - - // Step 0: Stop keepalive timers - try { - await stopTemperatureController() - shutdownBaseController() - shutdownKeepalives() - } - catch (error) { - console.error('Error shutting down keepalives:', error) - } - - // Step 1: Shutdown scheduler (waits for in-flight jobs internally) - try { - await shutdownJobManager() - } - catch (error) { - console.error('Error shutting down scheduler:', error) - } - - // Step 1a: Stop the Autopilot rules engine tick - try { - await shutdownAutomationEngine() - } - catch (error) { - console.error('Error shutting down automation engine:', error) - } - - // Step 2: Shutdown piezo stream server - try { - await shutdownPiezoStreamServer() - } - catch (error) { - console.error('Error shutting down piezo stream server:', error) - } - - // Step 2b: Shutdown MQTT bridge (publish offline + close client cleanly) - try { - await shutdownMqttBridge() - } - catch (error) { - console.error('Error shutting down MQTT bridge:', error) - } - - // Step 3: Stop Bonjour announcement - try { - stopBonjourAnnouncement() - } - catch (error) { - console.error('Error stopping Bonjour:', error) - } - - // Step 3a: Stop HomeKit bridge (unpublish HAP mDNS, close TCP listener) - try { - await shutdownHomeKit() - } - catch (error) { - console.error('Error shutting down HomeKit:', error) - } - - // Step 4: Stop auto-off watcher (await in-flight power-off calls) - try { - await stopAutoOffWatcher() - } - catch (error) { - console.error('Error stopping auto-off watcher:', error) - } - - // Step 5: Shutdown DAC monitor - try { - await shutdownDacMonitor() - } - catch (error) { - console.error('Error shutting down DacMonitor:', error) - } - - // Step 6: Stop biometrics retention and health sampling before closing DB - try { - stopBiometricsRetention() - stopAutomationRunsRetention() - stopHealthSampler() - } - catch (error) { - console.error('Error stopping biometrics retention:', error) - } - - // Step 7: Close database connections - try { - closeDatabase() - closeBiometricsDatabase() - } - catch (error) { - console.error('Error closing database:', error) - } - - process.exit(0) -} - -/** - * Register global process handlers (signal handlers, error handlers). - * Safe to call multiple times - only registers once. - */ -function registerGlobalHandlers(): void { - if (handlersRegistered) return - handlersRegistered = true - - // Centralized signal handlers - process.on('SIGTERM', () => gracefulShutdown('SIGTERM')) - process.on('SIGINT', () => gracefulShutdown('SIGINT')) - - // Global unhandled rejection handler - log but don't crash - process.on('unhandledRejection', (reason: unknown) => { - console.error('Unhandled promise rejection:', reason) - // Don't exit - let the process continue serving other requests - }) - - // Global uncaught exception handler - log and attempt graceful shutdown - process.on('uncaughtException', (error: Error & { code?: string, address?: string, port?: number }) => { - // mDNS EPERM is non-fatal — iptables blocks multicast, just log and continue - if (error.code === 'EPERM' && error.address === '224.0.0.251' && error.port === 5353) { - console.warn('[bonjour] mDNS send blocked by iptables (non-fatal):', error.message) - return - } - console.error('Uncaught exception:', error) - // Process state may be corrupted, attempt graceful shutdown - gracefulShutdown('uncaughtException') - }) -} - -/** - * Retry a function with exponential backoff. - */ -async function withRetry( - fn: () => Promise, - label: string, - maxAttempts: number = 3, - baseDelayMs: number = 500 -): Promise { - let lastError: unknown - for (let attempt = 1; attempt <= maxAttempts; attempt++) { - if (isShuttingDown) throw new Error('Startup cancelled during shutdown') - try { - return await fn() - } - catch (error) { - lastError = error - if (attempt < maxAttempts) { - const delay = baseDelayMs * Math.pow(2, attempt - 1) - console.warn( - `${label} failed (attempt ${attempt}/${maxAttempts}), retrying in ${delay}ms:`, - error instanceof Error ? error.message : error - ) - await new Promise(resolve => setTimeout(resolve, delay)) - } - } - } - throw lastError -} - -/** - * Validate hardware daemon connectivity on startup. - * Logs a warning if unavailable but does not crash. - */ -// Hardware validation removed — the DacMonitor handles connection lifecycle. -// The DAC socket server starts first, then the monitor waits for frankenfirmware. - -/** - * Initialize the DAC hardware monitor. - * Non-blocking — logs a warning on failure but does not crash. - */ -const initializeDacMonitor = async (): Promise => { - try { - await getDacMonitor() - if (isShuttingDown) { - await shutdownDacMonitor() - return - } - // A restart mid-alarm or mid-snooze (deploy, crash) resumes it. - const { restoreActiveAlarms } = await import('@/src/hardware/alarmState') - const { restoreSnoozes } = await import('@/src/hardware/snoozeManager') - await restoreActiveAlarms() - restoreSnoozes() - } - catch (error) { - console.warn( - 'WARNING: DacMonitor failed to start:', - error instanceof Error ? error.message : error - ) - } -} - -async function initializeHardware(): Promise { - if (hardwareReady || isShuttingDown) return - if (hardwarePromise) return hardwarePromise - hardwarePromise = prepareHardware().finally(() => { - hardwarePromise = null - }) - return hardwarePromise +/** Runtime-neutral entrypoint. Keep literal runtime checks around dynamic + * imports so Next can exclude hardware/database code from its Edge bundle. */ +export function shouldRunInstrumentation(env: Record = process.env): boolean { + return !env.NEXT_RUNTIME || env.NEXT_RUNTIME === 'nodejs' } -async function prepareHardware(): Promise { - // Rehydrate the pump stall guard from persisted un-acknowledged alerts - // before anything that consults shouldBlock (keepalives, scheduler jobs) - // can re-energize a side whose fault predates this boot. - try { - const { rehydrate } = await import('@/src/hardware/pumpStallGuard') - rehydrate() - } - catch (error) { - console.warn( - '[pumpStallGuard] rehydration failed:', - error instanceof Error ? error.message : error - ) - } - - if (isShuttingDown) return - - // Start DAC socket server FIRST — this is the single listener on dac.sock. - // frankenfirmware will connect to it. Everything else (DacMonitor, device - // router, health checks) uses this server's connection. - try { - const { startDacServer } = await import('@/src/hardware/dacMonitor.instance') - await startDacServer() - } - catch (error) { - console.warn('[DAC] Socket server failed to start:', error instanceof Error ? error.message : error) - } - - if (isShuttingDown) return - - // Start DAC monitor (non-blocking — waits for frankenfirmware to connect) - initializeDacMonitor() - - if (isShuttingDown) return - hardwareReady = true - try { - startPiezoStreamServer() - } - catch (error) { - console.warn('WARNING: Piezo stream server failed to start:', error instanceof Error ? error.message : error) +export async function register(): Promise { + if (!process.env.NEXT_RUNTIME || process.env.NEXT_RUNTIME === 'nodejs') { + const { registerNodeInstrumentation } = await import('./src/server/instrumentation-node') + await registerNodeInstrumentation() } - recordStartupPhase('hardware-services-started') } -/** - * Initialize the job scheduler - * Safe to call multiple times - will only initialize once - */ +/** Compatibility entrypoint for callers that initialize the scheduler directly. */ export async function initializeScheduler(): Promise { - if (isInitialized || isShuttingDown) return - if (initializationPromise) return initializationPromise - initializationPromise = initializeBackgroundServices().finally(() => { - initializationPromise = null - }) - return initializationPromise -} - -async function initializeBackgroundServices(): Promise { - try { - await initializeHardware() - if (isShuttingDown) return - - // Record System → Health's data-path history once a minute (non-blocking). - // Started before the scheduler so a scheduler that fails to load still - // leaves a history of the failure. - startHealthSampler() - - const schedulerStartedAt = performance.now() - console.log('Initializing job scheduler...') - const jobManager = await withRetry( - () => getJobManager(), - 'Job manager initialization' - ) - if (isShuttingDown) return - recordStartupPhase('scheduler-ready', schedulerStartedAt) - const scheduler = jobManager.getScheduler() - const jobs = scheduler.getJobs() - - console.log(`Job scheduler initialized with ${jobs.length} scheduled jobs`) - - // Log next scheduled jobs for visibility - const upcomingJobs = jobs - .map((job) => { - const nextRun = scheduler.getNextInvocation(job.id) - return { - id: job.id, - type: job.type, - nextRun: nextRun ? nextRun.toISOString() : 'N/A', - } - }) - .filter(job => job.nextRun !== 'N/A') - .sort((a, b) => { - if (a.nextRun === 'N/A' || b.nextRun === 'N/A') return 0 - return new Date(a.nextRun).getTime() - new Date(b.nextRun).getTime() - }) - .slice(0, 5) - - if (upcomingJobs.length > 0) { - console.log('Next scheduled jobs:') - for (const job of upcomingJobs) { - console.log(` - ${job.id}: ${job.nextRun}`) - } - } - - isInitialized = true - - // Optional integrations do not hold up HTTP or the remaining services. - void startMqttBridge().then(() => { - if (isShuttingDown) return shutdownMqttBridge() - recordStartupPhase('mqtt-started') - }).catch((error) => { - console.warn('WARNING: MQTT bridge failed to start:', error instanceof Error ? error.message : error) - }) - - // Start auto-off watcher (polls biometrics DB for bed-exit events) - startAutoOffWatcher() - - // Start Bonjour/mDNS announcement (non-blocking) - startBonjourAnnouncement() - - // Start HomeKit bridge if user has opted in (non-blocking) - startHomeKitIfEnabled().then(() => { - if (isShuttingDown) return shutdownHomeKit() - }).catch((error) => { - console.warn( - '[homekit] startup failed:', - error instanceof Error ? error.message : error, - ) - }) - - // Start biometrics time-series retention loop (non-blocking) - startBiometricsRetention() - startAutomationRunsRetention() - - // Boot the Autopilot rules engine beside the scheduler (non-blocking). - // Shares the same hardware path; no-op until automations are created. - getAutomationEngine().catch((error) => { - console.warn('[automation] engine failed to start:', error instanceof Error ? error.message : error) - }).then(async () => { - if (isShuttingDown) return shutdownAutomationEngine() - await startTemperatureController() - if (isShuttingDown) { - await stopTemperatureController() - return - } - initializeKeepalives() - }).catch((error) => { - console.warn( - '[temperature] controller failed to start:', - error instanceof Error ? error.message : error, - ) - }) - } - catch (error) { - console.error('Failed to initialize job scheduler:', error) - // Don't crash the app if scheduler fails to initialize - } -} - -/** - * Next.js instrumentation hook (if supported) - * Automatically called by Next.js on server startup - */ -/** - * Gate for `register()`. Returns true only when the instrumentation hook - * should execute its body. - * - * Next.js calls the instrumentation hook **once per server runtime** — both - * Node and Edge bundles trigger it. The previous gate (`NEXT_RUNTIME==='nodejs' - * || typeof window === 'undefined'`) was too permissive: `typeof window` is - * undefined in every server runtime, so Edge slipped through and caused - * double-init (most visibly: two MQTT bridge clients connecting from the - * same pod). Strict check on `NEXT_RUNTIME` when set; allow when unset - * (tests, scripts). - */ -export function shouldRunInstrumentation(env: Record = process.env): boolean { - if (env.NEXT_RUNTIME && env.NEXT_RUNTIME !== 'nodejs') return false - return true -} - -export async function register(): Promise { - if (shouldRunInstrumentation()) { - // Register global handlers first (before any initialization that could fail) - registerGlobalHandlers() - startPerformanceMonitoring() - const migrationsStartedAt = performance.now() - - // Run pending database migrations before starting the app - const { runMigrations, seedDefaultData } = await import('@/src/db/migrate') - await runMigrations() - await seedDefaultData() - recordStartupPhase('migrations-ready', migrationsStartedAt) - - // Skip hardware initialization in CI — no dac.sock, no sensors, no scheduler needed. - // The server still starts and serves API routes (including /api/openapi.json). - if (process.env.CI) { - console.log('CI environment detected — skipping hardware initialization') - return - } - - // Validate and auto-repair iptables rules (mDNS, LAN access, NTP) - try { - const { checkAndRepairIptables } = await import('@/src/hardware/iptablesCheck') - const result = checkAndRepairIptables() - if (result.repaired.length > 0) { - console.warn(`[startup] Repaired ${result.repaired.length} missing iptables rules:`, result.repaired.join(', ')) - } - else if (result.ok) { - console.log('[startup] iptables rules verified') - } - } - catch (e) { - console.warn('[startup] iptables check skipped:', e instanceof Error ? e.message : e) - } - - await initializeHardware() - if (isShuttingDown) return - startDatabaseIntegrityChecks() - // The clock gate and scheduler loading still finish before scheduled writers - // start, but no longer prevent clients from loading settings or diagnostics. - void initializeScheduler() - recordStartupPhase('http-initialization-ready') + if (!process.env.NEXT_RUNTIME || process.env.NEXT_RUNTIME === 'nodejs') { + const node = await import('./src/server/instrumentation-node') + await node.initializeScheduler() } } diff --git a/modules/common/biometrics/__init__.py b/modules/common/biometrics/__init__.py new file mode 100644 index 000000000..51da23d76 --- /dev/null +++ b/modules/common/biometrics/__init__.py @@ -0,0 +1 @@ +"""Versioned biometric configuration contracts shared with TypeScript.""" diff --git a/modules/common/biometrics/build_identity.py b/modules/common/biometrics/build_identity.py new file mode 100644 index 000000000..aad7c47a0 --- /dev/null +++ b/modules/common/biometrics/build_identity.py @@ -0,0 +1,64 @@ +"""Versioned identity of the shipped replay sources and installed runtime. + +Distribution RECORD hashes identify installed wheel metadata, not an attestation +that every installed binary is unmodified. No archive or patient data is read. +""" +import hashlib +import importlib.metadata +from pathlib import Path +import platform +import sys +import sysconfig + +from .config import config_hash + +SOURCE_FILES = ( + 'common/__init__.py', 'common/cbor_raw.py', + 'common/biometrics/__init__.py', 'common/biometrics/build_identity.py', + 'common/biometrics/config.py', 'common/biometrics/manifest.v1.json', + 'common/biometrics/frames.py', 'common/biometrics/gates.py', + 'common/biometrics/processing.py', 'common/biometrics/routing.py', + 'common/biometrics/signal.py', 'common/biometrics/raw_artifacts.py', + 'common/biometrics/raw_records.py', 'common/biometrics/retained_replay.py', + 'common/biometrics/execution_lock.py', + 'common/biometrics/replay.py', 'common/biometrics/replay_worker.py', + 'piezo-processor/pyproject.toml', 'piezo-processor/uv.lock', +) +MAX_FILE_BYTES = 16 * 1024 * 1024 + + +def _digest_file(path): + digest = hashlib.sha256() + size = 0 + with path.open('rb') as stream: + while True: + chunk = stream.read(65536) + if not chunk: + break + size += len(chunk) + if size > MAX_FILE_BYTES: + raise ValueError('Replay build identity file exceeds its bound') + digest.update(chunk) + return digest.hexdigest() + + +def replay_build_identity(): + root = Path(__file__).resolve().parents[2] + sources = {name: _digest_file(root / name) for name in SOURCE_FILES} + dependencies = {} + for name in ('numpy', 'scipy'): + distribution = importlib.metadata.distribution(name) + record = distribution.read_text('RECORD') + wheel = distribution.read_text('WHEEL') + if not record or not wheel or len(record.encode()) > MAX_FILE_BYTES or len(wheel.encode()) > MAX_FILE_BYTES: + raise ValueError('Replay dependency build metadata is unavailable') + dependencies[name] = {'version': distribution.version, + 'record_sha256': hashlib.sha256(record.encode()).hexdigest(), + 'wheel_sha256': hashlib.sha256(wheel.encode()).hexdigest()} + identity = {'version': 'waveform-python-build-v1', 'sources': sources, + 'dependencies': dependencies, 'runtime': { + 'implementation': platform.python_implementation(), 'version': sys.version, + 'cache_tag': sys.implementation.cache_tag, 'soabi': sysconfig.get_config_var('SOABI'), + 'machine': platform.machine(), 'system': platform.system(), 'byteorder': sys.byteorder, + }} + return {'hash': config_hash(identity), 'identity': identity} diff --git a/modules/common/biometrics/collection_worker.py b/modules/common/biometrics/collection_worker.py new file mode 100644 index 000000000..06a89a676 --- /dev/null +++ b/modules/common/biometrics/collection_worker.py @@ -0,0 +1,55 @@ +"""Bounded cleanup of server-owned retained waveforms, never the archive.""" +import argparse +from contextlib import closing +import json +from pathlib import Path +import sqlite3 +import sys +import time + +from .raw_artifacts import RawArtifactStore +from .replay_worker import _unique_object, _invalid_constant, configure_process_limits + + +def run(input_stream, output_stream, database_path, retained_directory): + payload = input_stream.read(1025) + if len(payload) > 1024: + raise ValueError('RAW cleanup input exceeds byte budget') + work = json.loads(payload, object_pairs_hook=_unique_object, parse_constant=_invalid_constant) + if work != {'version': 'raw-collection-work-v1'}: + raise ValueError('Invalid RAW cleanup envelope') + directory = Path(retained_directory) + removed_released = removed_orphans = 0 + if directory.is_symlink(): + raise ValueError('Invalid retained directory') + if directory.exists(): + database = Path(database_path) + if database.is_symlink() or not database.is_file(): + raise ValueError('Invalid biometrics database') + with closing(sqlite3.connect(database.resolve().as_uri() + '?mode=rw', uri=True, timeout=1)) as connection: + connection.execute('PRAGMA foreign_keys = ON') + store = RawArtifactStore(directory) + now_ms = time.time_ns() // 1_000_000 + removed_released = len(store.collect_released(connection, now_ms=now_ms)) + removed_orphans = len(store.collect_orphans(connection, now_ms=now_ms)) + result = {'version': 'raw-collection-result-v1', 'released': removed_released, 'orphans': removed_orphans} + output_stream.write((json.dumps(result, separators=(',', ':')) + '\n').encode()) + output_stream.flush() + + +def main(): + parser = argparse.ArgumentParser(description='Collect retained RAW files') + parser.add_argument('--database-path', required=True) + parser.add_argument('--retained-directory', required=True) + args = parser.parse_args() + try: + configure_process_limits() + run(sys.stdin.buffer, sys.stdout.buffer, args.database_path, args.retained_directory) + return 0 + except Exception: + sys.stderr.write('RAW collection worker failed\n') + return 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/modules/common/biometrics/config.py b/modules/common/biometrics/config.py new file mode 100644 index 000000000..c41d0c7f5 --- /dev/null +++ b/modules/common/biometrics/config.py @@ -0,0 +1,160 @@ +"""Read the shipped contract; never write measured sensor calibration. + +finite-binary64-json-v1 encodes exact decimal expansions of IEEE-754 values. +Unlike language-specific JSON float printers, this gives identical bytes on +both runtimes, including subnormals and negative zero. Object keys use UTF-16 +ordering (the JavaScript ordering); arrays retain their meaningful order. +""" +import copy +import hashlib +import json +import math +from pathlib import Path + +_MANIFEST = json.loads(Path(__file__).with_name("manifest.v1.json").read_text()) + + +def canonical_json(value): + nodes = 0 + + def visit(item, depth): + nonlocal nodes + nodes += 1 + if nodes > 20000 or depth > 32: + raise ValueError("JSON resource limit") + if item is None: + return "null" + if isinstance(item, bool): + return "true" if item else "false" + if isinstance(item, (int, float)): + number = float(item) + if not math.isfinite(number): + raise ValueError("Non-finite JSON number") + if number == 0: + return "0" + numerator, denominator = abs(number).as_integer_ratio() + places = denominator.bit_length() - 1 + digits = str(numerator * 5 ** places) + if places: + digits = digits.zfill(places + 1) + digits = (digits[:-places] + "." + digits[-places:]).rstrip("0").rstrip(".") + return ("-" if number < 0 else "") + digits + if isinstance(item, str): + try: + length = len(item.encode("utf-16-be")) // 2 + except UnicodeEncodeError as error: + raise ValueError("Invalid JSON string") from error + if length > 65536: + raise ValueError("Invalid JSON string") + return json.dumps(item, ensure_ascii=False, separators=(",", ":")) + if isinstance(item, list): + return "[" + ",".join(visit(child, depth + 1) for child in item) + "]" + if isinstance(item, dict): + if any(not isinstance(key, str) for key in item): + raise ValueError("Expected string keys") + try: + keys = sorted(item, key=lambda key: key.encode("utf-16-be")) + except UnicodeEncodeError as error: + raise ValueError("Invalid JSON string") from error + return "{" + ",".join(visit(key, depth + 1) + ":" + visit(item[key], depth + 1) + for key in keys) + "}" + raise ValueError("Expected finite JSON data") + + return visit(value, 0) + + +def config_hash(value): + return hashlib.sha256(canonical_json(value).encode("utf-8")).hexdigest() + + +def manifest(): + return copy.deepcopy(_MANIFEST) + + +def _validate(schema, value, path): + def fail(): + raise ValueError("Invalid biometric parameter: " + path) + if "const" in schema and value != schema["const"]: + fail() + if "enum" in schema and value not in schema["enum"]: + fail() + kind = schema.get("type") + if kind == "object": + if not isinstance(value, dict): + fail() + properties = schema.get("properties", {}) + if any(key not in value for key in schema.get("required", [])): + fail() + for key, child in value.items(): + if key not in properties: + fail() + _validate(properties[key], child, path + "." + key) + elif kind == "array": + if not isinstance(value, list) or len(value) > schema.get("maxItems", math.inf): + fail() + for child in value: + _validate(schema["items"], child, path) + elif kind in ("number", "integer"): + if isinstance(value, bool) or not isinstance(value, (float, int)): + fail() + if not math.isfinite(value): + fail() + if kind == "integer" and (value != int(value) or abs(value) > 9007199254740991): + fail() + if value < schema.get("minimum", -math.inf) or value > schema.get("maximum", math.inf): + fail() + elif kind == "string": + if not isinstance(value, str): + fail() + elif kind == "boolean": + if not isinstance(value, bool): + fail() + else: + raise ValueError("Unsupported contract schema: " + path) + + +def validate_config(value): + canonical_json(value) + _validate(_MANIFEST["parameters"], value, "config") + for path in _MANIFEST["orderedRanges"]: + node = value + for key in path: + node = node[key] + if node["min"] > node["max"]: + raise ValueError("Invalid ordered range: " + ".".join(path)) + return value + + +def resolve_config(overrides, base_hash=None): + expected = config_hash(_MANIFEST) + if base_hash is not None and base_hash != expected: + raise ValueError("PROFILE_INCOMPATIBLE: unknown base manifest") + if len(overrides) > 64: + raise ValueError("RESOURCE_LIMIT: maximum 64 overrides") + resolved = copy.deepcopy(_MANIFEST["defaults"]) + paths = set() + for override in overrides: + if set(override) != {"path", "value"}: + raise ValueError("Invalid override fields") + path = override["path"] + if path in paths: + raise ValueError("Duplicate override path") + paths.add(path) + parts = path.split(".") + schema = _MANIFEST["parameters"] + parent = resolved + for index, part in enumerate(parts): + if schema["type"] != "object" or part not in schema["properties"]: + raise ValueError("Unknown override path: " + path) + schema = schema["properties"][part] + if index == len(parts) - 1: + if schema["type"] == "object": + raise ValueError("Override leaves or entire ordered arrays") + _validate(schema, override["value"], path) + parent[part] = copy.deepcopy(override["value"]) + else: + parent = parent[part] + validate_config(resolved) + return {"resolved": resolved, "hash": config_hash(resolved), "manifestHash": expected, + "overrides": copy.deepcopy(overrides), + "provenance": {override["path"]: "override" for override in overrides}} diff --git a/modules/common/biometrics/execution_lock.py b/modules/common/biometrics/execution_lock.py new file mode 100644 index 000000000..45412bb50 --- /dev/null +++ b/modules/common/biometrics/execution_lock.py @@ -0,0 +1,29 @@ +"""Host-wide ownership for trusted numerical replay and shadow workers. + +Keep the legacy shadow path so workers from before an upgrade still contend. +No request or CLI option can select another lock. The optional function argument +is for trusted tests. This lock does not replace worker resource limits. +""" +import os +import stat + +EXECUTION_LOCK_PATH = '/run/sleepypod-shadow.lock' + + +def acquire_execution_lock(path=EXECUTION_LOCK_PATH): + """Nonblocking flock; never unlink or truncate the shared runtime inode.""" + import fcntl + fd = os.open(path, os.O_CREAT | os.O_RDWR | os.O_CLOEXEC | os.O_NOFOLLOW | os.O_NONBLOCK, 0o600) + try: + info = os.fstat(fd) + if (not stat.S_ISREG(info.st_mode) or info.st_nlink != 1 + or info.st_uid != os.geteuid() or info.st_mode & 0o077 or info.st_size != 0): + raise RuntimeError('Invalid biometric execution lock') + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + current = os.stat(path, follow_symlinks=False) + if (current.st_dev, current.st_ino) != (info.st_dev, info.st_ino): + raise RuntimeError('Biometric execution lock replaced') + return fd + except BaseException: + os.close(fd) + raise diff --git a/modules/common/biometrics/frames.py b/modules/common/biometrics/frames.py new file mode 100644 index 000000000..7d3dc4086 --- /dev/null +++ b/modules/common/biometrics/frames.py @@ -0,0 +1,33 @@ +"""Shared little-endian firmware sample decoding and lost-marker repair.""" +import numpy as np + + +LOST_SAMPLE = np.iinfo(np.int32).max + + +def decode_int32_samples(buf) -> np.ndarray: + """Decode an int32 sample buffer, tolerating truncated tails and + repairing lost-sample markers. + + A RAW record cut mid-write can carry a payload whose length is not a + multiple of 4; np.frombuffer would raise ValueError and kill the module. + Truncate to whole samples instead. A LOST_SAMPLE marker is replaced by + linear interpolation between its neighbours (the nearest valid sample at + a record edge); a record with no valid sample decodes as empty. + """ + if not isinstance(buf, (bytes, bytearray, memoryview)): + return np.empty(0, dtype=np.dtype(" None: + if record.get("type") != "frzHealth": + return + for side in ("left", "right"): + data = record.get(side) + if not isinstance(data, dict): + continue + rpm = 0.0 + for key in ("pumpRpm", "pump_rpm", "pumpRPM", "rpm"): + val = data.get(key) + if val is not None: + try: + rpm = float(val) + except (TypeError, ValueError): + rpm = 0.0 + break + # New NATS firmware nests the same reading at side.pump.rpm. + pump = data.get("pump") + if rpm == 0 and isinstance(pump, dict): + val = pump.get("rpm") + if val is not None: + try: + rpm = float(val) + except (TypeError, ValueError): + rpm = 0.0 + if rpm == 0: + duty = next((data.get(key) for key in + ("pumpDuty", "pump_duty", "duty") + if data.get(key) is not None), None) + if duty is None and isinstance(pump, dict): + duty = next((pump.get(key) for key in ("duty", "power") + if pump.get(key) is not None), None) + if duty is not None: + try: + rpm = PUMP_ACTIVE_RPM_MIN + 1.0 if float(duty) > 0 else 0.0 + except (TypeError, ValueError): + rpm = 0.0 + + now_active = rpm >= PUMP_ACTIVE_RPM_MIN + if self._was_active[side] and not now_active: + self._pump_off_at[side] = self._clock() + self._pump_rpm[side] = rpm + self._was_active[side] = now_active + + def is_side_pump_active(self, side: str) -> bool: + """Pump active = currently running OR within the trailing guard window.""" + if self._pump_rpm[side] >= PUMP_ACTIVE_RPM_MIN: + return True + return self._clock() - self._pump_off_at[side] < PUMP_OFF_GUARD_S + + def is_asymmetric_for(self, side: str) -> bool: + """Own pump active, opposite pump idle (and outside its own guard). + This is the configuration that produces phantom presence: pump + vibration on the powered side without competing signal on the + other side to balance/reject.""" + other = "right" if side == "left" else "left" + return self.is_side_pump_active(side) and not self.is_side_pump_active(other) + + def is_symmetric_active(self) -> bool: + """Both pumps active. RPM mismatch (Pod 5 live: 1940/2004) produces + a beat frequency in the cardiac band, generating bilateral phantom + vitals. PumpGate catches symmetric spikes but not steady-state.""" + return self.is_side_pump_active("left") and self.is_side_pump_active("right") + + +class PumpGate: + """Detects pump activity from dual-channel energy spikes. + + In streaming mode, each incoming record's L+R chunks are checked. + If a pump is detected, a guard period is set during which all + subsequent records are dropped (not ingested into side buffers). + """ + + def __init__(self, fs: float = 500, *, clock: Callable[[], float]): + self._clock = clock + self._fs = fs + self._baseline: Optional[float] = None + self._pump_until: float = 0.0 # monotonic seconds when guard expires + + def is_pump_active(self) -> bool: + return self._clock() < self._pump_until + + def check(self, left_chunk: np.ndarray, right_chunk: np.ndarray) -> bool: + """Check a single record's L+R chunks for pump activity. + + Returns True if the record should be DROPPED (pump detected or + guard period active), False if the record is clean. + """ + # If we're still in the guard period, drop + if self.is_pump_active(): + return True + + # Compute energy for both channels + le = float(np.mean(left_chunk.astype(np.float64) ** 2)) + re = float(np.mean(right_chunk.astype(np.float64) ** 2)) + + if le == 0 or re == 0: + return False + + ratio = min(le, re) / max(le, re) + + if self._baseline is not None and self._baseline > 0: + spike = max(le, re) / self._baseline + if ratio > PUMP_CORRELATION_MIN and spike > PUMP_ENERGY_MULTIPLIER: + # Pump detected — set guard period + self._pump_until = self._clock() + PUMP_GUARD_S + log.debug("Pump detected (spike=%.1f, ratio=%.2f), " + "guard until +%.0fs", spike, ratio, PUMP_GUARD_S) + return True + + # Update baseline with exponential moving average (only on clean data). + # Guard: if first record has unusually high energy (module started during + # pump), don't initialize baseline from it — wait for a quieter sample. + avg = (le + re) / 2 + if self._baseline is None: + self._baseline = avg + self._baseline_samples = 1 + elif avg < self._baseline * 3: + self._baseline = 0.95 * self._baseline + 0.05 * avg + self._baseline_samples = getattr(self, '_baseline_samples', 0) + 1 + # If early samples are settling, allow baseline to drop faster + if self._baseline_samples < 10: + self._baseline = min(self._baseline, avg) + + return False + +# --------------------------------------------------------------------------- +# Presence detection — hysteresis state machine (Paalasmaa et al. 2012) +# --------------------------------------------------------------------------- + +class PresenceDetector: + """Hysteresis-based presence detection with autocorrelation quality. + + Uses median std of 1-10 Hz filtered signal in 5-second windows as + the primary feature, with autocorrelation quality as secondary. + Requires 3 consecutive low windows to exit PRESENT state. + + The enter/exit gap protects against an empty bed that vibrates (pump, + prime). When the capacitance sensor already confirms a body is in bed + (`relaxed=True`), that protection costs an hour of missed vitals per + night: a sleeper who rolls into a weak-signal position exits after three + windows and then has to move hard to re-enter. Relaxed re-entry uses the + exit criteria, so presence resumes as soon as the signal stops being weak. + """ + ABSENT = 0 + PRESENT = 1 + + def __init__(self): + self.state = self.ABSENT + self.consecutive_low = 0 + self.enter_threshold = 400_000 + self.exit_threshold = 150_000 + self.exit_count = 3 # 3 consecutive low windows to declare absent + self.acr_threshold = 0.45 + + def _is_weak(self, window_std: float, acr_qual: float) -> bool: + return window_std < self.exit_threshold and acr_qual < self.acr_threshold * 0.5 + + def update(self, window_std: float, acr_qual: float, relaxed: bool = False) -> bool: + """Update state and return True if user is present. + + `relaxed`: the bed is known occupied by another sensor; re-enter on + anything that would not count as an exit window.""" + if self.state == self.ABSENT: + if relaxed: + entered = not self._is_weak(window_std, acr_qual) + else: + entered = window_std > self.enter_threshold or acr_qual > self.acr_threshold + if entered: + self.state = self.PRESENT + self.consecutive_low = 0 + return True + return False + else: + if self._is_weak(window_std, acr_qual): + self.consecutive_low += 1 + if self.consecutive_low >= self.exit_count: + self.state = self.ABSENT + self.consecutive_low = 0 + return False + return True # still present (hysteresis) + else: + self.consecutive_low = 0 + return True + diff --git a/modules/common/biometrics/inspection_worker.py b/modules/common/biometrics/inspection_worker.py new file mode 100644 index 000000000..07402fa0f --- /dev/null +++ b/modules/common/biometrics/inspection_worker.py @@ -0,0 +1,69 @@ +"""Inspect one cold archive per request without retaining or changing its bytes.""" +import argparse +from dataclasses import asdict +import json +import sys +import zlib + +from .raw_artifacts import SOURCE_NAME +from .raw_sources import ArchiveSource, archive_names +from .replay_worker import _unique_object, _invalid_constant, configure_process_limits + +MAX_INPUT_BYTES = 1024 +MAX_OUTPUT_BYTES = 65536 + + +def inspect_next(archive_directory, after_name): + # Names are a traversal cursor, not a snapshot of a mutable directory. A + # subsequent sweep from null discovers additions before the current cursor. + try: + names = archive_names(archive_directory) + except (OSError, ValueError): + return {'status': 'archive_unavailable', 'nextCursor': None} + name = next((name for name in names if after_name is None or name > after_name), None) + if name is None: + return {'status': 'complete', 'nextCursor': None} + try: + inspection = ArchiveSource(archive_directory, name).inspect() + inspection['artifact'] = asdict(inspection['artifact']) + except (OSError, ValueError, EOFError, zlib.error): + # Advance past bad sources so they cannot prevent a full sweep. Do not + # invalidate catalog rows by filename: another scan may have replaced it. + return {'status': 'source_unavailable', 'nextCursor': name} + return {'status': 'inspected', 'nextCursor': name, 'inspection': inspection} + + +def run(input_stream, output_stream, archive_directory): + payload = input_stream.read(MAX_INPUT_BYTES + 1) + if len(payload) > MAX_INPUT_BYTES: + raise ValueError('RAW inspection input exceeds byte budget') + work = json.loads(payload, object_pairs_hook=_unique_object, parse_constant=_invalid_constant) + if (not isinstance(work, dict) or set(work) != {'version', 'after_name'} + or work['version'] != 'raw-inspection-work-v1'): + raise ValueError('Invalid RAW inspection envelope') + after = work['after_name'] + if after is not None and (not isinstance(after, str) or len(after) > 255 or not SOURCE_NAME.fullmatch(after)): + raise ValueError('Invalid RAW inspection cursor') + result = inspect_next(archive_directory, after) + encoded = (json.dumps(result, allow_nan=False, separators=(',', ':')) + '\n').encode() + if len(encoded) > MAX_OUTPUT_BYTES: + raise ValueError('RAW inspection output exceeds byte budget') + output_stream.write(encoded) + output_stream.flush() + + +def main(): + parser = argparse.ArgumentParser(description='Inspect one server-owned RAW archive') + parser.add_argument('--archive-directory', required=True) + args = parser.parse_args() + try: + configure_process_limits() + run(sys.stdin.buffer, sys.stdout.buffer, args.archive_directory) + return 0 + except Exception: + sys.stderr.write('RAW inspection worker failed\n') + return 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/modules/common/biometrics/manifest.v1.json b/modules/common/biometrics/manifest.v1.json new file mode 100644 index 000000000..fdef253cd --- /dev/null +++ b/modules/common/biometrics/manifest.v1.json @@ -0,0 +1,646 @@ +{ + "id": "sleepypod-biometric-bundle", + "version": "1.0.0", + "schemaVersion": 1, + "canonicalization": "finite-binary64-json-v1", + "sampleRateHz": 500, + "hardware": [ + "pod3", + "pod4", + "pod5" + ], + "parameters": { + "type": "object", + "properties": { + "heart_rate": { + "type": "object", + "properties": { + "trackingToleranceBpm": { + "type": "number", + "minimum": 5, + "maximum": 30, + "default": 15, + "unit": "bpm", + "description": "Gaussian tracker consistency width; affects heart rate and downstream stages." + }, + "trackingHistoryLength": { + "type": "integer", + "minimum": 1, + "maximum": 10, + "default": 5, + "unit": "windows", + "description": "Number of accepted heart-rate estimates retained by the tracker." + } + }, + "required": [ + "trackingToleranceBpm", + "trackingHistoryLength" + ], + "additionalProperties": false + }, + "breathing_rate": { + "type": "object", + "properties": { + "intervalLowerRatio": { + "type": "number", + "minimum": 0.25, + "maximum": 0.9, + "default": 0.5, + "unit": "ratio", + "description": "Reject breathing intervals below this fraction of the median." + }, + "intervalUpperRatio": { + "type": "number", + "minimum": 1.1, + "maximum": 2, + "default": 1.5, + "unit": "ratio", + "description": "Reject breathing intervals above this fraction of the median." + } + }, + "required": [ + "intervalLowerRatio", + "intervalUpperRatio" + ], + "additionalProperties": false + }, + "hrv_index": { + "type": "object", + "properties": { + "harmonicTolerance": { + "type": "number", + "minimum": 0.05, + "maximum": 0.3, + "default": 0.18, + "unit": "ratio", + "description": "Tolerance for window IBI harmonic correction; does not produce beat-to-beat RMSSD." + }, + "minimumWindows": { + "type": "integer", + "minimum": 10, + "maximum": 30, + "default": 10, + "unit": "windows", + "description": "Minimum accepted 10-second IBI windows before reporting the index." + } + }, + "required": [ + "harmonicTolerance", + "minimumWindows" + ], + "additionalProperties": false + }, + "sleep_stages": { + "type": "object", + "properties": { + "engine": { + "type": "string", + "const": "rules" + }, + "ruleset": { + "type": "object", + "properties": { + "outliers": { + "type": "object", + "properties": { + "heartRate": { + "type": "object", + "properties": { + "min": { + "type": "number" + }, + "max": { + "type": "number" + } + }, + "required": [ + "min", + "max" + ], + "additionalProperties": false + }, + "hrv": { + "type": "object", + "properties": { + "min": { + "type": "number" + }, + "max": { + "type": "number" + } + }, + "required": [ + "min", + "max" + ], + "additionalProperties": false + }, + "breathingRate": { + "type": "object", + "properties": { + "min": { + "type": "number" + }, + "max": { + "type": "number" + } + }, + "required": [ + "min", + "max" + ], + "additionalProperties": false + }, + "heartRateSpikeStdDevs": { + "type": "number", + "minimum": 0, + "maximum": 10 + } + }, + "required": [ + "heartRate", + "hrv", + "breathingRate", + "heartRateSpikeStdDevs" + ], + "additionalProperties": false + }, + "lowSignalBelow": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "lowSignalRules": { + "maxItems": 32, + "type": "array", + "items": { + "type": "object", + "properties": { + "stage": { + "type": "string", + "enum": [ + "wake", + "light", + "deep", + "rem" + ] + }, + "when": { + "maxItems": 8, + "type": "array", + "items": { + "type": "object", + "properties": { + "metric": { + "type": "string", + "enum": [ + "hrRatio", + "heartRate", + "hrv", + "breathingRate", + "movement" + ] + }, + "op": { + "type": "string", + "enum": [ + "<", + "<=", + ">", + ">=" + ] + }, + "value": { + "type": "number" + } + }, + "required": [ + "metric", + "op", + "value" + ], + "additionalProperties": false + } + } + }, + "required": [ + "stage", + "when" + ], + "additionalProperties": false + } + }, + "rules": { + "maxItems": 32, + "type": "array", + "items": { + "type": "object", + "properties": { + "stage": { + "type": "string", + "enum": [ + "wake", + "light", + "deep", + "rem" + ] + }, + "when": { + "maxItems": 8, + "type": "array", + "items": { + "type": "object", + "properties": { + "metric": { + "type": "string", + "enum": [ + "hrRatio", + "heartRate", + "hrv", + "breathingRate", + "movement" + ] + }, + "op": { + "type": "string", + "enum": [ + "<", + "<=", + ">", + ">=" + ] + }, + "value": { + "type": "number" + } + }, + "required": [ + "metric", + "op", + "value" + ], + "additionalProperties": false + } + } + }, + "required": [ + "stage", + "when" + ], + "additionalProperties": false + } + }, + "fallback": { + "type": "string", + "enum": [ + "wake", + "light", + "deep", + "rem" + ] + }, + "smoothIsolatedEpochs": { + "type": "boolean" + }, + "transitions": { + "maxItems": 16, + "type": "array", + "items": { + "type": "object", + "properties": { + "from": { + "type": "string", + "enum": [ + "wake", + "light", + "deep", + "rem" + ] + }, + "to": { + "type": "string", + "enum": [ + "wake", + "light", + "deep", + "rem" + ] + }, + "via": { + "type": "string", + "enum": [ + "wake", + "light", + "deep", + "rem" + ] + } + }, + "required": [ + "from", + "to", + "via" + ], + "additionalProperties": false + } + } + }, + "required": [ + "outliers", + "lowSignalBelow", + "lowSignalRules", + "rules", + "fallback", + "smoothIsolatedEpochs", + "transitions" + ], + "additionalProperties": false + } + }, + "required": [ + "engine", + "ruleset" + ], + "additionalProperties": false + } + }, + "required": [ + "heart_rate", + "breathing_rate", + "hrv_index", + "sleep_stages" + ], + "additionalProperties": false + }, + "defaults": { + "heart_rate": { + "trackingToleranceBpm": 15, + "trackingHistoryLength": 5 + }, + "breathing_rate": { + "intervalLowerRatio": 0.5, + "intervalUpperRatio": 1.5 + }, + "hrv_index": { + "harmonicTolerance": 0.18, + "minimumWindows": 10 + }, + "sleep_stages": { + "engine": "rules", + "ruleset": { + "outliers": { + "heartRate": { + "min": 45, + "max": 130 + }, + "hrv": { + "min": 1, + "max": 300 + }, + "breathingRate": { + "min": 8, + "max": 25 + }, + "heartRateSpikeStdDevs": 2 + }, + "lowSignalBelow": 0.3, + "lowSignalRules": [ + { + "stage": "wake", + "when": [ + { + "metric": "movement", + "op": ">", + "value": 200 + } + ] + } + ], + "rules": [ + { + "stage": "wake", + "when": [ + { + "metric": "movement", + "op": ">", + "value": 200 + } + ] + }, + { + "stage": "deep", + "when": [ + { + "metric": "hrRatio", + "op": "<", + "value": 0.92 + } + ] + }, + { + "stage": "rem", + "when": [ + { + "metric": "hrRatio", + "op": ">=", + "value": 0.95 + }, + { + "metric": "hrv", + "op": "<", + "value": 25 + }, + { + "metric": "movement", + "op": "<", + "value": 30 + } + ] + }, + { + "stage": "rem", + "when": [ + { + "metric": "hrRatio", + "op": ">=", + "value": 0.95 + }, + { + "metric": "movement", + "op": "<", + "value": 50 + }, + { + "metric": "hrv", + "op": "<", + "value": 40 + } + ] + }, + { + "stage": "wake", + "when": [ + { + "metric": "hrRatio", + "op": ">=", + "value": 0.95 + }, + { + "metric": "movement", + "op": ">", + "value": 100 + } + ] + } + ], + "fallback": "light", + "smoothIsolatedEpochs": true, + "transitions": [ + { + "from": "wake", + "to": "deep", + "via": "light" + }, + { + "from": "deep", + "to": "wake", + "via": "light" + }, + { + "from": "deep", + "to": "rem", + "via": "light" + }, + { + "from": "rem", + "to": "deep", + "via": "light" + } + ] + } + } + }, + "orderedRanges": [ + [ + "sleep_stages", + "ruleset", + "outliers", + "heartRate" + ], + [ + "sleep_stages", + "ruleset", + "outliers", + "hrv" + ], + [ + "sleep_stages", + "ruleset", + "outliers", + "breathingRate" + ] + ], + "components": { + "piezo_preprocessing": { + "algorithmId": "piezo-presence-pump-v2", + "version": "1.0.0", + "runtime": "python3.10", + "requiredInputs": [ + "dual-channel-piezo", + "pump-state", + "occupancy", + "sensor-calibration", + "side-mode" + ], + "dependencies": [], + "metricId": null, + "unit": null, + "description": "Shared presence, occupancy veto, pump gating and channel arbitration; not user tunable.", + "causal": true + }, + "heart_rate": { + "algorithmId": "shs-tracker-v2", + "version": "1.0.0", + "runtime": "python3.10", + "requiredInputs": [ + "piezo-30s" + ], + "dependencies": [ + "piezo_preprocessing" + ], + "metricId": "heart_rate", + "unit": "bpm", + "description": "SHS heart-rate estimate with inter-window tracking.", + "causal": true + }, + "breathing_rate": { + "algorithmId": "hilbert-envelope-v2", + "version": "1.0.0", + "runtime": "python3.10", + "requiredInputs": [ + "piezo-60s" + ], + "dependencies": [ + "piezo_preprocessing" + ], + "metricId": "breathing_rate", + "unit": "breaths/min", + "description": "Breathing rate from the cardiac-band amplitude envelope.", + "causal": true + }, + "hrv_index": { + "algorithmId": "window-ibi-variability-v1", + "version": "1.0.0", + "runtime": "python3.10", + "requiredInputs": [ + "piezo-300s" + ], + "dependencies": [ + "piezo_preprocessing" + ], + "metricId": "window_ibi_variability_v1", + "unit": "ms", + "description": "Successive differences of window-level IBI estimates. Not clinical beat-to-beat RMSSD or SDNN.", + "causal": true + }, + "sleep_stages": { + "algorithmId": "retrospective-stage-rules-v1", + "version": "1.0.0", + "runtime": "node", + "requiredInputs": [ + "vitals", + "movement", + "piezo-calibration-quality" + ], + "dependencies": [ + "heart_rate", + "breathing_rate", + "hrv_index" + ], + "metricId": "sleep_stages", + "unit": "stage", + "description": "Retrospective wake/light/deep/REM classification using nightly mean HR and neighboring epochs.", + "causal": false + } + }, + "sourceHashes": { + "modules/common/biometrics/signal.py": "fff73469815673c8daebba9e2c9e7bcf97708939ad7a5639a52f0a53c9d75c18", + "modules/piezo-processor/main.py": "75db2ea2f005fd537d994b306332c1abf97c0c28abb496a8d3af03816ccf8550", + "src/lib/sleep-stages.ts": "b1800e49ce8525d314333da6a29a63131a002d160bb294fa97b24df6e655e228", + "modules/piezo-processor/runtime_adapter.py": "0a22a2fa05aae6569bfc4b53fba80d75e89f702f021ccb61f82ce64e284c5bb8", + "modules/common/biometrics/runtime.py": "dbe327ce6e6ae315cad83439903e4c5d6a5c594468086739504caa8ed028a944", + "modules/common/biometrics/gates.py": "c46285ff1cdf95bd4184823fabd6ceb66b005f7c410ee87880de4e563edd6e7c", + "modules/common/biometrics/routing.py": "04872bab0f797e0bedd6e7114e686bbf7a5b4110ed2052c9b160e5aa1f7d0839", + "modules/common/biometrics/processing.py": "2d04538499498f3b4fa9fa7383d1d3e6c0f66c1db3455bfd67ede53be7d080ef", + "modules/common/biometrics/frames.py": "99427d69cf63613c96e45f2f280b53a6e7db18997a28e579cd5e412684db259f", + "modules/common/biometrics/replay.py": "50c70408dca033dcf17d5bf6a1aa7afc9ed531d09aa3151eb61941f7c504639f" + }, + "dependencyHashes": { + "modules/piezo-processor/uv.lock": "2803397e553a32588071d6be84c2bce5326552da03076261f48afce0ba919869", + "pnpm-lock.yaml": "bf4c8635e3f34acd1637bdc33fab0c1a9ee1287dcad827d60c8b9b4027118a41" + }, + "legacySourceHashes": { + "modules/piezo-processor/main.py": "c98bdb3d6ee38b5de5d36760238978a8fefe2153a0cdf3e82ea1872d2d0da34f", + "src/lib/sleep-stages.ts": "853f6b1f14618fbac11da43d3246b95aac55af8b782b2cbb1be3f7627bf560b4" + } +} diff --git a/modules/common/biometrics/processing.py b/modules/common/biometrics/processing.py new file mode 100644 index 000000000..c4ecf8895 --- /dev/null +++ b/modules/common/biometrics/processing.py @@ -0,0 +1,225 @@ +"""Stateful per-channel processing shared by live ingestion and replay. + +All time, occupancy, routing, provenance and output are caller-owned. The +clock returns Unix seconds, and estimates and candidate timestamps use the +same event time. Retain +left-before-right ingestion: cross-channel rejection intentionally sees the +other side's most recently evaluated features, not a simultaneous recompute. +There are no database, hardware, transport or wall-clock reads here. +""" +from collections import deque +from datetime import datetime, timezone +import logging +from typing import Callable, Optional + +import numpy as np + +from .signal import (SAMPLE_RATE, _bandpass, _autocorr_quality, + ProcessingParameters, DEFAULT_PARAMETERS, WindowEstimator) +from .gates import (FrzHealthPumpState, PresenceDetector, + PUMP_COUPLING_STD_FACTOR, PUMP_COUPLING_ACR_THRESHOLD) +from .routing import VITALS_INTERVAL_S + +HR_WINDOW_S = 30 +BREATHING_WINDOW_S = 60 +HRV_WINDOW_S = 300 +log = logging.getLogger(__name__) + + +class StatefulSideProcessor: + def __init__(self, side: str, *, clock: Callable[[], float], + writer: Callable[[object], bool], candidate_factory: Callable, + pump_state: Optional[FrzHealthPumpState] = None, + parameters: ProcessingParameters = DEFAULT_PARAMETERS, + autocorr_quality: Callable = _autocorr_quality): + self.side = side + self._clock = clock + self._writer = writer + self._candidate_factory = candidate_factory + self._quality = autocorr_quality + self._hr_buf: deque = deque(maxlen=HR_WINDOW_S * SAMPLE_RATE) + self._hrv_buf: deque = deque(maxlen=HRV_WINDOW_S * SAMPLE_RATE) + self._br_buf: deque = deque(maxlen=BREATHING_WINDOW_S * SAMPLE_RATE) + self._last_write = 0.0 + self._presence = PresenceDetector() + self.parameters = parameters + self.run_context: Optional[object] = None + self._estimator = WindowEstimator(parameters) + self._hr_tracker = self._estimator.hr_tracker + self._other: Optional['StatefulSideProcessor'] = None # set after both sides created + self._last_med_std: float = 0.0 # cached for cross-channel comparison + self._last_acr_qual: float = 0.0 + self._pump_state = pump_state + # An optional arbitration sink; None uses the caller-provided writer. + # main() routes both sides through SingleSleeperVitals.submit. + self.sink: Optional[Callable[[object], bool]] = None + # The bed's own answer to "is anyone here" (sleep-detector presence): + # False vetoes piezo presence, None (unknown) leaves it alone. + self.bed_occupied: Optional[Callable[[], Optional[bool]]] = None + # Single-sleeper home side (None in per-side mode). The sleep-detector + # merges both zones into the home side, so only the home side's + # processor may treat `bed_occupied() is True` as its own evidence. + self.home_side: Optional[Callable[[], Optional[str]]] = None + + def install_runtime(self, parameters, run_context, reset_side=False): + if parameters != self.parameters or reset_side: + self._hr_buf.clear() + self._hrv_buf.clear() + self._br_buf.clear() + self._estimator = WindowEstimator(parameters) + self._hr_tracker = self._estimator.hr_tracker + self._last_write = 0.0 + self._last_med_std = self._last_acr_qual = 0.0 + self._presence = PresenceDetector() + self.parameters = parameters + self.run_context = run_context + + def ingest(self, samples: np.ndarray) -> None: + self._hr_buf.extend(samples) + self._hrv_buf.extend(samples) + self._br_buf.extend(samples) + self._maybe_write() + + def _maybe_write(self) -> None: + now = self._clock() + if now - self._last_write < VITALS_INTERVAL_S: + return + + hr_arr = np.array(self._hr_buf) + if len(hr_arr) < int(10 * SAMPLE_RATE): + return # Not enough data yet + + # --- Presence detection (hysteresis + autocorrelation quality) --- + filt = _bandpass(hr_arr, 1.0, 10.0, SAMPLE_RATE) + w = int(5 * SAMPLE_RATE) + stds = [np.std(filt[j:j + w]) + for j in range(0, len(filt) - w + 1, w)] + med_std = float(np.median(stds)) if stds else 0.0 + acr_qual = self._quality(hr_arr) + cross_channel_rejected = False + + # Cross-channel rejection: if the other side has stronger or similar + # signal AND strong autocorrelation (someone is actually there), this + # side's signal is likely vibration coupling, not a person. + # A real person produces ~3-5x stronger signal on their side. + # + # Note: due to sequential processing in main() (left before right), + # left sees right's values from the previous cycle (~60s stale). + # This is acceptable — coupling produces similar energy regardless + # of which side checks first. + if self._other is not None and self._other._last_med_std > 0: + other_std = self._other._last_med_std + other_acr = self._other._last_acr_qual + # Only suppress if: other side has real periodicity (someone there), + # other side's energy is >= ours, and our energy is below threshold + if (other_acr > 0.3 + and other_std > med_std * 0.7 + and med_std < self._presence.enter_threshold): + acr_qual = 0.0 + cross_channel_rejected = True + + # Cache AFTER suppression so the other side sees post-suppression values + self._last_med_std = med_std + self._last_acr_qual = acr_qual + + # The bed's own answer to "is anyone here" (sleep-detector capacitance + # presence), read once per cycle: True relaxes re-entry below, False + # vetoes piezo presence, None (unknown) leaves the piezo alone. + bed = self.bed_occupied() if self.bed_occupied is not None else None + # Only occupancy measured on this side may relax the piezo. With one + # side away the reading is merged under the home side, so the away + # side inherits a True it cannot attribute to its own zone and keeps + # the strict path. A side whose energy cross-channel rejection has + # already attributed to the other sleeper keeps it too. + own_side = self.home_side is None or self.home_side() in (None, self.side) + relaxed = bed is True and own_side and not cross_channel_rejected + + # Pump-coupling guard. PumpGate handles symmetric spikes (ramps) + # but misses two steady-state modes: asymmetric (own-pump-on with + # opposite idle — vibration couples into same-side piezo, no + # competing signal for cross-channel rejection) and symmetric + # (both pumps on at mismatched RPMs — beat frequency lands in the + # cardiac band, both sides look "real"). In both, require elevated + # energy AND strong autocorrelation to enter PRESENT; coupling + # rarely produces both, only a real person does. + # + # The guard answers "is this vibration a person or the pump". When + # the capacitance sensor says a person is in bed on this side and + # the energy is not the other sleeper's, that question is settled; + # on Pod 5 both pumps run all night, so without this bypass a + # sleeper who drifted to a weak-signal position stayed ABSENT for + # an hour at a time (05:57-07:01 on 2026-10-11) until a hard + # movement cleared the raised bar. + if (self._pump_state is not None and not relaxed + and self._presence.state == PresenceDetector.ABSENT): + symmetric = self._pump_state.is_symmetric_active() + asymmetric = self._pump_state.is_asymmetric_for(self.side) + if symmetric or asymmetric: + std_threshold = self._presence.enter_threshold * PUMP_COUPLING_STD_FACTOR + if not (med_std > std_threshold and acr_qual > PUMP_COUPLING_ACR_THRESHOLD): + log.debug( + "%s: pump-coupling guard suppressed presence " + "(med_std=%.0f, acr=%.2f, mode=%s)", + self.side, med_std, acr_qual, + "symmetric" if symmetric else "asymmetric", + ) + return + + if relaxed and self._presence.state == PresenceDetector.ABSENT: + log.debug("%s: bed reads occupied, relaxed piezo re-entry (med_std=%.0f, acr=%.2f)", + self.side, med_std, acr_qual) + if cross_channel_rejected: + # The other side owns this energy. Zeroing acr alone cannot end + # presence (exit needs weak energy too), so count the window as + # weak and let the normal hysteresis exit instead of latching, + # and never submit vitals computed from it meanwhile. + self._presence.update(0.0, 0.0) + present = False + else: + present = self._presence.update(med_std, acr_qual, relaxed=relaxed) + # Piezo presence is vibration energy and rhythm; a prime or the pump + # shaking an empty bed passes it. Trust the capacitance-based + # presence when the sleep-detector says nobody is there. + if present and bed is False: + present = False + + if not present: + # Reset the interval cursor so a return from extended absence + # doesn't trigger burst processing on every ingest until the + # first write completes (#325). + self._last_write = now + return # No user detected — skip + + # Shared numerical estimator. Gating and live sinks remain in this adapter. + estimate = self._estimator.estimate(now, hr_arr, np.array(self._br_buf), + np.array(self._hrv_buf)) + hr, hrv, br = estimate.heart_rate, estimate.hrv_index, estimate.breathing_rate + hr_raw, hr_score = estimate.hr_raw, estimate.hr_score + + if hr is not None or hrv is not None or br is not None: + ts = datetime.fromtimestamp(now, timezone.utc) + snr = max(0.0, min(1.0, acr_qual)) + hr_conf = max(0.0, min(1.0, hr_score)) if hr is not None else 0.0 + quality = round(0.55 * snr + 0.45 * hr_conf, 3) + flags = [] + if hr is None: + flags.append("no_hr") + if hrv is None: + flags.append("no_hrv") + if br is None: + flags.append("no_br") + if med_std < self._presence.enter_threshold: + flags.append("low_signal") + cand = self._candidate_factory(self.side, ts, hr, hrv, br, quality, flags or None, hr_raw, self.run_context) + wrote = self.sink(cand) if self.sink is not None else self._writer(cand) + log.info("vitals %s — HR=%.1f HRV=%.1f BR=%.1f q=%.2f", self.side, + hr or 0, hrv or 0, br or 0, quality) + # Only advance the downsample cursor when the write actually + # committed — otherwise a transient sqlite error would skip + # VITALS_INTERVAL_S of valid samples. + if wrote: + self._last_write = now + else: + # No metric to write — advance so we don't recompute on every ingest. + self._last_write = now + diff --git a/modules/common/biometrics/publication_worker.py b/modules/common/biometrics/publication_worker.py new file mode 100644 index 000000000..5dd21c6fa --- /dev/null +++ b/modules/common/biometrics/publication_worker.py @@ -0,0 +1,54 @@ +"""Fixed server-owned entry point for atomic waveform dataset publication.""" +import argparse +import json +from pathlib import Path +import sqlite3 +import sys + +from .replay_worker import _unique_object, _invalid_constant, configure_process_limits + +MAX_INPUT_BYTES = 10 * 1024 * 1024 + + +def run(input_stream, output_stream, *, database, archive_directory, retained_directory): + payload = input_stream.read(MAX_INPUT_BYTES + 1) + if len(payload) > MAX_INPUT_BYTES: + raise ValueError('Waveform publication input exceeds byte budget') + work = json.loads(payload, object_pairs_hook=_unique_object, parse_constant=_invalid_constant) + if not isinstance(work, dict) or set(work) != {'version', 'prepared', 'selections'} or work['version'] != 'waveform-publication-work-v1': + raise ValueError('Invalid waveform publication envelope') + from .raw_artifacts import RawArtifactStore + from .raw_publication import publish_waveform_dataset + # Never create an empty database at a mistyped deployment location. + connection = sqlite3.connect(Path(database).resolve().as_uri() + '?mode=rw', uri=True, timeout=5) + try: + connection.execute('PRAGMA foreign_keys = ON') + result = publish_waveform_dataset(RawArtifactStore(retained_directory), connection, + archive_directory, work['selections'], work['prepared']) + # The coordinator re-reads the descriptor by ID. Keep IPC independent of + # the manifest size, and permit idempotent recovery after a lost reply. + output_stream.write((json.dumps({'type': 'result', 'id': result['id'], + 'contentHash': result['contentHash'], 'available': result['available']}, separators=(',', ':')) + '\n').encode()) + output_stream.flush() + finally: + connection.close() + + +def main(): + parser = argparse.ArgumentParser(description='Publish fixed retained waveform inputs') + parser.add_argument('--database', required=True) + parser.add_argument('--archive-directory', required=True) + parser.add_argument('--retained-directory', required=True) + args = parser.parse_args() + try: + configure_process_limits() + run(sys.stdin.buffer, sys.stdout.buffer, database=args.database, + archive_directory=args.archive_directory, retained_directory=args.retained_directory) + return 0 + except Exception: + sys.stderr.write('Waveform publication worker failed\n') + return 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/modules/common/biometrics/raw_artifacts.py b/modules/common/biometrics/raw_artifacts.py new file mode 100644 index 000000000..ce8422626 --- /dev/null +++ b/modules/common/biometrics/raw_artifacts.py @@ -0,0 +1,388 @@ +"""Private retained RAW artifacts; all directories are server-owned configuration. + +Copy gzip bytes rather than keeping a checksum or a link into the rolling +archive. Publication and quota checks share a process lock. Readers verify the +retained checksum before exposing a bounded decompression stream. +""" +from contextlib import contextmanager +from dataclasses import dataclass +import fcntl +import gzip +import hashlib +import os +from pathlib import Path +import re +import shutil +import stat +import uuid + +CHUNK = 64 * 1024 +SOURCE_NAME = re.compile(r'[A-Za-z0-9_.-]+\.RAW\.gz\Z') +ARTIFACT_NAME = re.compile(r'[a-f0-9]{64}\.RAW\.gz\Z') +PART_NAME = re.compile(r'\.part-[a-f0-9]{32}\Z') + + +@dataclass(frozen=True) +class RawLimits: + compressed_bytes: int = 256 * 1024 * 1024 + expanded_bytes: int = 512 * 1024 * 1024 + total_bytes: int = 1024 * 1024 * 1024 + minimum_free_bytes: int = 512 * 1024 * 1024 + max_artifacts: int = 1024 + + def __post_init__(self): + if any(type(value) is not int or value <= 0 for value in vars(self).values()): + raise ValueError('RAW limits must be positive integers') + + +@dataclass(frozen=True) +class RawArtifact: + name: str + sha256: str + compressed_bytes: int + expanded_bytes: int + + def __post_init__(self): + if not ARTIFACT_NAME.fullmatch(self.name) or self.name != self.sha256 + '.RAW.gz': + raise ValueError('Invalid retained RAW identity') + if any(type(value) is not int or value < 0 for value in (self.compressed_bytes, self.expanded_bytes)): + raise ValueError('Invalid retained RAW size') + + +def _identity(info): + return (info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, info.st_ctime_ns) + + +def _regular(fd): + info = os.fstat(fd) + if not stat.S_ISREG(info.st_mode): + raise ValueError('RAW input must be a regular file') + return info + + +class BoundedRawReader: + def __init__(self, stream, limit): + self.stream, self.limit, self.position = stream, limit, 0 + + def read(self, size=-1): + if type(size) is not int or size < 0 or size > 1_000_000: + raise ValueError('RAW reads must be bounded to at most one record') + data = self.stream.read(min(size, self.limit - self.position + 1)) + self.position += len(data) + if self.position > self.limit: + raise ValueError('RAW expanded byte limit exceeded') + if size > 0 and not data and self.position != self.limit: + raise ValueError('RAW expanded size mismatch') + return data + + def tell(self): + return self.position + + +class RawArtifactStore: + def __init__(self, directory, limits=RawLimits(), *, read_only=False): + self.read_only = read_only + self.directory = Path(directory) + self.limits = limits + if not read_only: + self.directory.mkdir(mode=0o700, parents=True, exist_ok=True) + if self.directory.is_symlink() or not self.directory.is_dir(): + raise ValueError('Retained RAW directory must not be a symlink') + if not read_only: + os.chmod(self.directory, 0o700) + + @contextmanager + def _lock(self): + flags = os.O_RDONLY if self.read_only else os.O_CREAT | os.O_RDWR + fd = os.open(self.directory / '.lock', flags | os.O_NOFOLLOW | os.O_NONBLOCK, 0o600) + try: + _regular(fd) + fcntl.flock(fd, fcntl.LOCK_EX) + yield + finally: + os.close(fd) + + def _usage(self): + used, count = 0, 0 + for entry in self.directory.iterdir(): + if ARTIFACT_NAME.fullmatch(entry.name) or PART_NAME.fullmatch(entry.name): + info = entry.lstat() + if not stat.S_ISREG(info.st_mode): + raise ValueError('Invalid retained RAW directory entry') + used += info.st_size + count += int(bool(ARTIFACT_NAME.fullmatch(entry.name))) + return used, count + + def _sync_directory(self): + fd = os.open(self.directory, os.O_RDONLY) + try: + os.fsync(fd) + finally: + os.close(fd) + + def retain(self, archive_directory, name, expected=None, *, publish=None): + """Retain one source, optionally committing references under the lock.""" + if publish is not None and (not callable(publish) or expected is None): + raise ValueError('Publication requires a callable and inspected descriptor') + if self.read_only: + raise ValueError('Retained RAW store is read-only') + with self._lock(): + descriptor = self._retain_locked(archive_directory, name, expected) + if publish is not None: + publish(descriptor) + return descriptor + + def retain_batch(self, archive_directory, selections, *, publish): + """Publish a whole selected batch while holding the file lifecycle lock. + + The trusted callback begins and commits SQLite only after every copy is + verified. Never enter with an existing SQLite transaction or reacquire + the file lock inside the callback. A failed callback leaves complete + unreferenced files, but cannot publish a partial dataset. + """ + if self.read_only: + raise ValueError('Retained RAW store is read-only') + if not isinstance(selections, (list, tuple)) or not 1 <= len(selections) <= self.limits.max_artifacts or not callable(publish): + raise ValueError('Invalid retained RAW batch') + frozen, hashes = [], set() + for selection in selections: + if not isinstance(selection, (list, tuple)) or len(selection) != 2: + raise ValueError('Expected source name and inspected descriptor') + name, expected = selection + if not isinstance(name, str) or not SOURCE_NAME.fullmatch(name) or not isinstance(expected, RawArtifact): + raise ValueError('Invalid inspected RAW selection') + if expected.sha256 in hashes: + raise ValueError('Select each retained artifact once') + hashes.add(expected.sha256) + frozen.append((name, expected)) + with self._lock(): + descriptors = tuple(self._retain_locked(archive_directory, name, expected) for name, expected in frozen) + return publish(descriptors) + + def _retain_locked(self, archive_directory, name, expected): + if self.read_only: + raise ValueError('Retained RAW store is read-only') + if not isinstance(name, str) or not SOURCE_NAME.fullmatch(name): + raise ValueError('Invalid RAW archive filename') + if expected is not None and not isinstance(expected, RawArtifact): + raise ValueError('Expected an inspected RAW descriptor') + # Exact retained bytes are independent of the rolling source. Reuse + # them only after checking both compressed identity and expanded size. + if expected is not None: + cached = self.directory / expected.name + if cached.exists() or cached.is_symlink(): + fd = self._verify(expected) + with os.fdopen(fd, 'rb') as source, gzip.GzipFile(fileobj=source) as stream: + reader = BoundedRawReader(stream, expected.expanded_bytes) + while reader.read(CHUNK): + pass + self._sync_directory() + return expected + archive = Path(archive_directory) + if archive.is_symlink() or not archive.is_dir(): + raise ValueError('Invalid RAW archive directory') + # A dead copier can leave a private temporary file, never a published + # descriptor. The shared lock proves no other copier is using it. + for entry in self.directory.iterdir(): + if PART_NAME.fullmatch(entry.name): + entry.unlink() + source_fd = os.open(archive / name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + temporary = self.directory / ('.part-' + uuid.uuid4().hex) + try: + before = _regular(source_fd) + if before.st_size > self.limits.compressed_bytes: + raise ValueError('RAW compressed byte limit exceeded') + if shutil.disk_usage(self.directory).free - before.st_size < self.limits.minimum_free_bytes: + raise ValueError('Insufficient free space for retained RAW input') + digest, copied = hashlib.sha256(), 0 + target_fd = os.open(temporary, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600) + with os.fdopen(target_fd, 'wb') as target: + while True: + data = os.read(source_fd, CHUNK) + if not data: + break + copied += len(data) + if copied > self.limits.compressed_bytes: + raise ValueError('RAW compressed byte limit exceeded') + digest.update(data) + target.write(data) + target.flush() + os.fsync(target.fileno()) + if _identity(before) != _identity(_regular(source_fd)): + raise ValueError('RAW archive changed during retention') + expanded = 0 + with gzip.open(temporary, 'rb') as stream: + while True: + data = stream.read(min(CHUNK, self.limits.expanded_bytes - expanded + 1)) + if not data: + break + expanded += len(data) + if expanded > self.limits.expanded_bytes: + raise ValueError('RAW expanded byte limit exceeded') + sha256 = digest.hexdigest() + descriptor = RawArtifact(sha256 + '.RAW.gz', sha256, copied, expanded) + if expected is not None and descriptor != expected: + raise ValueError('RAW archive changed since inspection') + destination = self.directory / descriptor.name + if destination.exists() or destination.is_symlink(): + os.close(self._verify(descriptor)) + self._sync_directory() + return descriptor + # Include the unpublished copy when checking the publication quota. + used, count = self._usage() + if count >= self.limits.max_artifacts: + raise ValueError('Retained RAW artifact count exceeded') + if used > self.limits.total_bytes: + raise ValueError('Retained RAW storage quota exceeded') + os.replace(temporary, destination) + self._sync_directory() + return descriptor + finally: + os.close(source_fd) + temporary.unlink(missing_ok=True) + + def _verify(self, descriptor): + if not isinstance(descriptor, RawArtifact): + raise ValueError('Expected retained RAW descriptor') + if descriptor.compressed_bytes > self.limits.compressed_bytes or descriptor.expanded_bytes > self.limits.expanded_bytes: + raise ValueError('Retained RAW exceeds configured limits') + fd = os.open(self.directory / descriptor.name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + try: + info = _regular(fd) + if info.st_size != descriptor.compressed_bytes: + raise ValueError('Retained RAW size mismatch') + digest, count = hashlib.sha256(), 0 + while True: + data = os.read(fd, CHUNK) + if not data: + break + count += len(data) + if count > descriptor.compressed_bytes: + raise ValueError('Retained RAW changed during verification') + digest.update(data) + if _identity(info) != _identity(_regular(fd)): + raise ValueError('Retained RAW changed during verification') + if digest.hexdigest() != descriptor.sha256: + raise ValueError('Retained RAW checksum mismatch') + os.lseek(fd, 0, os.SEEK_SET) + return fd + except BaseException: + os.close(fd) + raise + + @contextmanager + def open_raw(self, descriptor): + with self._lock(): + fd = self._verify(descriptor) + with os.fdopen(fd, 'rb') as source, gzip.GzipFile(fileobj=source) as stream: + yield BoundedRawReader(stream, descriptor.expanded_bytes) + + def collect_released(self, connection, *, now_ms, minimum_age_ms=3_600_000, + maximum_entries=10_000): + """Collect only explicitly released metadata with no live references. + + Unknown files and metadata are preserved, including crash orphans. The + file mutex also covers readers and reference publication. BEGIN + IMMEDIATE prevents reference changes between the last check and unlink. + Tombstones remain in SQLite; scanning present files avoids repeatedly + selecting absent tombstones and starving later eligible files. + """ + if self.read_only: + raise ValueError('Retained RAW store is read-only') + if (type(now_ms) is not int or now_ms < 0 or type(minimum_age_ms) is not int + or minimum_age_ms < 0 or type(maximum_entries) is not int + or maximum_entries <= 0): + raise ValueError('Invalid RAW collection bounds') + if connection.in_transaction: + raise ValueError('RAW collection requires an independent transaction') + removed = [] + with self._lock(): + # Bound traversal before deleting anything; never partially process + # a directory that exceeds the agreed inventory budget. + names = [] + with os.scandir(self.directory) as entries: + for count, entry in enumerate(entries, 1): + if count > maximum_entries: + raise ValueError('RAW collection entry limit exceeded') + if ARTIFACT_NAME.fullmatch(entry.name): + names.append(entry.name) + connection.execute('BEGIN IMMEDIATE') + try: + for name in sorted(names): + eligible = connection.execute( + """SELECT content_hash FROM biometric_raw_artifacts AS artifact + WHERE name = ? AND released_at_ms IS NOT NULL + AND released_at_ms <= ? + AND NOT EXISTS ( + SELECT 1 FROM biometric_dataset_raw_artifacts AS reference + WHERE reference.artifact_hash = artifact.content_hash + AND reference.released_at_ms IS NULL)""", + (name, now_ms - minimum_age_ms)).fetchone() + if eligible is None: + continue + if name != eligible[0] + '.RAW.gz': + raise ValueError('Released RAW metadata identity mismatch') + path = self.directory / name + if not stat.S_ISREG(path.lstat().st_mode): + raise ValueError('Released RAW artifact must be a regular file') + path.unlink() + removed.append(name) + if removed: + self._sync_directory() + connection.commit() + except BaseException: + connection.rollback() + raise + return removed + + def collect_orphans(self, connection, *, now_ms, minimum_age_ms=3_600_000, + maximum_entries=10_000): + """Reconcile old unpublished copies without guessing metadata ownership. + + Publication holds this same mutex through its SQLite commit. A file with + any metadata row is preserved, including released tombstones; its normal + release collector owns that lifecycle. Unknown names are never touched. + """ + if self.read_only: + raise ValueError('Retained RAW store is read-only') + if (type(now_ms) is not int or now_ms < 0 or type(minimum_age_ms) is not int + or minimum_age_ms < 0 or type(maximum_entries) is not int + or maximum_entries <= 0): + raise ValueError('Invalid RAW collection bounds') + if connection.in_transaction: + raise ValueError('RAW collection requires an independent transaction') + with self._lock(): + names = [] + with os.scandir(self.directory) as entries: + for count, entry in enumerate(entries, 1): + if count > maximum_entries: + raise ValueError('RAW collection entry limit exceeded') + if ARTIFACT_NAME.fullmatch(entry.name) or PART_NAME.fullmatch(entry.name): + names.append(entry.name) + connection.execute('BEGIN IMMEDIATE') + try: + candidates = [] + for name in sorted(names): + # Check both identities so inconsistent metadata cannot make + # a referenced file look like an orphan. + if ARTIFACT_NAME.fullmatch(name) and connection.execute( + 'SELECT 1 FROM biometric_raw_artifacts WHERE name = ? OR content_hash = ?', + (name, name[:-7])).fetchone() is not None: + continue + path = self.directory / name + info = path.lstat() + if not stat.S_ISREG(info.st_mode): + raise ValueError('Orphan RAW artifact must be a regular file') + if info.st_mtime_ns > (now_ms - minimum_age_ms) * 1_000_000: + continue + candidates.append(path) + # Complete the bounded inventory and metadata checks first. + for path in candidates: + path.unlink() + if candidates: + self._sync_directory() + connection.commit() + return [path.name for path in candidates] + except BaseException: + connection.rollback() + raise diff --git a/modules/common/biometrics/raw_publication.py b/modules/common/biometrics/raw_publication.py new file mode 100644 index 000000000..c8d4d101e --- /dev/null +++ b/modules/common/biometrics/raw_publication.py @@ -0,0 +1,160 @@ +"""Atomic publication of server-prepared waveform datasets and RAW references. + +The coordinator validates and hashes the complete snapshot with its shared +TypeScript data-hash contract. This internal publisher never accepts a dataset +row directly from an RPC request. It verifies selected archive identities, +then inserts that exact prepared snapshot and all references in one transaction. +""" +import json +import re +import uuid + +from .raw_artifacts import RawArtifact, SOURCE_NAME + +HASH = re.compile(r'[a-f0-9]{64}\Z') +MAX_DATASET_BYTES = 8 * 1024 * 1024 +MAX_TOTAL_BYTES = 64 * 1024 * 1024 + + +def _json(value): + return json.dumps(value, ensure_ascii=False, allow_nan=False, separators=(',', ':')) + + +def _describe(row): + keys = ('id', 'side', 'subjectId', 'requestHash', 'contentHash', 'manifest', + 'byteLength', 'createdAtMs', 'releasedAtMs') + result = dict(zip(keys, row)) + result['manifest'] = json.loads(result['manifest']) + result['available'] = result['releasedAtMs'] is None + return result + + +def _existing(connection, key, request_hash): + row = connection.execute('''SELECT id, side, subject_id, request_hash, content_hash, + manifest, byte_length, created_at_ms, released_at_ms FROM biometric_datasets + WHERE idempotency_key = ?''', (key,)).fetchone() + if row is None: + return None + result = _describe(row) + if result['requestHash'] != request_hash: + raise ValueError('Dataset idempotency key changed') + return result + + +def publish_waveform_dataset(store, connection, archive_directory, selections, prepared): + """Copy under the file mutex; keep SQLite's write lock out of file I/O.""" + from .retained_replay import RetainedReplayInput, _integer, _keys + if connection.in_transaction: + raise ValueError('Waveform publication requires an independent transaction') + if not connection.execute('PRAGMA foreign_keys').fetchone()[0]: + raise ValueError('Waveform publication requires foreign keys') + # Freeze caller-owned dictionaries before copying potentially large files. + serialized = _json(prepared) + if len(serialized.encode('utf-8')) > MAX_DATASET_BYTES: + raise ValueError('Waveform dataset exceeds its byte budget') + dataset = json.loads(serialized) + _keys(dataset, 'id side subjectId idempotencyKey requestHash contentHash manifest snapshot createdAtMs') + if str(uuid.UUID(dataset['id'])) != dataset['id']: + raise ValueError('Invalid waveform dataset identity') + if dataset['side'] not in ('left', 'right'): + raise ValueError('Invalid waveform dataset side') + for key in ('subjectId', 'idempotencyKey'): + if not isinstance(dataset[key], str) or not 1 <= len(dataset[key]) <= 200: + raise ValueError('Invalid waveform dataset key') + for key in ('requestHash', 'contentHash'): + if not isinstance(dataset[key], str) or not HASH.fullmatch(dataset[key]): + raise ValueError('Invalid waveform dataset hash') + _integer(dataset['createdAtMs']) + snapshot, manifest = dataset['snapshot'], dataset['manifest'] + if (not isinstance(snapshot, dict) or snapshot.get('version') != 'waveform-dataset-v1' + or snapshot.get('side') != dataset['side'] or snapshot.get('subjectId') != dataset['subjectId'] + or not isinstance(manifest, dict) or manifest.get('version') != 'waveform-dataset-manifest-v1'): + raise ValueError('Invalid prepared waveform snapshot') + cohorts = manifest.get('splitProtocol', {}).get('assignments', []) + if not isinstance(cohorts, list) or not 1 <= len(cohorts) <= 60: + raise ValueError('Invalid waveform cohort assignments') + cohort_map = {item['cohortId']: item for item in cohorts} + if len(cohort_map) != len(cohorts): + raise ValueError('Duplicate waveform cohort assignment') + inputs = snapshot.get('waveformInputs') + if not isinstance(inputs, list) or not 1 <= len(inputs) <= 60: + raise ValueError('Invalid waveform dataset inputs') + artifacts, evidence, selected_cohorts = {}, {}, set() + for item in inputs: + _keys(item, 'cohortId input') + cohort_id = item['cohortId'] + if cohort_id not in cohort_map or cohort_id in selected_cohorts: + raise ValueError('Waveform inputs must match whole-night cohorts') + selected_cohorts.add(cohort_id) + retained = RetainedReplayInput.parse(item['input']) + cohort = cohort_map[cohort_id] + if retained.score_start_ms != cohort['startMs'] or retained.score_end_ms != cohort['endMs']: + raise ValueError('Waveform scoring window must match its whole-night cohort') + for source in retained.artifacts: + artifact = source.artifact + identity = (artifact, source.first_event_ms, source.last_event_ms) + if artifact.sha256 in evidence and evidence[artifact.sha256] != identity: + raise ValueError('Conflicting waveform artifact descriptors') + artifacts[artifact.sha256] = artifact + evidence[artifact.sha256] = identity + if len(artifacts) > store.limits.max_artifacts or sum(item.compressed_bytes for item in artifacts.values()) > store.limits.total_bytes: + raise ValueError('Waveform dataset RAW storage budget exceeded') + if selected_cohorts != set(cohort_map): + raise ValueError('Retain waveform inputs for every selected cohort') + if not isinstance(selections, list) or len(selections) != len(artifacts): + raise ValueError('Archive selection must match frozen waveform inputs') + names = {} + for item in selections: + _keys(item, 'source_name artifact') + _keys(item['artifact'], 'name sha256 compressed_bytes expanded_bytes') + artifact = RawArtifact(**item['artifact']) + name = item['source_name'] + if not isinstance(name, str) or not SOURCE_NAME.fullmatch(name): + raise ValueError('Invalid archive selection name') + if artifact.sha256 in names or artifacts.get(artifact.sha256) != artifact: + raise ValueError('Archive selection must match frozen waveform inputs') + names[artifact.sha256] = name + existing = _existing(connection, dataset['idempotencyKey'], dataset['requestHash']) + if existing is not None: + return existing # Includes released tombstones; never resurrect a retry. + manifest_json, snapshot_json = _json(manifest), _json(snapshot) + byte_length = len(manifest_json.encode('utf-8')) + len(snapshot_json.encode('utf-8')) + + def publish(descriptors): + connection.execute('BEGIN IMMEDIATE') + try: + raced = _existing(connection, dataset['idempotencyKey'], dataset['requestHash']) + if raced is not None: + connection.commit() + return raced + total = connection.execute('SELECT coalesce(sum(byte_length), 0) FROM biometric_datasets').fetchone()[0] + if total + byte_length > MAX_TOTAL_BYTES: + raise ValueError('Waveform datasets exceeded the retained storage budget') + connection.execute('''INSERT INTO biometric_datasets + (id, side, subject_id, idempotency_key, request_hash, content_hash, manifest, + snapshot, byte_length, created_at_ms) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''', + (dataset['id'], dataset['side'], dataset['subjectId'], dataset['idempotencyKey'], + dataset['requestHash'], dataset['contentHash'], manifest_json, snapshot_json, + byte_length, dataset['createdAtMs'])) + for position, descriptor in enumerate(descriptors): + existing_artifact = connection.execute('''SELECT name, compressed_bytes, expanded_bytes + FROM biometric_raw_artifacts WHERE content_hash = ?''', (descriptor.sha256,)).fetchone() + identity = (descriptor.name, descriptor.compressed_bytes, descriptor.expanded_bytes) + if existing_artifact is not None and tuple(existing_artifact) != identity: + raise ValueError('Conflicting retained artifact metadata') + connection.execute('''INSERT INTO biometric_raw_artifacts + (content_hash, name, compressed_bytes, expanded_bytes, created_at_ms) + VALUES (?, ?, ?, ?, ?) ON CONFLICT(content_hash) DO UPDATE SET released_at_ms = NULL''', + (descriptor.sha256, *identity, dataset['createdAtMs'])) + connection.execute('''INSERT INTO biometric_dataset_raw_artifacts + (id, dataset_id, artifact_hash, position) VALUES (?, ?, ?, ?)''', + (str(uuid.uuid4()), dataset['id'], descriptor.sha256, position)) + result = _existing(connection, dataset['idempotencyKey'], dataset['requestHash']) + connection.commit() + return result + except BaseException: + connection.rollback() + raise + + return store.retain_batch(archive_directory, + [(names[digest], artifact) for digest, artifact in artifacts.items()], publish=publish) diff --git a/modules/common/biometrics/raw_records.py b/modules/common/biometrics/raw_records.py new file mode 100644 index 000000000..706eaa2b9 --- /dev/null +++ b/modules/common/biometrics/raw_records.py @@ -0,0 +1,234 @@ +"""Bounded firmware CBOR records for retained replay inputs. + +Decode containers incrementally instead of trusting attacker/corruption supplied +container lengths to a decoder allocation. Firmware uses indefinite maps, so +these remain supported along with definite containers and strings. Semantic +tags and non-text map keys are outside this versioned firmware-record contract. +""" +from dataclasses import dataclass +import struct + +from common.cbor_raw import read_raw_record + + +@dataclass(frozen=True) +class DecodeLimits: + record_bytes: int = 128 * 1024 + string_bytes: int = 64 * 1024 + nodes: int = 4096 + depth: int = 16 + + def __post_init__(self): + if any(type(value) is not int or value <= 0 for value in vars(self).values()): + raise ValueError('Invalid RAW decoding limits') + + +class _Decoder: + def __init__(self, payload, limits): + self.payload, self.limits, self.position, self.nodes = payload, limits, 0, 0 + + def take(self, size): + if size > len(self.payload) - self.position: + raise ValueError('Truncated inner CBOR record') + start = self.position + self.position += size + return self.payload[start:self.position] + + def argument(self, additional): + if additional < 24: + return additional + size = {24: 1, 25: 2, 26: 4, 27: 8}.get(additional) + if size is None: + raise ValueError('Invalid CBOR length encoding') + return int.from_bytes(self.take(size), 'big') + + def node(self): + self.nodes += 1 + if self.nodes > self.limits.nodes: + raise ValueError('CBOR node budget exceeded') + + def ended(self): + if self.position >= len(self.payload): + raise ValueError('Truncated indefinite CBOR container') + if self.payload[self.position] == 0xff: + self.position += 1 + return True + return False + + def value(self, depth=0): + self.node() + if depth > self.limits.depth: + raise ValueError('CBOR nesting budget exceeded') + header = self.take(1)[0] + major, additional = header >> 5, header & 31 + if major in (0, 1): + number = self.argument(additional) + return number if major == 0 else -1 - number + if major in (2, 3): + if additional == 31: + chunks, total = [], 0 + while not self.ended(): + self.node() + chunk = self.take(1)[0] + if chunk >> 5 != major or chunk & 31 == 31: + raise ValueError('Invalid indefinite CBOR string chunk') + size = self.argument(chunk & 31) + total += size + if total > self.limits.string_bytes: + raise ValueError('CBOR string budget exceeded') + data = self.take(size) + if major == 3: + data.decode('utf-8', errors='strict') + chunks.append(data) + data = b''.join(chunks) + else: + size = self.argument(additional) + if size > self.limits.string_bytes: + raise ValueError('CBOR string budget exceeded') + data = self.take(size) + return data if major == 2 else data.decode('utf-8', errors='strict') + if major in (4, 5): + count = None if additional == 31 else self.argument(additional) + multiplier = 2 if major == 5 else 1 + if count is not None and count * multiplier > self.limits.nodes - self.nodes: + raise ValueError('CBOR container budget exceeded') + result = {} if major == 5 else [] + consumed = 0 + while count is None or consumed < count: + if count is None and self.ended(): + break + item = self.value(depth + 1) + if major == 5: + if not isinstance(item, str): + raise ValueError('CBOR record map keys must be text') + if item in result: + raise ValueError('Duplicate CBOR record key') + result[item] = self.value(depth + 1) + else: + result.append(item) + consumed += 1 + return result + if major == 7: + if additional in (20, 21, 22): + return {20: False, 21: True, 22: None}[additional] + encoding = {25: ('>e', 2), 26: ('>f', 4), 27: ('>d', 8)}.get(additional) + if encoding: + return struct.unpack(encoding[0], self.take(encoding[1]))[0] + raise ValueError('Unsupported CBOR record value') + + +def decode_record(payload, limits=DecodeLimits()): + if not isinstance(payload, bytes) or len(payload) > limits.record_bytes: + raise ValueError('RAW inner record byte budget exceeded') + decoder = _Decoder(payload, limits) + result = decoder.value() + if decoder.position != len(payload): + raise ValueError('Trailing bytes after CBOR record') + if not isinstance(result, dict) or not isinstance(result.get('type'), str): + raise ValueError('RAW record must have a textual type') + return result + + +class RawRecordBudget: + """One count budget shared by all artifacts, including empty placeholders.""" + def __init__(self, maximum): + if type(maximum) is not int or maximum <= 0: + raise ValueError('Invalid RAW record count limit') + self.maximum, self.consumed = maximum, 0 + + def consume(self): + self.consumed += 1 + if self.consumed > self.maximum: + raise ValueError('RAW record count limit exceeded') + + +def iter_retained_records(store, descriptor, limits=DecodeLimits(), max_records=1_000_000, *, budget=None): + """Yield decoded records without buffering a file or concealing truncation.""" + if type(max_records) is not int or max_records <= 0: + raise ValueError('Invalid RAW record count limit') + if budget is not None and not isinstance(budget, RawRecordBudget): + raise ValueError('Invalid shared RAW record budget') + with store.open_raw(descriptor) as stream: + count = 0 + while True: + offset = stream.tell() + try: + payload = read_raw_record(stream) + except EOFError: + if stream.tell() != offset: + raise ValueError('Truncated retained RAW framing') from None + return + count += 1 + if budget is not None: + budget.consume() + if count > max_records: + raise ValueError('RAW record count limit exceeded') + if payload is not None: + yield decode_record(payload, limits) + + +def inventory_retained(store, descriptor, limits=DecodeLimits(), max_records=1_000_000): + """Inventory recorded event times and channel bytes, not inferred occupancy. + + Timestamp bounds are event bounds, not a claim of continuous signal support. + Missing timestamps or reversals make this ordered replay contract unavailable. + """ + import math + + result = {'version': 'retained-raw-inventory-v1', 'artifact_sha256': descriptor.sha256, + 'decoded_records': 0, 'piezo_records': 0, 'pump_records': 0, + 'usable_dual_frames': 0, 'empty_channel_frames': 0, + 'oversize_channel_frames': 0, 'invalid_timestamps': 0, + 'timestamp_reversals': 0, 'duplicate_timestamps': 0, + 'first_event_ms': None, 'last_event_ms': None, + 'channels': {side: {'samples': 0, 'lost_markers': 0, 'truncated_tail_bytes': 0} + for side in ('left1', 'right1')}} + previous = None + for record in iter_retained_records(store, descriptor, limits, max_records): + result['decoded_records'] += 1 + kind = record['type'] + if kind not in ('piezo-dual', 'frzHealth'): + continue + timestamp = record.get('ts') + if (type(timestamp) not in (int, float) or not math.isfinite(timestamp) + or timestamp < 0 or timestamp > 9_007_199_254_740): + result['invalid_timestamps'] += 1 + else: + timestamp_ms = math.floor(timestamp * 1000 + 0.5) + if previous is not None: + result['timestamp_reversals'] += int(timestamp_ms < previous) + result['duplicate_timestamps'] += int(timestamp_ms == previous) + previous = timestamp_ms + result['first_event_ms'] = timestamp_ms if result['first_event_ms'] is None else min(result['first_event_ms'], timestamp_ms) + result['last_event_ms'] = timestamp_ms if result['last_event_ms'] is None else max(result['last_event_ms'], timestamp_ms) + if kind == 'frzHealth': + result['pump_records'] += 1 + continue + result['piezo_records'] += 1 + usable, oversized = True, False + for side, totals in result['channels'].items(): + payload = record.get(side) + if not isinstance(payload, bytes): + usable = False + continue + if len(payload) > 20_000: + oversized = True + usable = False + continue + size = len(payload) - len(payload) % 4 + count = size // 4 + lost = sum(value == 2_147_483_647 for (value,) in struct.iter_unpack(' lost + result['usable_dual_frames'] += int(usable) + result['empty_channel_frames'] += int(not usable and not oversized) + result['oversize_channel_frames'] += int(oversized) + result['ordered_waveform_available'] = (result['usable_dual_frames'] > 0 + and result['invalid_timestamps'] == 0 and result['timestamp_reversals'] == 0 + and result['oversize_channel_frames'] == 0) + result['limitations'] = ['Event timestamps do not establish continuous waveform coverage.', + 'Pump record counts do not prove complete historical pump context.', + 'Sleeper mode, occupancy and calibration require separately retained context.'] + return result diff --git a/modules/common/biometrics/raw_sources.py b/modules/common/biometrics/raw_sources.py new file mode 100644 index 000000000..3b3e385d5 --- /dev/null +++ b/modules/common/biometrics/raw_sources.py @@ -0,0 +1,100 @@ +"""Read-only inspection of server-owned cold archives before selected retention. + +This adapter has the same bounded reader interface as retained artifacts, but +its inspection is only evidence about the current source. Selection must pass +the inspected descriptor to RawArtifactStore.retain(expected=...) before a +replay dataset can claim that bytes have been frozen. +""" +from contextlib import contextmanager +import gzip +import hashlib +import os +from pathlib import Path + +from .raw_artifacts import (RawArtifact, RawLimits, BoundedRawReader, SOURCE_NAME, + CHUNK, _identity, _regular) +from .raw_records import inventory_retained + + +def archive_names(directory, *, maximum_entries=10000): + if type(maximum_entries) is not int or maximum_entries <= 0: + raise ValueError('Invalid archive discovery bound') + root = Path(directory) + if root.is_symlink() or not root.is_dir(): + raise ValueError('Invalid RAW archive directory') + names = [] + with os.scandir(root) as entries: + for count, entry in enumerate(entries, 1): + if count > maximum_entries: + raise ValueError('RAW archive discovery entry limit exceeded') + if SOURCE_NAME.fullmatch(entry.name) and entry.is_file(follow_symlinks=False): + names.append(entry.name) + return sorted(names) + + +class ArchiveSource: + def __init__(self, directory, name, limits=RawLimits()): + root = Path(directory) + if root.is_symlink() or not root.is_dir() or not isinstance(name, str) or not SOURCE_NAME.fullmatch(name): + raise ValueError('Invalid RAW archive source') + self.path, self.limits = root / name, limits + + def _open(self): + fd = os.open(self.path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + try: + info = _regular(fd) + if info.st_size > self.limits.compressed_bytes: + raise ValueError('RAW compressed byte limit exceeded') + return fd, info + except BaseException: + os.close(fd) + raise + + def _hash(self, fd): + digest, size = hashlib.sha256(), 0 + while True: + chunk = os.read(fd, CHUNK) + if not chunk: + break + size += len(chunk) + if size > self.limits.compressed_bytes: + raise ValueError('RAW compressed byte limit exceeded') + digest.update(chunk) + return digest.hexdigest(), size + + def inspect(self): + fd, before = self._open() + with os.fdopen(fd, 'rb') as source: + digest, size = self._hash(source.fileno()) + source.seek(0) + expanded = 0 + with gzip.GzipFile(fileobj=source) as stream: + while True: + chunk = stream.read(min(CHUNK, self.limits.expanded_bytes - expanded + 1)) + if not chunk: + break + expanded += len(chunk) + if expanded > self.limits.expanded_bytes: + raise ValueError('RAW expanded byte limit exceeded') + if _identity(before) != _identity(_regular(source.fileno())): + raise ValueError('RAW archive changed during inspection') + descriptor = RawArtifact(digest + '.RAW.gz', digest, size, expanded) + # Reopening with this exact hash detects replacement between passes. + inventory = inventory_retained(self, descriptor) + return {'source_name': self.path.name, 'retained': False, + 'artifact': descriptor, 'inventory': inventory} + + @contextmanager + def open_raw(self, descriptor): + if not isinstance(descriptor, RawArtifact) or descriptor.expanded_bytes > self.limits.expanded_bytes: + raise ValueError('Invalid inspected RAW descriptor') + fd, before = self._open() + with os.fdopen(fd, 'rb') as source: + digest, size = self._hash(source.fileno()) + if (digest, size) != (descriptor.sha256, descriptor.compressed_bytes): + raise ValueError('RAW archive changed since inspection') + source.seek(0) + with gzip.GzipFile(fileobj=source) as stream: + yield BoundedRawReader(stream, descriptor.expanded_bytes) + if _identity(before) != _identity(_regular(source.fileno())): + raise ValueError('RAW archive changed during reading') diff --git a/modules/common/biometrics/replay.py b/modules/common/biometrics/replay.py new file mode 100644 index 000000000..ccd490ce0 --- /dev/null +++ b/modules/common/biometrics/replay.py @@ -0,0 +1,194 @@ +"""Bounded event-time waveform replay using the shipped stateful processor. + +Callers stream retained records and complete recorded context snapshots. This +module never opens live state or a database. Output is delivered incrementally +to an experiment sink; the driver retains only bounded processor buffers. +""" +from dataclasses import dataclass +from datetime import datetime +from typing import Callable, Mapping, Optional, Tuple + +from .config import config_hash, validate_config +from .frames import decode_int32_samples +from .gates import FrzHealthPumpState, PumpGate +from .processing import StatefulSideProcessor +from .routing import SingleSleeperRouter +from .signal import ProcessingParameters + +SIDES = ('left', 'right') +MAX_CHANNEL_BYTES = 5000 * 4 # Ten seconds at the supported 500 Hz rate. + + +@dataclass(frozen=True) +class RecordedContext: + active_sides: Optional[Tuple[str, ...]] = None + left_occupied: Optional[bool] = None + right_occupied: Optional[bool] = None + pump_context_known: bool = False + + def __post_init__(self): + if self.active_sides not in (None, (), ('left',), ('right',), SIDES): + raise ValueError('Invalid recorded sleeper mode') + for value in (self.left_occupied, self.right_occupied): + if value is not None and type(value) is not bool: + raise ValueError('Invalid recorded occupancy') + if type(self.pump_context_known) is not bool: + raise ValueError('Invalid pump context availability') + + +@dataclass(frozen=True) +class ReplayProvenance: + side: str + source_side: str + config_hash: str + + +@dataclass(frozen=True) +class ReplayCandidate: + side: str + ts: datetime + heart_rate: Optional[float] + hrv: Optional[float] + breathing_rate: Optional[float] + quality_score: float + flags: Optional[Tuple[str, ...]] + hr_raw: Optional[float] + run_context: ReplayProvenance + + def __post_init__(self): + object.__setattr__(self, 'flags', tuple(self.flags) if self.flags else None) + + +@dataclass(frozen=True) +class ReplayReading: + candidate: ReplayCandidate + side: str + timestamp_ms: int + limitations: Tuple[str, ...] + + +class WaveformReplay: + """Replay [retained_start_ms, score_end_ms); score only the requested window. + + Timestamps must be integer Unix milliseconds in retained record order. + Equal timestamps are allowed (pump/context and piezo records can coincide). + Both physical channels are always processed, left first. Configurations are + fixed experiment snapshots indexed by logical side, not live bindings. + """ + def __init__(self, configs: Mapping[str, dict], *, retained_start_ms: int, + score_start_ms: int, score_end_ms: int, + sink: Callable[[ReplayReading], None], clock_mode='recorded_event_time'): + for value in (retained_start_ms, score_start_ms, score_end_ms): + if type(value) is not int or value < 0: + raise ValueError('Invalid replay timestamp') + if not retained_start_ms <= score_start_ms < score_end_ms: + raise ValueError('Invalid replay window') + if set(configs) != set(SIDES): + raise ValueError('Both logical-side configurations are required') + for config in configs.values(): + validate_config(config) + self.parameters = {side: ProcessingParameters.from_config(configs[side]) for side in SIDES} + self.hashes = {side: config_hash(configs[side]) for side in SIDES} + self.retained_start_ms, self.score_start_ms, self.score_end_ms = retained_start_ms, score_start_ms, score_end_ms + self._now_ms = retained_start_ms + self._closed = False + self._context = RecordedContext() + self._sink = sink + if clock_mode not in ('recorded_event_time', 'observed_arrival_time'): + raise ValueError('Invalid processing clock mode') + self.limitations = {'cold_start_without_serialized_state'} + if clock_mode == 'recorded_event_time': + self.limitations.add('event_time_replaces_arrival_time') + if score_start_ms - retained_start_ms < 300_000: + self.limitations.add('less_than_five_minutes_preroll') + self.counts = {'records': 0, 'accepted_frames': 0, 'empty_channel_frames': 0, + 'pump_gated_frames': 0, 'preroll_outputs': 0, 'scored_outputs': 0} + clock = lambda: self._now_ms / 1000 + self.pumps = FrzHealthPumpState(clock=clock) + self.gate = PumpGate(clock=clock) + self.router = SingleSleeperRouter(self, self._write, clock=clock) + self.processors = {} + for side in SIDES: + processor = StatefulSideProcessor(side, clock=clock, + writer=lambda candidate: self._write(candidate, None), + candidate_factory=ReplayCandidate, pump_state=self.pumps, + parameters=self.parameters[side]) + processor.sink = self.router.submit + processor.home_side = self.home_side + processor.bed_occupied = lambda side=side: self._occupied(self.home_side() or side) + self.processors[side] = processor + self.processors['left']._other = self.processors['right'] + self.processors['right']._other = self.processors['left'] + + def active_sides(self): + # The fallback is explicitly approximate, never reported as known mode. + return self._context.active_sides if self._context.active_sides is not None else SIDES + + def home_side(self): + active = self.active_sides() + return active[0] if len(active) == 1 else None + + def _occupied(self, side): + return self._context.left_occupied if side == 'left' else self._context.right_occupied + + def _write(self, candidate, destination): + timestamp_ms = round(candidate.ts.timestamp() * 1000) + if self.score_start_ms <= timestamp_ms < self.score_end_ms: + # Like live provenance, a held candidate retains its computed side + # even when the recorded mode changes before its sink is flushed. + self._sink(ReplayReading(candidate, candidate.run_context.side, + timestamp_ms, tuple(sorted(self.limitations)))) + self.counts['scored_outputs'] += 1 + else: + self.counts['preroll_outputs'] += 1 + return True + + def feed(self, timestamp_ms: int, record: dict, context: RecordedContext): + if self._closed: + raise ValueError('Replay is closed') + if type(timestamp_ms) is not int or not self._now_ms <= timestamp_ms < self.score_end_ms: + raise ValueError('Replay records must be ordered inside the retained window') + if not isinstance(record, dict) or not isinstance(context, RecordedContext): + raise ValueError('Invalid replay record or context') + for side in SIDES: + payload = record.get(side + '1', b'') + if isinstance(payload, (bytes, bytearray, memoryview)) and len(payload) > MAX_CHANNEL_BYTES: + raise ValueError('Replay channel exceeds bounded frame size') + self._now_ms, self._context = timestamp_ms, context + self.counts['records'] += 1 + if context.active_sides is None: + self.limitations.add('missing_sleeper_mode') + if context.left_occupied is None or context.right_occupied is None: + self.limitations.add('missing_occupancy_context') + if not context.pump_context_known: + self.limitations.add('missing_pump_context') + self.router.tick() + if record.get('type') == 'frzHealth': + self.pumps.update(record) + return + if record.get('type') != 'piezo-dual': + return + left = decode_int32_samples(record.get('left1', b'')) + right = decode_int32_samples(record.get('right1', b'')) + if left.size == 0 or right.size == 0: + self.counts['empty_channel_frames'] += 1 + return + if self.gate.check(left, right): + self.counts['pump_gated_frames'] += 1 + return + for side in SIDES: + processor = self.processors[side] + logical = self.home_side() or side + previous = processor.run_context + provenance = ReplayProvenance(logical, side, self.hashes[logical]) + processor.install_runtime(self.parameters[logical], provenance, + reset_side=previous is None or previous.side != logical) + self.processors['left'].ingest(left) + self.processors['right'].ingest(right) + self.counts['accepted_frames'] += 1 + + def close(self): + if not self._closed: + self._now_ms = self.score_end_ms + self.router.flush() + self._closed = True diff --git a/modules/common/biometrics/replay_worker.py b/modules/common/biometrics/replay_worker.py new file mode 100644 index 000000000..8d5cf8be6 --- /dev/null +++ b/modules/common/biometrics/replay_worker.py @@ -0,0 +1,167 @@ +"""Fixed shipped waveform worker; JSON input and bounded JSON-lines output. + +This is process/resource separation for trusted shipped algorithms, not a +sandbox for uploaded executable code. The coordinator owns staging and fenced +result publication; reading events alone never constitute a successful job. +""" +import argparse +import json +import os +import subprocess +import re +import sys +import time + +from .execution_lock import acquire_execution_lock + +MAX_INPUT_BYTES = 8 * 1024 * 1024 +MAX_OUTPUT_BYTES = 16 * 1024 * 1024 + + +def _unique_object(pairs): + value = {} + for key, item in pairs: + if key in value: + raise ValueError('Duplicate input field') + value[key] = item + return value + + +def _invalid_constant(_): + raise ValueError('Nonfinite JSON input') + + +def peak_rss_bytes(): + """Kernel process-lifetime high water mark; never substitute current RSS. + + Linux procfs also works on Yocto without the optional resource extension. + Missing or unrecognized telemetry remains unknown, never zero. + """ + try: + if sys.platform == 'linux': + with open('/proc/self/status', 'r', encoding='ascii') as source: + status = source.read(16385) + if len(status) > 16384: + return None + matches = re.findall(r'^VmHWM:\s+(\d+) kB$', status, re.MULTILINE) + value = int(matches[0]) * 1024 if len(matches) == 1 else None + elif sys.platform == 'darwin': + import resource + value = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + else: + return None + return value if type(value) is int and 0 < value <= 2 ** 53 - 1 else None + except (OSError, UnicodeError, ImportError, ValueError): + return None + + +def run(input_stream, output_stream, retained_directory): + started_wall = time.perf_counter_ns() + started_cpu = time.process_time_ns() + payload = input_stream.read(MAX_INPUT_BYTES + 1) + if len(payload) > MAX_INPUT_BYTES: + raise ValueError('Waveform work input exceeds byte budget') + work = json.loads(payload, object_pairs_hook=_unique_object, parse_constant=_invalid_constant) + if not isinstance(work, dict) or set(work) not in ({'version', 'input', 'configs'}, {'version', 'input', 'configs', 'expected_build_hash'}) or work['version'] != 'retained-waveform-work-v1': + raise ValueError('Invalid waveform work envelope') + from .build_identity import replay_build_identity + build = replay_build_identity() + if 'expected_build_hash' in work and (not isinstance(work['expected_build_hash'], str) + or not re.fullmatch(r'[a-f0-9]{64}', work['expected_build_hash']) + or work['expected_build_hash'] != build['hash']): + raise ValueError('Replay build identity changed') + # Import numerical dependencies after CLI resource/BLAS setup. + from .raw_artifacts import RawArtifactStore + from .retained_replay import replay_retained + store = RawArtifactStore(retained_directory, read_only=True) + output_bytes = 0 + + def emit(event): + nonlocal output_bytes + encoded = (json.dumps(event, allow_nan=False, separators=(',', ':')) + '\n').encode('utf-8') + output_bytes += len(encoded) + if output_bytes > MAX_OUTPUT_BYTES: + raise ValueError('Waveform work output exceeds byte budget') + output_stream.write(encoded) + output_stream.flush() + + def reading(row): + candidate = row.candidate + emit({'type': 'reading', 'reading': { + 'side': row.side, 'timestampMs': row.timestamp_ms, + 'heartRate': candidate.heart_rate, 'hrvIndex': candidate.hrv, + 'hrvMetric': 'window_ibi_variability_v1', + 'breathingRate': candidate.breathing_rate, 'qualityScore': candidate.quality_score, + 'flags': candidate.flags, 'hrRaw': candidate.hr_raw, + 'sourceSide': candidate.run_context.source_side, + 'configHash': candidate.run_context.config_hash, 'limitations': row.limitations, + }}) + + result = replay_retained(store, work['input'], work['configs'], reading) + # Detect source/dependency metadata changes during this process as well. + if replay_build_identity()['hash'] != build['hash']: + raise ValueError('Replay build identity changed during execution') + emit({'type': 'result', 'result': {**result, 'build_hash': build['hash'], + 'resources': {'version': 'waveform-process-resources-v1', + 'wallMs': (time.perf_counter_ns() - started_wall) / 1_000_000, + 'cpuMs': (time.process_time_ns() - started_cpu) / 1_000_000, + 'peakRssBytes': peak_rss_bytes(), + 'scope': 'python_process_until_result', + 'memoryScope': 'process_lifetime_high_water_mark'}}}) + + +def configure_process_limits(cpu_seconds=120): + if type(cpu_seconds) is not int or not 1 <= cpu_seconds <= 3600: + raise ValueError('Invalid waveform CPU budget') + # Provisional execution caps; qualified scheduling/admission belongs to + # the parent. It must also enforce wall time and kill on cancellation. + try: + import resource + except ModuleNotFoundError as error: + if error.name != 'resource' or sys.platform != 'linux': + raise + # Yocto may omit Python's optional resource extension. Use the fixed + # system utility to limit this process before importing numerical code. + # Missing utility, refusal, or timeout remains a startup failure. + subprocess.run(['/usr/bin/prlimit', '--pid', str(os.getpid()), + '--core=0:0', f'--cpu={cpu_seconds}:{cpu_seconds + 1}', '--nofile=64:64'], + check=True, timeout=5, stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + else: + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + resource.setrlimit(resource.RLIMIT_CPU, (cpu_seconds, cpu_seconds + 1)) + resource.setrlimit(resource.RLIMIT_NOFILE, (64, 64)) + os.nice(19) + for key in ('OPENBLAS_NUM_THREADS', 'OMP_NUM_THREADS', 'MKL_NUM_THREADS', 'NUMEXPR_NUM_THREADS'): + os.environ[key] = '1' + + +def main(): + parser = argparse.ArgumentParser(description='Replay fixed retained waveform inputs') + parser.add_argument('--cpu-seconds', type=int, default=120) + target = parser.add_mutually_exclusive_group(required=True) + target.add_argument('--retained-directory') + target.add_argument('--describe-build', action='store_true') + args = parser.parse_args() + lock_fd = None + try: + if not args.describe_build: + lock_fd = acquire_execution_lock() + configure_process_limits(args.cpu_seconds) + if args.describe_build: + from .build_identity import replay_build_identity + sys.stdout.write(json.dumps(replay_build_identity(), allow_nan=False, separators=(',', ':')) + '\n') + else: + run(sys.stdin.buffer, sys.stdout.buffer, args.retained_directory) + return 0 + except Exception: + # Never echo reference exports, frame values, paths or tracebacks. + sys.stderr.write('Retained waveform worker failed\n') + return 1 + finally: + if lock_fd is not None: + os.close(lock_fd) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/modules/common/biometrics/retained_replay.py b/modules/common/biometrics/retained_replay.py new file mode 100644 index 000000000..1286d3fab --- /dev/null +++ b/modules/common/biometrics/retained_replay.py @@ -0,0 +1,163 @@ +"""Replay immutable artifact descriptors with a frozen context timeline. + +Sink rows are provisional until this function returns successfully: a later +archive can fail verification. The durable worker must publish them atomically +with its successful result, never into canonical observations. +""" +from dataclasses import dataclass +import math + +from .raw_artifacts import RawArtifact +from .raw_records import RawRecordBudget, iter_retained_records +from .replay import RecordedContext, WaveformReplay + +MAX_SAFE_INTEGER = 9_007_199_254_740_991 + + +def _integer(value): + if type(value) is not int or not 0 <= value <= MAX_SAFE_INTEGER: + raise ValueError('Expected a nonnegative safe integer') + return value + + +def _keys(value, expected): + if not isinstance(value, dict) or set(value) != set(expected.split()): + raise ValueError('Unexpected retained replay fields') + + +def _timestamp(record): + value = record.get('ts') + if type(value) not in (int, float) or not math.isfinite(value) or not 0 <= value <= 9_007_199_254_740: + raise ValueError('Invalid retained event timestamp') + # Same rounding contract as the inspected inventory; no local timezone or + # Python bankers-rounding dependency at half-millisecond boundaries. + return _integer(math.floor(value * 1000 + 0.5)) + + +@dataclass(frozen=True) +class ArtifactInput: + artifact: RawArtifact + first_event_ms: int + last_event_ms: int + + +@dataclass(frozen=True) +class ContextInterval: + start_ms: int + end_ms: int + context: RecordedContext + + +@dataclass(frozen=True) +class RetainedReplayInput: + retained_start_ms: int + score_start_ms: int + score_end_ms: int + artifacts: tuple + contexts: tuple + + @classmethod + def parse(cls, raw): + _keys(raw, 'version retained_start_ms score_start_ms score_end_ms artifacts contexts') + if raw['version'] != 'retained-waveform-input-v1': + raise ValueError('Unsupported retained waveform input version') + start, score, end = (_integer(raw[key]) for key in ('retained_start_ms', 'score_start_ms', 'score_end_ms')) + if not start <= score < end: + raise ValueError('Invalid retained replay window') + if not isinstance(raw['artifacts'], list) or not 1 <= len(raw['artifacts']) <= 64: + raise ValueError('Select between one and 64 retained artifacts') + artifacts, hashes, compressed = [], set(), 0 + for item in raw['artifacts']: + _keys(item, 'artifact first_event_ms last_event_ms') + _keys(item['artifact'], 'name sha256 compressed_bytes expanded_bytes') + artifact = RawArtifact(**item['artifact']) + first, last = _integer(item['first_event_ms']), _integer(item['last_event_ms']) + if first > last or last < start or first >= end: + raise ValueError('Artifact has no events in the retained window') + if artifact.sha256 in hashes or (artifacts and first <= artifacts[-1].last_event_ms): + raise ValueError('Retained artifacts must be distinct and have nonoverlapping ordered event bounds') + hashes.add(artifact.sha256) + compressed += artifact.compressed_bytes + if compressed > 1024 * 1024 * 1024: + raise ValueError('Retained replay compressed input budget exceeded') + artifacts.append(ArtifactInput(artifact, first, last)) + if not isinstance(raw['contexts'], list) or len(raw['contexts']) > 20000: + raise ValueError('Retained context interval budget exceeded') + contexts = [] + for item in raw['contexts']: + _keys(item, 'start_ms end_ms active_sides left_occupied right_occupied pump_context_known') + first, last = _integer(item['start_ms']), _integer(item['end_ms']) + if not start <= first < last <= end or (contexts and first < contexts[-1].end_ms): + raise ValueError('Context intervals must be ordered and nonoverlapping inside the retained window') + active = item['active_sides'] + if active is not None and not isinstance(active, list): + raise ValueError('Recorded active sides must be a list or unknown') + context = RecordedContext(None if active is None else tuple(active), + item['left_occupied'], item['right_occupied'], item['pump_context_known']) + contexts.append(ContextInterval(first, last, context)) + return cls(start, score, end, tuple(artifacts), tuple(contexts)) + + +def replay_retained(store, raw_input, configs, sink, *, cancelled=lambda: False, + maximum_readings=20000, maximum_records=1_000_000): + """Run fixed shipped code; no archive fallback, live DB, or control access. + + Cross-artifact overlap is rejected rather than guessing which duplicate + frames to drop. Gaps are preserved; event bounds never imply continuity. + Every selected artifact is consumed to EOF, including out-of-window records, + so decompression/framing and the inspected event bounds are verified. + """ + if type(maximum_readings) is not int or maximum_readings <= 0 or type(maximum_records) is not int or maximum_records <= 0: + raise ValueError('Invalid retained replay resource bounds') + snapshot = RetainedReplayInput.parse(raw_input) + readings = 0 + + def write(row): + nonlocal readings + if cancelled(): + raise ValueError('Retained replay cancelled') + readings += 1 + if readings > maximum_readings: + raise ValueError('Retained replay output budget exceeded') + sink(row) + + replay = WaveformReplay(configs, retained_start_ms=snapshot.retained_start_ms, + score_start_ms=snapshot.score_start_ms, score_end_ms=snapshot.score_end_ms, sink=write) + budget = RawRecordBudget(maximum_records) + index, decoded = 0, 0 + previous = None + for source in snapshot.artifacts: + if cancelled(): + raise ValueError('Retained replay cancelled') + first = last = None + for record in iter_retained_records(store, source.artifact, max_records=maximum_records, budget=budget): + if cancelled(): + raise ValueError('Retained replay cancelled') + decoded += 1 + if decoded > maximum_records: + raise ValueError('Retained replay decoded record budget exceeded') + if record['type'] not in ('piezo-dual', 'frzHealth'): + continue + timestamp = _timestamp(record) + if previous is not None and timestamp < previous: + raise ValueError('Retained event timestamps reversed') + previous = timestamp + first = timestamp if first is None else first + last = timestamp + if not source.first_event_ms <= timestamp <= source.last_event_ms: + raise ValueError('Retained event lies outside inspected bounds') + if not snapshot.retained_start_ms <= timestamp < snapshot.score_end_ms: + continue + while index < len(snapshot.contexts) and timestamp >= snapshot.contexts[index].end_ms: + index += 1 + interval = snapshot.contexts[index] if index < len(snapshot.contexts) else None + context = interval.context if interval and timestamp >= interval.start_ms else RecordedContext() + replay.feed(timestamp, record, context) + if first != source.first_event_ms or last != source.last_event_ms: + raise ValueError('Retained event bounds differ from inspected source') + if cancelled(): + raise ValueError('Retained replay cancelled') + replay.close() + return {'version': 'retained-waveform-run-v1', 'decoded_records': decoded, 'raw_records': budget.consumed, + 'counts': dict(replay.counts), 'limitations': sorted(replay.limitations), + 'config_hashes': dict(replay.hashes), 'artifact_hashes': [item.artifact.sha256 for item in snapshot.artifacts]} diff --git a/modules/common/biometrics/routing.py b/modules/common/biometrics/routing.py new file mode 100644 index 000000000..2e1d5a628 --- /dev/null +++ b/modules/common/biometrics/routing.py @@ -0,0 +1,102 @@ +"""Single-sleeper arbitration shared by live processing and replay. + +The caller supplies recorded/current mode, clock, and output sink. Candidates +are opaque except for physical side and quality. This module cannot read +current occupancy, query a database, or select a canonical output destination. +""" +from typing import Callable, Generic, Optional, Protocol, Sequence, TypeVar + +VITALS_INTERVAL_S = 60 + + +class Candidate(Protocol): + side: str + quality_score: float + + +class SleeperMode(Protocol): + def home_side(self) -> Optional[str]: ... + def active_sides(self) -> Sequence[str]: ... + + +CandidateT = TypeVar("CandidateT", bound=Candidate) + + +class SingleSleeperRouter(Generic[CandidateT]): + """Routes vitals rows when one side is in away mode (single sleeper). + + A solo sleeper who rolls onto the away side is picked up by that side's + piezo too — the same heartbeat, within a few bpm. Each cycle's + candidates from the two sides are paired and + only the higher-quality one is written, under the home side, so the + home session shows one heart-rate series with no gaps or duplicates. + A candidate whose partner doesn't arrive within VITALS_INTERVAL_S (the + other side saw no one) is written alone under the home side. + + Outside single-sleeper mode rows are written under their own side. + """ + + def __init__(self, mode: SleeperMode, writer: Callable[[CandidateT, Optional[str]], bool], + *, clock: Callable[[], float]): + self._writer = writer + self._mode = mode + self._clock = clock + self._pending: Optional[CandidateT] = None + self._retry: Optional[CandidateT] = None + self._retry_home: Optional[str] = None + self._pending_home: Optional[str] = None + self._pending_at = 0.0 + + def submit(self, cand: CandidateT) -> bool: + """Accept one side's candidate. True means the side may advance its + write cursor: the row was written, or is held (for pairing or a + write retry). A held candidate is only dropped once written.""" + # Completed intervals must never pair with a later candidate. + if self._retry is not None and not self.flush(): + return False + home = self._mode.home_side() + if not self._mode.active_sides(): + self._pending = self._retry = None + return True + if home is None: + if not self.flush(): + return False + return self._writer(cand, None) + pending = self._pending + if pending is not None and pending.side != cand.side and self._pending_home == home: + best = cand if cand.quality_score > pending.quality_score else pending + self._pending = best + self.flush() + return True + # A same-side repeat means its partner never arrived: write the + # older one first. If that fails, refuse the new candidate so its + # side keeps it and resubmits, rather than dropping the older row. + if not self.flush(): + return False + self._pending, self._pending_home, self._pending_at = cand, home, self._clock() + return True + + def tick(self) -> None: + """Write a held candidate whose partner didn't arrive in time.""" + if not self._mode.active_sides(): + self._pending = self._retry = None + return + held = self._pending is not None or self._retry is not None + if held and self._clock() - self._pending_at >= VITALS_INTERVAL_S: + self.flush() + + def flush(self) -> bool: + """Write the held candidate, if any. On failure it stays held and + idle retries wait VITALS_INTERVAL_S; a new submission retries first. + True when nothing is left held.""" + if self._retry is None: + self._retry, self._retry_home = self._pending, self._pending_home + self._pending = None + if self._retry is None: + return True + if self._writer(self._retry, self._retry_home): + self._retry = None + return True + self._pending_at = self._clock() + return False + diff --git a/modules/common/biometrics/runtime.py b/modules/common/biometrics/runtime.py new file mode 100644 index 000000000..8d1e59744 --- /dev/null +++ b/modules/common/biometrics/runtime.py @@ -0,0 +1,139 @@ +"""Validated runtime projections and sample-boundary selection. + +This module never changes desired bindings or fabricates applied receipts. +Callers install a selected snapshot in their estimator before acknowledging it. +Unknown occupancy is deliberately not an empty-bed boundary. +""" +from dataclasses import dataclass +import json +from uuid import UUID + +from .config import canonical_json, config_hash, manifest, validate_config + +_MAX_PAYLOAD_BYTES = 65536 +_FIELDS = {"schemaVersion", "side", "generation", "revisionId", "manifestHash", + "resolved", "resolvedHash", "boundary"} + + +def _unique_object(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError("Duplicate runtime payload field") + result[key] = value + return result + + +@dataclass(frozen=True) +class RuntimeSnapshot: + """Deep immutability: mutable configuration is decoded afresh on access.""" + side: str + generation: int + resolved_hash: str + manifest_hash: str + boundary: str + revision_id: object + _resolved_json: str + + @property + def resolved(self): + return json.loads(self._resolved_json) + + @classmethod + def parse(cls, payload): + if not isinstance(payload, str) or len(payload.encode("utf-8")) > _MAX_PAYLOAD_BYTES: + raise ValueError("Runtime payload exceeds resource limit") + try: + value = json.loads(payload, object_pairs_hook=_unique_object) + except (RecursionError, json.JSONDecodeError) as error: + raise ValueError("Invalid runtime payload JSON") from error + if not isinstance(value, dict) or set(value) != _FIELDS: + raise ValueError("Invalid runtime payload fields") + if type(value["schemaVersion"]) is not int or value["schemaVersion"] != 1: + raise ValueError("Unsupported runtime schema") + if value["side"] not in ("left", "right"): + raise ValueError("Invalid runtime side") + generation = value["generation"] + if type(generation) is not int or not 0 < generation <= 9007199254740991: + raise ValueError("Invalid runtime generation") + if value["boundary"] not in ("immediate", "next_session"): + raise ValueError("Invalid runtime boundary") + revision = value["revisionId"] + if revision is not None: + if not isinstance(revision, str): + raise ValueError("Invalid runtime revision") + try: + if str(UUID(revision)) != revision.lower(): + raise ValueError("Invalid runtime revision") + except (ValueError, AttributeError) as error: + raise ValueError("Invalid runtime revision") from error + if value["manifestHash"] != config_hash(manifest()): + raise ValueError("PROFILE_INCOMPATIBLE: unknown runtime manifest") + resolved = validate_config(value["resolved"]) + if config_hash(resolved) != value["resolvedHash"]: + raise ValueError("Runtime configuration hash mismatch") + return cls(value["side"], generation, value["resolvedHash"], + value["manifestHash"], value["boundary"], revision, + canonical_json(resolved)) + + +def read_latest_snapshot(connection, side): + """Read one bounded projection. Missing schema is surfaced, never migrated. + + An unavailable/corrupt newest generation must not silently select an older + generation. The daemon decides its explicit fallback and reports the error. + """ + if side not in ("left", "right"): + raise ValueError("Invalid runtime side") + # Do not materialize an oversized SQLite TEXT into the daemon process. + row = connection.execute( + "SELECT side, generation, resolved_hash, boundary, " + "CASE WHEN length(CAST(payload AS BLOB)) <= ? THEN payload ELSE NULL END " + "FROM biometric_runtime_configs WHERE side = ? " + "ORDER BY generation DESC LIMIT 1", (_MAX_PAYLOAD_BYTES, side)).fetchone() + if row is None: + return None + snapshot = RuntimeSnapshot.parse(row[4]) + if (snapshot.side, snapshot.generation, snapshot.resolved_hash, snapshot.boundary) != tuple(row[:4]): + raise ValueError("Runtime projection envelope mismatch") + return snapshot + + +class BoundarySelection: + """Two-phase selection: no application claim until the consumer commits. + + Supply a previously applied snapshot only after restoring its estimator. + A restart with no restored snapshot and an occupied/unknown bed waits for + the next empty boundary for next-session requests. + """ + def __init__(self, side, applied=None): + if side not in ("left", "right") or (applied is not None and applied.side != side): + raise ValueError("Runtime side mismatch") + self.side = side + self.applied = applied + self.pending = None + + def offer(self, snapshot): + if snapshot.side != self.side: + raise ValueError("Runtime side mismatch") + current = self.pending or self.applied + if current is not None: + if snapshot.generation < current.generation: + raise ValueError("Runtime generation regression") + if snapshot.generation == current.generation: + if snapshot != current: + raise ValueError("Runtime generation conflict") + return + self.pending = snapshot + + def ready(self, occupied): + if self.pending is None: + return None + if self.pending.boundary == "immediate" or occupied is False: + return self.pending + return None + + def commit(self, snapshot): + if self.pending != snapshot: + raise ValueError("Runtime selection changed before adoption") + self.applied, self.pending = snapshot, None diff --git a/modules/common/biometrics/shadow.py b/modules/common/biometrics/shadow.py new file mode 100644 index 000000000..6ed1a5259 --- /dev/null +++ b/modules/common/biometrics/shadow.py @@ -0,0 +1,135 @@ +"""Prospective adapter for the shipped core, with an isolated candidate state. + +No database, hardware, code loading or source selection. A separately admitted +host supplies observed arrival times/context and connects the sink to shadow +storage. CPU checks are cooperative; the host must also enforce process limits. +""" +from copy import deepcopy +from dataclasses import dataclass +import hashlib +import time + +from .config import canonical_json +from .replay import MAX_CHANNEL_BYTES, RecordedContext, ReplayReading, WaveformReplay + + +@dataclass(frozen=True) +class ShadowOutput: + sequence: int + input_hash: str + reading: ReplayReading + + +class ProspectiveShadow: + def __init__(self, configs, *, side, start_ms, end_ms, maximum_records, + maximum_outputs, maximum_cpu_ms, sink, cpu_clock=time.process_time_ns): + if side not in ('left', 'right'): + raise ValueError('Invalid shadow side') + for value in (maximum_records, maximum_outputs, maximum_cpu_ms): + if type(value) is not int or not 0 < value <= 9_007_199_254_740_991: + raise ValueError('Invalid shadow resource budget') + if type(start_ms) is not int or type(end_ms) is not int or not 0 <= start_ms < end_ms <= 9_007_199_254_740_991 or end_ms - start_ms > 7 * 86400000: + raise ValueError('Invalid prospective window') + self.side, self.start_ms, self.end_ms = side, start_ms, end_ms + self.maximum_records, self.maximum_outputs = maximum_records, maximum_outputs + self.maximum_cpu_ns = maximum_cpu_ms * 1_000_000 + self._cpu_clock, self._sink = cpu_clock, sink + self._cpu_ns, self._records, self._outputs = 0, 0, 0 + self._staged = [] + self._digest = hashlib.sha256(b'biometric-shadow-input-v1\n').digest() + self.state, self.reason = 'running', None + self._processor = WaveformReplay(deepcopy(configs), retained_start_ms=start_ms, + score_start_ms=start_ms, score_end_ms=end_ms, sink=self._capture, + clock_mode='observed_arrival_time') + + def _capture(self, reading): + if reading.side != self.side: + return + if len(self._staged) >= 4 or self._outputs + len(self._staged) >= self.maximum_outputs: + raise ValueError('Shadow output budget exceeded') + self._staged.append(reading) + + def _read_cpu(self): + value = self._cpu_clock() + if type(value) is not int or value < 0: + raise ValueError('Invalid shadow CPU clock') + return value + + def _execute(self, action): + if self.state != 'running': + raise ValueError('Shadow processor is stopped') + self._staged = [] + try: + before = self._read_cpu() + if self._cpu_ns >= self.maximum_cpu_ns: + raise ValueError('Shadow CPU budget exceeded') + try: + action() + finally: + # Failed numerical/validation calls still consumed CPU. Do not + # report a zero-cost failure or permit reuse after clock failure. + elapsed = self._read_cpu() - before + if elapsed < 0: + raise ValueError('Shadow CPU clock regressed') + self._cpu_ns += elapsed + if self._cpu_ns > self.maximum_cpu_ns: + raise ValueError('Shadow CPU budget exceeded') + # Do not publish this frame's provisional output after a budget or + # processing failure. A sink failure terminates further processing. + for reading in self._staged: + self._sink(ShadowOutput(self._outputs, self._digest.hex(), reading)) + self._outputs += 1 + except Exception as error: + self.state, self.reason = 'failed', str(error)[:500] + raise + finally: + self._staged = [] + + def feed(self, observed_at_ms, record, context): + """Observed arrival time matches the live core's clock semantics. + Preserve source event time in record metadata, but never use it as the + live processing clock. Both channels/context feed the isolated state. + """ + def action(): + if self._records >= self.maximum_records: + raise ValueError('Shadow record budget exceeded') + if not isinstance(record, dict) or record.get('type') not in ('piezo-dual', 'frzHealth') or not isinstance(context, RecordedContext): + raise ValueError('Invalid shadow record or context') + for channel in ('left1', 'right1'): + if channel in record: + value = record[channel] + if not isinstance(value, (bytes, bytearray, memoryview)) or len(value) > MAX_CHANNEL_BYTES: + raise ValueError('Invalid bounded shadow channel') + frozen = {key: bytes(value) if key in ('left1', 'right1') else deepcopy(value) for key, value in record.items()} + identity = dict(frozen) + for channel in ('left1', 'right1'): + if channel in frozen: + value = frozen[channel] + if not isinstance(value, (bytes, bytearray, memoryview)) or len(value) > MAX_CHANNEL_BYTES: + raise ValueError('Invalid bounded shadow channel') + frozen[channel] = bytes(value) + identity[channel] = {'bytes': len(value), 'sha256': hashlib.sha256(value).hexdigest()} + encoded = canonical_json({'observedAtMs': observed_at_ms, 'record': identity, + 'context': {'activeSides': list(context.active_sides) if context.active_sides is not None else None, + 'leftOccupied': context.left_occupied, 'rightOccupied': context.right_occupied, + 'pumpContextKnown': context.pump_context_known}}).encode() + self._digest = hashlib.sha256(self._digest + encoded).digest() + self._processor.feed(observed_at_ms, frozen, context) + self._records += 1 + self._execute(action) + + def finish(self): + """Normal completion flushes arbitration once. Cancellation does not.""" + self._execute(self._processor.close) + self.state, self.reason = 'completed', 'window_complete' + + def stop(self, reason='cancelled'): + if self.state == 'running': + self.state, self.reason = 'stopped', str(reason)[:500] + self._staged = [] + + def health(self): + return {'state': self.state, 'reason': self.reason, 'records': self._records, + 'outputs': self._outputs, 'cpu_ms': self._cpu_ns / 1_000_000, + 'input_hash': self._digest.hex(), 'counts': dict(self._processor.counts), + 'limitations': tuple(sorted(self._processor.limitations))} diff --git a/modules/common/biometrics/shadow_build.py b/modules/common/biometrics/shadow_build.py new file mode 100644 index 000000000..16d901efd --- /dev/null +++ b/modules/common/biometrics/shadow_build.py @@ -0,0 +1,55 @@ +"""Read-only identity probe for the shipped prospective processor. + +Includes shared replay dependencies and shadow-specific control/transport code. +Wheel metadata hashes are provenance, not full installed-binary attestation. +""" +import json +import importlib.metadata +from pathlib import Path +import sys + +from . import build_identity +from .config import config_hash + +SHADOW_SOURCE_FILES = ( + 'common/biometrics/shadow_build.py', + 'common/biometrics/shadow.py', + 'common/biometrics/shadow_worker.py', + 'common/biometrics/shadow_process.py', +) + + +def shadow_build_identity(): + identity = build_identity.replay_build_identity()['identity'] + identity['version'] = 'shadow-python-build-v1' + root = Path(__file__).resolve().parents[2] + identity['sources'].update({name: build_identity._digest_file(root / name) + for name in SHADOW_SOURCE_FILES}) + dependency_files = {} + for name in ('numpy', 'scipy'): + distribution = importlib.metadata.distribution(name) + files = distribution.files or [] + paths = {} + for key, filename in (('record', 'RECORD'), ('wheel', 'WHEEL')): + matches = [item for item in files if item.name == filename + and item.parent.name.endswith('.dist-info')] + if len(matches) != 1: + raise ValueError('Shadow dependency metadata paths unavailable') + paths[key] = str(Path(distribution.locate_file(matches[0])).absolute()) + dependency_files[name] = paths + return {'hash': config_hash(identity), 'identity': identity, + 'dependencyFiles': dependency_files} + + +def main(): + try: + sys.stdout.write(json.dumps(shadow_build_identity(), allow_nan=False, + separators=(',', ':')) + '\n') + return 0 + except Exception: + sys.stderr.write('Shadow build identity unavailable\n') + return 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/modules/common/biometrics/shadow_process.py b/modules/common/biometrics/shadow_process.py new file mode 100644 index 000000000..324a144cf --- /dev/null +++ b/modules/common/biometrics/shadow_process.py @@ -0,0 +1,106 @@ +"""Linux process entrypoint for the trusted, shipped shadow algorithm. + +Install kernel limits before importing the numerical processor. This is resource +containment, not an isolation boundary for uploaded code. The parent must still +bound wall/idle time, sample RSS, and reap the process on cancellation or loss of +its lease. Address space is not RSS and must be budgeted separately. +""" +import argparse +import os +import signal +import subprocess +import sys + +from .replay_worker import configure_process_limits +from .execution_lock import EXECUTION_LOCK_PATH as SHADOW_LOCK_PATH, acquire_execution_lock + + +def configure_parent_death(expected_parent_pid): + """Kill this worker if its launching host dies, including the arm-time race. + + Linux PR_SET_PDEATHSIG applies to this process, not future forked children: + https://man7.org/linux/man-pages/man2/PR_SET_PDEATHSIG.2const.html + """ + if sys.platform != 'linux': + raise RuntimeError('Shadow parent-death protection requires Linux') + if type(expected_parent_pid) is not int or not 1 <= expected_parent_pid <= 2_147_483_647: + raise ValueError('Invalid shadow parent PID') + if os.getppid() != expected_parent_pid: + raise RuntimeError('Shadow host is no longer the parent') + import ctypes + libc = ctypes.CDLL(None, use_errno=True) + libc.prctl.argtypes = [ctypes.c_int, ctypes.c_ulong, ctypes.c_ulong, ctypes.c_ulong, ctypes.c_ulong] + libc.prctl.restype = ctypes.c_int + if libc.prctl(1, signal.SIGKILL, 0, 0, 0) != 0: # PR_SET_PDEATHSIG + raise OSError(ctypes.get_errno(), 'Shadow parent-death protection unavailable') + # No signal is delivered if the parent died before prctl was armed. + if os.getppid() != expected_parent_pid: + raise RuntimeError('Shadow host changed while arming parent-death protection') + + +def acquire_shadow_lock(path=SHADOW_LOCK_PATH): + if sys.platform != 'linux': + raise RuntimeError('Shadow execution lock requires Linux') + return acquire_execution_lock(path) + + +def configure_wall_deadline(wall_ms): + """Worker-local elapsed-time limit, independent of the Node event loop.""" + if sys.platform != 'linux': + raise RuntimeError('Shadow wall deadline requires Linux') + if type(wall_ms) is not int or not 0 < wall_ms <= 7 * 86_400_000 + 300_000: + raise ValueError('Invalid shadow wall budget') + signal.signal(signal.SIGALRM, signal.SIG_DFL) + signal.pthread_sigmask(signal.SIG_UNBLOCK, {signal.SIGALRM}) + signal.setitimer(signal.ITIMER_REAL, wall_ms / 1000) + + +def configure_shadow_limits(cpu_seconds, address_space_bytes): + if sys.platform != 'linux': + raise RuntimeError('Shadow process limits require Linux') + if type(address_space_bytes) is not int or not 0 < address_space_bytes <= 9_007_199_254_740_991: + raise ValueError('Invalid shadow address-space budget') + # Reuse the replay worker's CPU/core/file limits, low priority, and BLAS + # thread settings. A failure must prevent all numerical imports and input. + configure_process_limits(cpu_seconds) + try: + import resource + except ModuleNotFoundError as error: + if error.name != 'resource': + raise + # Yocto builds can omit the optional Python extension. + subprocess.run(['/usr/bin/prlimit', '--pid', str(os.getpid()), + f'--as={address_space_bytes}:{address_space_bytes}'], + check=True, timeout=5, stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + else: + resource.setrlimit(resource.RLIMIT_AS, (address_space_bytes, address_space_bytes)) + + +def main(argv=None): + parser = argparse.ArgumentParser(description='Run a resource-limited shipped shadow processor') + parser.add_argument('--parent-pid', type=int, required=True) + parser.add_argument('--wall-ms', type=int, required=True) + parser.add_argument('--cpu-seconds', type=int, required=True) + parser.add_argument('--address-space-bytes', type=int, required=True) + args = parser.parse_args(argv) + lock_fd = None + try: + configure_parent_death(args.parent_pid) + lock_fd = acquire_shadow_lock() + configure_wall_deadline(args.wall_ms) + configure_shadow_limits(args.cpu_seconds, args.address_space_bytes) + from .shadow_worker import run + run(sys.stdin.buffer, sys.stdout.buffer) + return 0 + except Exception: + # No frame values, paths, reference data or tracebacks in host logs. + sys.stderr.write('Shadow process failed\n') + return 1 + finally: + if lock_fd is not None: + os.close(lock_fd) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/modules/common/biometrics/shadow_worker.py b/modules/common/biometrics/shadow_worker.py new file mode 100644 index 000000000..aa566ca6f --- /dev/null +++ b/modules/common/biometrics/shadow_worker.py @@ -0,0 +1,132 @@ +"""Bounded JSONL interface for an externally supervised shipped shadow worker. + +stdin carries configuration and frames; stdout carries acknowledgments and +candidate-only readings. No DB, live source, network or hardware is opened. +The parent must impose OS process budgets and terminate idle/cancelled workers. +""" +import base64 +import json +import sys +from uuid import UUID + +from .config import config_hash +from .replay import MAX_CHANNEL_BYTES, RecordedContext +from .shadow import ProspectiveShadow +from .processing import HRV_WINDOW_S + +MAX_LINE_BYTES = 128 * 1024 + + +def positive(value): + if type(value) is not int or not 0 < value <= 9_007_199_254_740_991: + raise ValueError('Invalid transport budget') + return value + + +def exact(value, keys): + if not isinstance(value, dict) or set(value) != set(keys.split()): + raise ValueError('Unexpected shadow protocol fields') + + +def run(source, destination): + input_bytes = output_bytes = 0 + maximum_input = MAX_LINE_BYTES + maximum_output = MAX_LINE_BYTES + + def read(): + nonlocal input_bytes + line = source.readline(MAX_LINE_BYTES + 1) + if not line: + raise ValueError('Unexpected shadow protocol EOF') + input_bytes += len(line) + if len(line) > MAX_LINE_BYTES or input_bytes > maximum_input or not line.endswith(b'\n'): + raise ValueError('Shadow input transport limit exceeded') + return json.loads(line.decode('utf-8')) + + init = read() + exact(init, 'schema runId attemptId configs configHashes side startMs endMs maximumRecords maximumOutputs maximumCpuMs maximumInputBytes maximumOutputBytes') + if init['schema'] != 'biometric-shadow-worker-v1': + raise ValueError('Unsupported shadow worker protocol') + for key in ('runId', 'attemptId'): + if not isinstance(init[key], str) or str(UUID(init[key])) != init[key]: + raise ValueError('Invalid shadow attempt identity') + maximum_input, maximum_output = positive(init['maximumInputBytes']), positive(init['maximumOutputBytes']) + if input_bytes > maximum_input: + raise ValueError('Shadow input transport limit exceeded') + if not isinstance(init['configs'], dict) or set(init['configs']) != {'left', 'right'}: + raise ValueError('Both shadow configurations required') + hashes = {side: config_hash(config) for side, config in init['configs'].items()} + if hashes != init['configHashes']: + raise ValueError('Shadow configuration identity mismatch') + + def write(payload): + nonlocal output_bytes + response = {'runId': init['runId'], 'attemptId': init['attemptId'], **payload} + line = (json.dumps(response, allow_nan=False, separators=(',', ':')) + '\n').encode() + output_bytes += len(line) + if len(line) > MAX_LINE_BYTES or output_bytes > maximum_output: + raise ValueError('Shadow output transport limit exceeded') + destination.write(line) + destination.flush() + + pending = [] + def sink(output): + candidate = output.reading.candidate + pending.append({'sequence': output.sequence, 'inputHash': output.input_hash, + 'side': output.reading.side, 'sourceSide': candidate.run_context.source_side, + 'timestampMs': output.reading.timestamp_ms, 'configHash': candidate.run_context.config_hash, + 'readings': [ + {'metric': 'heart_rate', 'unit': 'bpm', 'windowMs': None, 'value': candidate.heart_rate, 'quality': candidate.quality_score}, + {'metric': 'breathing_rate', 'unit': 'breaths/min', 'windowMs': None, 'value': candidate.breathing_rate, 'quality': candidate.quality_score}, + {'metric': 'window_ibi_variability_v1', 'unit': 'ms', 'windowMs': HRV_WINDOW_S * 1000, 'value': candidate.hrv, 'quality': candidate.quality_score}, + ], 'flags': candidate.flags, 'limitations': output.reading.limitations}) + + processor = ProspectiveShadow(init['configs'], side=init['side'], start_ms=init['startMs'], end_ms=init['endMs'], + maximum_records=init['maximumRecords'], maximum_outputs=init['maximumOutputs'], maximum_cpu_ms=init['maximumCpuMs'], sink=sink) + write({'type': 'ready', 'configHashes': hashes}) + expected_sequence = 0 + try: + while True: + message = read() + if message == {'type': 'finish'}: + processor.finish() + write({'type': 'finished', 'outputs': pending, 'health': processor.health()}) + return + if message == {'type': 'cancel'}: + processor.stop() + write({'type': 'cancelled', 'health': processor.health()}) + return + exact(message, 'type sequence observedAtMs record context') + if message['type'] != 'frame' or type(message['sequence']) is not int or message['sequence'] != expected_sequence: + raise ValueError('Out-of-order shadow protocol frame') + exact(message['context'], 'activeSides leftOccupied rightOccupied pumpContextKnown') + context = message['context'] + if context['activeSides'] is not None and not isinstance(context['activeSides'], list): + raise ValueError('Invalid shadow active sides') + recorded = RecordedContext(tuple(context['activeSides']) if context['activeSides'] is not None else None, + context['leftOccupied'], context['rightOccupied'], context['pumpContextKnown']) + if not isinstance(message['record'], dict): + raise ValueError('Invalid shadow record') + record = dict(message['record']) + for channel in ('left1', 'right1'): + if channel in record: + encoded = record[channel] + if not isinstance(encoded, str) or len(encoded) > ((MAX_CHANNEL_BYTES + 2) // 3) * 4: + raise ValueError('Invalid encoded shadow channel') + record[channel] = base64.b64decode(encoded, validate=True) + if len(record[channel]) > MAX_CHANNEL_BYTES: + raise ValueError('Shadow channel limit exceeded') + processor.feed(message['observedAtMs'], record, recorded) + write({'type': 'frame_result', 'sequence': expected_sequence, 'outputs': pending, 'health': processor.health()}) + pending.clear() + expected_sequence += 1 + finally: + processor.stop('transport_closed') + + +if __name__ == '__main__': + try: + run(sys.stdin.buffer, sys.stdout.buffer) + except Exception as error: + sys.stderr.write(str(error)[:500] + '\n') + sys.exit(1) diff --git a/modules/common/biometrics/signal.py b/modules/common/biometrics/signal.py new file mode 100644 index 000000000..dface3b08 --- /dev/null +++ b/modules/common/biometrics/signal.py @@ -0,0 +1,498 @@ +"""Shared numerical core for live processing and deterministic replay. + +No database, sensor transport, hardware or wall-clock access. Callers own +occupancy, pump gating, window boundaries, state reset, timestamps and sinks. +Defaults were captured against dev 3e9027ed before extraction. +""" +from collections import deque +from dataclasses import dataclass +import logging +from typing import Optional + +import numpy as np +from scipy.signal import butter, sosfiltfilt, hilbert, find_peaks + +from .config import manifest, resolve_config, validate_config + +SAMPLE_RATE = 500 +HR_BAND = (0.8, 8.5) +log = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class ProcessingParameters: + tracking_tolerance_bpm: float + tracking_history_length: int + breathing_interval_lower_ratio: float + breathing_interval_upper_ratio: float + hrv_harmonic_tolerance: float + hrv_minimum_windows: int + + def __post_init__(self): + resolve_config([ + {"path": "heart_rate.trackingToleranceBpm", "value": self.tracking_tolerance_bpm}, + {"path": "heart_rate.trackingHistoryLength", "value": self.tracking_history_length}, + {"path": "breathing_rate.intervalLowerRatio", "value": self.breathing_interval_lower_ratio}, + {"path": "breathing_rate.intervalUpperRatio", "value": self.breathing_interval_upper_ratio}, + {"path": "hrv_index.harmonicTolerance", "value": self.hrv_harmonic_tolerance}, + {"path": "hrv_index.minimumWindows", "value": self.hrv_minimum_windows}, + ]) + + @classmethod + def from_config(cls, config): + validate_config(config) + return cls(config["heart_rate"]["trackingToleranceBpm"], + config["heart_rate"]["trackingHistoryLength"], + config["breathing_rate"]["intervalLowerRatio"], + config["breathing_rate"]["intervalUpperRatio"], + config["hrv_index"]["harmonicTolerance"], + config["hrv_index"]["minimumWindows"]) + + +DEFAULT_PARAMETERS = ProcessingParameters.from_config(manifest()["defaults"]) + + +def _bandpass(sig: np.ndarray, lo: float, hi: float, fs: float, + order: int = 4) -> np.ndarray: + """Bandpass filter using SOS form for numerical stability.""" + sos = butter(order, [lo, hi], btype='band', fs=fs, output='sos') + return sosfiltfilt(sos, sig) + + + +def _autocorr_quality(sig: np.ndarray, fs: float = SAMPLE_RATE) -> float: + """Peak height of best autocorrelation peak in cardiac range. + + Person on bed produces periodic signal with high peak. + Empty bed produces noise with low peak. + """ + try: + filtered = _bandpass(sig, 0.8, 8.5, fs) + seg = filtered[:int(min(len(filtered), 10 * fs))] + seg = seg - np.mean(seg) + norm = np.std(seg) + if norm < 1e-10: + return 0.0 + seg = seg / norm + + n = len(seg) + fft_size = 1 + while fft_size < 2 * n: + fft_size *= 2 + fft_f = np.fft.rfft(seg, n=fft_size) + acr = np.fft.irfft(fft_f * np.conj(fft_f), n=fft_size)[:n] + if acr[0] == 0: + return 0.0 + acr = acr / acr[0] + + min_lag = int(fs * 60 / 150) + max_lag = min(int(fs * 60 / 40), len(acr) - 1) + if max_lag <= min_lag: + return 0.0 + return float(max(np.max(acr[min_lag:max_lag + 1]), 0.0)) + except Exception: + return 0.0 + + + + +def _compute_autocorr(filtered: np.ndarray, fs: float) -> Optional[np.ndarray]: + """Compute normalised autocorrelation via FFT.""" + n = len(filtered) + fft_size = 1 + while fft_size < 2 * n: + fft_size *= 2 + fft_f = np.fft.rfft(filtered, n=fft_size) + acr = np.fft.irfft(fft_f * np.conj(fft_f), n=fft_size)[:n] + if acr[0] == 0: + return None + return acr / acr[0] + + +def subharmonic_summation_hr(samples: np.ndarray, fs: float = SAMPLE_RATE, + bpm_range: tuple = (45, 120), + n_harmonics: int = 3): + """Subharmonic summation for robust fundamental HR detection. + + Only scores autocorrelation PEAKS (not every lag) to avoid edge effects + where long lags accumulate noise. For each candidate peak lag L, + score = weighted sum of autocorr at L, L/2, L/3. + + Hermes 1988 (JASA), adapted for BCG by Bruser et al. 2011. + + Returns (heart_rate_bpm, score) or (None, 0.0) on failure. + """ + filtered = _bandpass(samples, HR_BAND[0], HR_BAND[1], fs) + acr = _compute_autocorr(filtered, fs) + if acr is None: + return None, 0.0 + + min_lag = int(fs * 60 / bpm_range[1]) + max_lag = int(fs * 60 / bpm_range[0]) + max_lag = min(max_lag, len(acr) - 1) + + if max_lag <= min_lag: + return None, 0.0 + + # Find actual peaks in autocorrelation (not every lag) + search = acr[min_lag:max_lag + 1] + peaks, _props = find_peaks(search, height=0.02, distance=int(fs * 0.15)) + if len(peaks) == 0: + return None, 0.0 + + candidate_lags = peaks + min_lag + weights = [1.0, 0.8, 0.6][:n_harmonics] + + best_lag = None + best_score = 0.0 + + for lag in candidate_lags: + score = 0.0 + for k in range(n_harmonics): + exact_lag = lag / (k + 1) + lag_lo = int(exact_lag) + lag_hi = lag_lo + 1 + if lag_hi >= len(acr): + continue + frac = exact_lag - lag_lo + val = acr[lag_lo] * (1 - frac) + acr[lag_hi] * frac + score += weights[k] * max(val, 0) + if score > best_score: + best_score = score + best_lag = lag + + if best_lag is None or best_score < 0.1: + return None, best_score + + hr = 60.0 * fs / best_lag + return hr, best_score + + +class HRTracker: + """Inter-window HR tracking with Gaussian consistency constraint. + + HR cannot jump > max_delta BPM between consecutive windows at rest. + Catches residual harmonic escapes via half/double correction. + Bruser et al. 2011. + """ + + # After this many consecutive uncorrectable readings, re-seed history at + # the new level. A lone outlier must not poison the median, but a + # sustained shift (e.g. different occupant) previously froze the tracker: + # the stale median rejected every new reading forever. + FALLBACK_RESET_STREAK = 3 + + def __init__(self, max_delta: float = DEFAULT_PARAMETERS.tracking_tolerance_bpm, + history_len: int = DEFAULT_PARAMETERS.tracking_history_length): + # Bounded deque: only the last history_len entries are ever consulted, + # so retaining the full list would leak ~1440 floats/day (#325). + self.history: deque = deque(maxlen=history_len) + self.max_delta = max_delta + self.history_len = history_len + self._fallback_streak = 0 + + def update(self, hr_candidate: Optional[float], + score: float) -> Optional[float]: + if hr_candidate is None: + return None + + if not self.history: + self.history.append(hr_candidate) + return hr_candidate + + recent = float(np.median(list(self.history))) + delta = abs(hr_candidate - recent) + + # Gaussian consistency weight + consistency = np.exp(-0.5 * (delta / self.max_delta) ** 2) + if consistency > 0.3: + self._fallback_streak = 0 + self.history.append(hr_candidate) + return hr_candidate + + # Try half (harmonic correction) + half_hr = hr_candidate / 2 + if 40 <= half_hr <= 120: + half_delta = abs(half_hr - recent) + half_consistency = np.exp(-0.5 * (half_delta / self.max_delta) ** 2) + if half_consistency > 0.3: + self._fallback_streak = 0 + self.history.append(half_hr) + return half_hr + + # Try double (sub-harmonic correction) + double_hr = hr_candidate * 2 + if 40 <= double_hr <= 180: + double_delta = abs(double_hr - recent) + double_consistency = np.exp( + -0.5 * (double_delta / self.max_delta) ** 2) + if double_consistency > 0.3: + self._fallback_streak = 0 + self.history.append(double_hr) + return double_hr + + # Accept anyway but keep a lone outlier out of history. A sustained + # streak of uncorrectable readings means the tracked level itself is + # stale (e.g. different occupant) — previously the frozen median then + # rejected every reading forever, so re-seed at the new level. + self._fallback_streak += 1 + if self._fallback_streak >= self.FALLBACK_RESET_STREAK: + self.history.clear() + self.history.append(hr_candidate) + self._fallback_streak = 0 + return hr_candidate + +# --------------------------------------------------------------------------- +# Breathing rate — Hilbert envelope (PMC9354426) +# --------------------------------------------------------------------------- + +def compute_breathing_rate(samples: np.ndarray, + fs: float = SAMPLE_RATE, + parameters: ProcessingParameters = DEFAULT_PARAMETERS) -> Optional[float]: + """Breathing rate via Hilbert envelope of the cardiac band. + + Pipeline: bandpass 0.8-10 Hz -> Hilbert transform -> amplitude envelope + -> bandpass 0.1-0.7 Hz -> peak counting. + """ + try: + cardiac = _bandpass(samples, 0.8, 10.0, fs) + envelope = np.abs(hilbert(cardiac)) + resp = _bandpass(envelope, 0.1, 0.7, fs) + peaks, _ = find_peaks(resp, distance=int(2.0 * fs)) + if len(peaks) < 3: + return None + intervals = np.diff(peaks) / fs + med = np.median(intervals) + valid = intervals[(intervals > med * parameters.breathing_interval_lower_ratio) + & (intervals < med * parameters.breathing_interval_upper_ratio)] + if len(valid) < 2: + return None + br = 60.0 / float(np.mean(valid)) + return br if 6 <= br <= 30 else None + except Exception as e: + log.debug("Breathing rate computation failed: %s", e) + return None + +# --------------------------------------------------------------------------- +# HR variability index — window-level IBI with harmonic gate +# +# NOTE: This is NOT clinical RMSSD. It computes successive differences of +# window-level IBI estimates (~59 per 5 min with 10s windows), not beat-to-beat +# intervals (~300+ per 5 min). The output measures HR stability across windows, +# not vagal beat-to-beat modulation. See #221 for the beat-level upgrade path. +# --------------------------------------------------------------------------- + +# Harmonic tolerance for IBI gating (18% per DSP expert recommendation) +_HRV_HARMONIC_TOL = DEFAULT_PARAMETERS.hrv_harmonic_tolerance +# Sub-window length for IBI estimation (10s per DSP expert recommendation — +# gives ~59 windows per 5 min vs ~19 with 30s windows) +_HRV_WINDOW_S = 10 +# Minimum accepted IBI estimates after filtering +_HRV_MIN_IBIS = DEFAULT_PARAMETERS.hrv_minimum_windows +# Maximum gap between consecutive accepted windows before skipping the diff +_HRV_MAX_GAP_WINDOWS = 3 + + +def _harmonic_gate(ibi_ms: float, tracker_ibi_ms: float, + tolerance: float = DEFAULT_PARAMETERS.hrv_harmonic_tolerance) -> Optional[float]: + """Reject or correct IBIs that are harmonic multiples of the expected value. + + Returns the accepted/corrected IBI in ms, or None to discard. + """ + if tracker_ibi_ms <= 0: + return ibi_ms # no prior yet — accept as-is + + for multiplier in [1.0, 2.0, 0.5, 3.0, 1.0 / 3.0]: + candidate = tracker_ibi_ms * multiplier + if candidate > 0 and abs(ibi_ms - candidate) / candidate < tolerance: + if multiplier == 1.0: + return ibi_ms # fundamental — accept + else: + return tracker_ibi_ms # harmonic — substitute tracker value + return None # outside all harmonic relationships — discard + + +def compute_hrv(samples: np.ndarray, + fs: float = SAMPLE_RATE, + parameters: ProcessingParameters = DEFAULT_PARAMETERS) -> Optional[float]: + """HR variability index via window-level IBI with harmonic gate. + + Pipeline: + 1. Bandpass 0.8-8.5 Hz + 2. 10s sub-windows with 50% overlap (~59 windows per 5 min) + 3. SHS autocorrelation per window → raw IBI + 4. Harmonic gate: reject/correct IBIs using trimmed-mean tracker + 5. Hampel filter on accepted IBI series + 6. Gap-aware RMSSD (skip diffs across rejected-window gaps) + 7. Range gate: 5-100 ms + + NOT clinical RMSSD — see module docstring and #221. + """ + try: + filtered = _bandpass(samples, HR_BAND[0], HR_BAND[1], fs) + sub_window = int(_HRV_WINDOW_S * fs) + step = sub_window // 2 + min_lag = int(fs * 60 / 150) + max_lag = int(fs * 60 / 40) + + # --- Pass 1: SHS autocorrelation per window → raw IBIs --- + raw_ibis: list = [] # (window_index, ibi_ms) + for idx, start in enumerate( + range(0, len(filtered) - sub_window + 1, step)): + chunk = filtered[start:start + sub_window] + acr = _compute_autocorr(chunk, fs) + if acr is None: + continue + search = acr[min_lag:min(max_lag + 1, len(acr))] + if len(search) == 0: + continue + + # SHS scoring — find peaks first, then score + peaks, _props = find_peaks(search, height=0.02, + distance=int(fs * 0.15)) + if len(peaks) == 0: + continue + + candidate_lags = peaks + min_lag + weights = [1.0, 0.8, 0.6] + best_lag, best_score = None, 0.0 + for lag in candidate_lags: + score = 0.0 + for k in range(3): + exact = lag / (k + 1) + lo_i = int(exact) + hi_i = lo_i + 1 + if hi_i >= len(acr): + continue + frac = exact - lo_i + val = acr[lo_i] * (1 - frac) + acr[hi_i] * frac + score += weights[k] * max(val, 0) + if score > best_score: + best_score = score + best_lag = lag + + if best_lag is None or best_score < 0.1: + continue + ibi_ms = best_lag / fs * 1000 + if 400 <= ibi_ms <= 1500: + raw_ibis.append((idx, ibi_ms)) + + if len(raw_ibis) < parameters.hrv_minimum_windows: + return None + + # --- Pass 2: Harmonic gate with trimmed-mean tracker --- + tracker_history: list = [] + gated: list = [] # (window_index, ibi_ms) + + for win_idx, ibi_ms in raw_ibis: + # Compute tracker IBI as trimmed mean of last 5 accepted + if len(tracker_history) >= 3: + sorted_h = sorted(tracker_history[-5:]) + # Drop min+max if we have enough + if len(sorted_h) >= 4: + trimmed = sorted_h[1:-1] + else: + trimmed = sorted_h + tracker_ibi = sum(trimmed) / len(trimmed) + elif tracker_history: + tracker_ibi = sum(tracker_history) / len(tracker_history) + else: + tracker_ibi = 0 # no prior — accept first value + + accepted = _harmonic_gate(ibi_ms, tracker_ibi, parameters.hrv_harmonic_tolerance) + if accepted is not None: + gated.append((win_idx, accepted)) + tracker_history.append(accepted) + + if len(gated) < parameters.hrv_minimum_windows: + return None + + # --- Pass 3: Hampel filter on gated IBI series --- + ibi_values = np.array([ibi for _, ibi in gated]) + win_indices = [idx for idx, _ in gated] + clean_mask = np.ones(len(ibi_values), dtype=bool) + + for i in range(len(ibi_values)): + lo_i = max(0, i - 3) + hi_i = min(len(ibi_values), i + 4) + local = ibi_values[lo_i:hi_i] + med = np.median(local) + mad = max(np.median(np.abs(local - med)), 1e-6) + if abs(ibi_values[i] - med) > 3.0 * 1.4826 * mad: + clean_mask[i] = False + + clean_ibis = ibi_values[clean_mask] + clean_indices = [win_indices[i] for i in range(len(win_indices)) + if clean_mask[i]] + + if len(clean_ibis) < parameters.hrv_minimum_windows: + return None + + # --- Pass 4: Gap-aware RMSSD --- + sq_diffs: list = [] + for i in range(1, len(clean_ibis)): + gap = clean_indices[i] - clean_indices[i - 1] + if gap >= _HRV_MAX_GAP_WINDOWS: + continue # skip diff across rejected-window gap + diff = clean_ibis[i] - clean_ibis[i - 1] + sq_diffs.append(diff ** 2) + + if len(sq_diffs) < 5: + return None + + hrv_index = float(np.sqrt(np.mean(sq_diffs))) + # Range gate: 5-100 ms. Window-level HRV above 100 ms is artifact + # even after harmonic correction (cardiologist recommendation). + return hrv_index if 5 <= hrv_index <= 100 else None + except Exception as e: + log.debug("HRV computation failed: %s", e) + return None + + +@dataclass(frozen=True) +class WindowEstimate: + timestamp_s: float + heart_rate: Optional[float] + hrv_index: Optional[float] + breathing_rate: Optional[float] + hr_raw: Optional[float] + hr_score: float + rejection_reasons: tuple + + +class WindowEstimator: + """Stateful numerical estimator used by live and historical adapters. + + Inputs are already gated, timestamped windows, in their processing order. + This layer cannot infer missing occupancy/pump/channel context and must not + be advertised as a full raw replay by itself. One instance per side and + configuration boundary prevents tracker history crossing profiles. + """ + def __init__(self, parameters: ProcessingParameters = DEFAULT_PARAMETERS): + self.parameters = parameters + self.hr_tracker = HRTracker(parameters.tracking_tolerance_bpm, + parameters.tracking_history_length) + + def estimate(self, timestamp_s: float, hr_samples: np.ndarray, + breathing_samples: np.ndarray, hrv_samples: np.ndarray, + fs: float = SAMPLE_RATE) -> WindowEstimate: + if not np.isfinite(timestamp_s): + raise ValueError("Expected a finite event timestamp") + if fs != SAMPLE_RATE: + raise ValueError("PROFILE_INCOMPATIBLE: this engine requires 500 Hz") + # Match live warmup exactly; never pad unavailable history with zeroes. + if len(hr_samples) < int(10 * fs): + return WindowEstimate(timestamp_s, None, None, None, None, 0.0, ("hr_warmup",)) + hr_raw, hr_score = subharmonic_summation_hr(hr_samples, fs) + hr = self.hr_tracker.update(hr_raw, hr_score) + br = compute_breathing_rate(breathing_samples, fs, self.parameters) + hrv = (compute_hrv(hrv_samples, fs, self.parameters) + if len(hrv_samples) >= int(300 * fs) else None) + reasons = [] + if hr is None: + reasons.append("no_hr") + if br is None: + reasons.append("no_br") + if hrv is None: + reasons.append("hrv_warmup" if len(hrv_samples) < int(300 * fs) else "no_hrv") + return WindowEstimate(timestamp_s, hr, hrv, br, hr_raw, hr_score, tuple(reasons)) diff --git a/modules/common/test_bed_presence.py b/modules/common/test_bed_presence.py index ece4c2165..af4a9bced 100644 --- a/modules/common/test_bed_presence.py +++ b/modules/common/test_bed_presence.py @@ -1,17 +1,20 @@ import json import os -import time import pytest from common.bed_presence import BED_PRESENCE_STALE_S, BedPresence def _write(path, left=True, right=False, version=1): + # Match evidence and file time exactly; filesystem timestamp precision can + # otherwise make fresh evidence appear to be in the future on CI. + stamp = 1_700_000_000 path.write_text(json.dumps({ "version": version, - "left": {"debounced_present": left, "vitals_presence": left, "vitals_evidence_ts": time.time(), "session_start": None}, - "right": {"debounced_present": right, "vitals_presence": right, "vitals_evidence_ts": time.time(), "session_start": None}, + "left": {"debounced_present": left, "vitals_presence": left, "vitals_evidence_ts": stamp, "session_start": None}, + "right": {"debounced_present": right, "vitals_presence": right, "vitals_evidence_ts": stamp, "session_start": None}, })) + os.utime(path, (stamp, stamp)) class Clock: diff --git a/modules/common/test_biometric_config.py b/modules/common/test_biometric_config.py new file mode 100644 index 000000000..9b678453c --- /dev/null +++ b/modules/common/test_biometric_config.py @@ -0,0 +1,66 @@ +"""Contract tests run with common's stdlib-only test environment.""" +import math + +import pytest + +from common.biometrics.config import canonical_json, config_hash, manifest, resolve_config, validate_config + + +def test_defaults_do_not_mutate_the_shipped_manifest(): + profile = resolve_config([]) + profile["resolved"]["heart_rate"]["trackingToleranceBpm"] = 25 + assert resolve_config([])["resolved"]["heart_rate"]["trackingToleranceBpm"] == 15 + assert manifest()["components"]["hrv_index"]["metricId"] == "window_ibi_variability_v1" + assert manifest()["components"]["sleep_stages"]["causal"] is False + + +def test_false_zero_and_rule_order_are_preserved(): + resolved = resolve_config([ + {"path": "sleep_stages.ruleset.smoothIsolatedEpochs", "value": False}, + {"path": "sleep_stages.ruleset.lowSignalBelow", "value": 0}]) + assert resolved["resolved"]["sleep_stages"]["ruleset"]["lowSignalBelow"] == 0 + assert resolved["resolved"]["sleep_stages"]["ruleset"]["smoothIsolatedEpochs"] is False + rules = [{"stage": "wake", "when": []}, {"stage": "light", "when": []}] + forward = resolve_config([{"path": "sleep_stages.ruleset.rules", "value": rules}]) + reverse = resolve_config([{"path": "sleep_stages.ruleset.rules", "value": rules[::-1]}]) + assert forward["hash"] != reverse["hash"] + + +@pytest.mark.parametrize("path,value", [ + ("heart_rate.trackingToleranceBpm", True), + ("heart_rate.trackingToleranceBpm", 0), + ("heart_rate.trackingHistoryLength", 1.5), + ("hrv_index.harmonicTolerance", math.nan), + ("sleep_stages.ruleset.smoothIsolatedEpochs", 0), + ("sleep_stages.ruleset.fallback", None), + ("sleep_stages.ruleset.outliers.heartRate.min", 150), + ("sleep_stages.ruleset.rules.0.stage", "wake"), + ("sleep_stages.ruleset.rules", [{"stage": "unknown", "when": []}]), + ("sleep_stages.ruleset", {}), + ("sensor_calibration.noise_floor", 0), +]) +def test_invalid_overrides_are_rejected(path, value): + with pytest.raises(ValueError): + resolve_config([{"path": path, "value": value}]) + + +def test_unknown_fields_and_bases_are_rejected(): + with pytest.raises(ValueError, match="PROFILE_INCOMPATIBLE"): + resolve_config([], "unknown") + with pytest.raises(ValueError): + resolve_config([{"path": "heart_rate.trackingToleranceBpm", "value": 15, "ignored": True}]) + value = manifest()["defaults"] + value["calibration"] = {} + with pytest.raises(ValueError): + validate_config(value) + with pytest.raises(ValueError, match="Duplicate"): + resolve_config([{"path": "heart_rate.trackingToleranceBpm", "value": 15}] * 2) + + +def test_canonical_numeric_json(): + assert canonical_json({"b": -0.0, "a": .5}) == '{"a":0.5,"b":0}' + assert canonical_json(.1) == "0.1000000000000000055511151231257827021181583404541015625" + assert config_hash({"a": 1, "b": 2}) == config_hash({"b": 2, "a": 1}) + for bad in [math.inf, math.nan, {1: "key"}, "\ud800", object(), [None] * 20001]: + with pytest.raises(ValueError): + canonical_json(bad) diff --git a/modules/common/test_biometric_routing.py b/modules/common/test_biometric_routing.py new file mode 100644 index 000000000..252bb1cb7 --- /dev/null +++ b/modules/common/test_biometric_routing.py @@ -0,0 +1,96 @@ +"""Routing parity with an experiment-only sink and recorded mode/time.""" +from dataclasses import dataclass + +from common.biometrics.routing import SingleSleeperRouter + + +@dataclass(frozen=True) +class Reading: + side: str + quality_score: float + sequence: int + + +class Mode: + home = 'left' + active = ('left',) + + def home_side(self): + return self.home + + def active_sides(self): + return self.active + + +def setup_router(): + now = [0.0] + mode = Mode() + output = [] + accepted = [True] + + def write(candidate, destination): + if not accepted[0]: + return False + output.append((candidate, destination)) + return True + + router = SingleSleeperRouter(mode, write, clock=lambda: now[0]) + return router, now, mode, output, accepted + + +def test_pairing_prefers_quality_and_preserves_first_candidate_on_ties(): + router, _, _, output, _ = setup_router() + left, right = Reading('left', 0.3, 1), Reading('right', 0.8, 1) + assert router.submit(left) + assert not output + assert router.submit(right) + assert output == [(right, 'left')] + tied_left, tied_right = Reading('left', 0.8, 2), Reading('right', 0.8, 2) + assert router.submit(tied_left) + assert router.submit(tied_right) + assert output[-1] == (tied_left, 'left') + + +def test_recorded_clock_expires_unpaired_reading_at_exact_cadence(): + router, now, _, output, _ = setup_router() + reading = Reading('right', 0.8, 1) + assert router.submit(reading) + now[0] = 59.999 + router.tick() + assert not output + now[0] = 60.0 + router.tick() + assert output == [(reading, 'left')] + router.tick() + assert len(output) == 1 + + +def test_failed_pair_cannot_be_overwritten_or_relabelled_after_mode_change(): + router, _, mode, output, accepted = setup_router() + winner = Reading('right', 0.8, 1) + router.submit(Reading('left', 0.3, 1)) + accepted[0] = False + assert router.submit(winner) + mode.home, mode.active = 'right', ('right',) + later = Reading('left', 0.9, 2) + assert not router.submit(later) + assert not output + accepted[0] = True + assert router.submit(later) + assert output == [(winner, 'left')] + assert router.flush() + assert output[-1] == (later, 'right') + + +def test_two_sleepers_and_empty_bed_use_recorded_mode_only(): + router, now, mode, output, _ = setup_router() + router.submit(Reading('left', 0.5, 1)) + mode.active = () + now[0] = 60 + router.tick() + assert router.flush() + assert not output + mode.home, mode.active = None, ('left', 'right') + reading = Reading('right', 0.4, 2) + assert router.submit(reading) + assert output == [(reading, None)] diff --git a/modules/common/test_biometric_runtime.py b/modules/common/test_biometric_runtime.py new file mode 100644 index 000000000..1227be18f --- /dev/null +++ b/modules/common/test_biometric_runtime.py @@ -0,0 +1,106 @@ +"""Runtime delivery integrity and boundary regression tests.""" +import json +from pathlib import Path +import sqlite3 + +import pytest + +from common.biometrics.config import resolve_config +from common.biometrics.runtime import BoundarySelection, RuntimeSnapshot, read_latest_snapshot + + +def payload(generation=1, boundary="next_session", **changes): + config = resolve_config([]) + value = dict(schemaVersion=1, side="left", generation=generation, + revisionId=None, manifestHash=config["manifestHash"], + resolved=config["resolved"], resolvedHash=config["hash"], boundary=boundary) + value.update(changes) + return json.dumps(value) + + +def test_snapshot_is_deeply_immutable(): + snapshot = RuntimeSnapshot.parse(payload()) + snapshot.resolved["heart_rate"]["trackingToleranceBpm"] = 25 + assert snapshot.resolved["heart_rate"]["trackingToleranceBpm"] == 15 + + +@pytest.mark.parametrize("changes", [ + {"generation": True}, {"generation": 0}, {"generation": 1.5}, + {"schemaVersion": True}, {"revisionId": "bad"}, {"side": "both"}, + {"manifestHash": "old-engine"}, {"resolvedHash": "forged"}, + {"boundary": "whenever"}, {"extra": 1}, +]) +def test_invalid_projection_is_rejected(changes): + with pytest.raises(ValueError): + RuntimeSnapshot.parse(payload(**changes)) + + +def test_duplicate_fields_and_oversized_payload_are_rejected(): + with pytest.raises(ValueError, match="Duplicate"): + RuntimeSnapshot.parse(payload()[:-1] + ', "generation": 2}') + with pytest.raises(ValueError, match="resource limit"): + RuntimeSnapshot.parse(" " * 65537) + + +def test_actual_migration_projection_envelope_and_latest_generation(): + connection = sqlite3.connect(":memory:") + connection.execute("CREATE TABLE vitals (id INTEGER)") + migration = Path(__file__).parents[2] / "src/db/biometrics-migrations/0020_black_patch.sql" + connection.executescript(migration.read_text()) + assert read_latest_snapshot(connection, "left") is None + snapshot = RuntimeSnapshot.parse(payload()) + connection.execute("INSERT INTO biometric_runtime_configs VALUES (?, ?, ?, ?, ?, ?)", + ("left", 1, snapshot.resolved_hash, payload(), "next_session", 0)) + assert read_latest_snapshot(connection, "left") == snapshot + connection.execute("INSERT INTO biometric_runtime_configs VALUES (?, ?, ?, ?, ?, ?)", + ("left", 2, snapshot.resolved_hash, payload(2), "immediate", 1)) + with pytest.raises(ValueError, match="envelope mismatch"): + read_latest_snapshot(connection, "left") + connection.execute("UPDATE biometric_runtime_configs SET payload = ? WHERE generation = 2", (" " * 65537,)) + with pytest.raises(ValueError, match="resource limit"): + read_latest_snapshot(connection, "left") + + +def test_missing_schema_is_not_silently_created(): + connection = sqlite3.connect(":memory:") + with pytest.raises(sqlite3.OperationalError): + read_latest_snapshot(connection, "left") + assert connection.execute("SELECT name FROM sqlite_master").fetchall() == [] + + +def test_next_session_waits_through_restart_and_unknown_occupancy(): + selection = BoundarySelection("left") + snapshot = RuntimeSnapshot.parse(payload()) + selection.offer(snapshot) + assert selection.ready(True) is None + assert selection.ready(None) is None + assert selection.ready(False) == snapshot + assert selection.applied is None # selection alone does not acknowledge + selection.commit(snapshot) + assert selection.applied == snapshot + selection.offer(snapshot) + assert selection.pending is None + + +def test_newer_request_supersedes_pending_and_fences_late_commit(): + selection = BoundarySelection("left") + first = RuntimeSnapshot.parse(payload()) + second = RuntimeSnapshot.parse(payload(2, "immediate")) + selection.offer(first) + selection.offer(second) + assert selection.ready(True) == second + with pytest.raises(ValueError, match="changed"): + selection.commit(first) + with pytest.raises(ValueError, match="regression"): + selection.offer(first) + with pytest.raises(ValueError, match="conflict"): + selection.offer(RuntimeSnapshot.parse(payload(2))) + selection.commit(second) + assert selection.applied == second + + +def test_side_mismatch_never_changes_selection(): + selection = BoundarySelection("right") + with pytest.raises(ValueError, match="side mismatch"): + selection.offer(RuntimeSnapshot.parse(payload())) + assert selection.pending is None diff --git a/modules/common/test_build_identity.py b/modules/common/test_build_identity.py new file mode 100644 index 000000000..6cafee1e3 --- /dev/null +++ b/modules/common/test_build_identity.py @@ -0,0 +1,57 @@ +import hashlib +import importlib.metadata +from pathlib import Path + +import pytest + +from common.biometrics import build_identity +from common.biometrics.config import config_hash + + +def test_identity_is_deterministic_and_contains_sources_and_installed_versions(): + first = build_identity.replay_build_identity() + assert first == build_identity.replay_build_identity() + assert first['hash'] == config_hash(first['identity']) + assert set(first['identity']['sources']) == set(build_identity.SOURCE_FILES) + assert 'common/biometrics/build_identity.py' in first['identity']['sources'] + assert first['identity']['dependencies']['numpy']['version'] == importlib.metadata.version('numpy') + assert first['identity']['dependencies']['scipy']['version'] == importlib.metadata.version('scipy') + root = Path(build_identity.__file__).resolve().parents[2] + assert first['identity']['sources']['piezo-processor/uv.lock'] == hashlib.sha256((root / 'piezo-processor/uv.lock').read_bytes()).hexdigest() + + +def test_changed_sources_and_dependency_metadata_change_identity(monkeypatch): + before = build_identity.replay_build_identity() + original = build_identity._digest_file + monkeypatch.setattr(build_identity, '_digest_file', lambda path: '0' * 64 if path.name == 'signal.py' else original(path)) + assert build_identity.replay_build_identity()['hash'] != before['hash'] + monkeypatch.setattr(build_identity, '_digest_file', original) + distribution = importlib.metadata.distribution + + class Changed: + version = 'changed-build' + + def read_text(self, name): + return name + '-changed' + + monkeypatch.setattr(importlib.metadata, 'distribution', lambda name: Changed() if name == 'numpy' else distribution(name)) + assert build_identity.replay_build_identity()['hash'] != before['hash'] + + +def test_missing_wheel_metadata_and_oversize_sources_fail_closed(tmp_path, monkeypatch): + source = tmp_path / 'source.py' + source.write_bytes(b'12345') + monkeypatch.setattr(build_identity, 'MAX_FILE_BYTES', 4) + with pytest.raises(ValueError, match='bound'): + build_identity._digest_file(source) + monkeypatch.setattr(build_identity, 'MAX_FILE_BYTES', 16 * 1024 * 1024) + + class Missing: + version = '1' + + def read_text(self, name): + return None + + monkeypatch.setattr(importlib.metadata, 'distribution', lambda name: Missing()) + with pytest.raises(ValueError, match='unavailable'): + build_identity.replay_build_identity() diff --git a/modules/common/test_execution_lock.py b/modules/common/test_execution_lock.py new file mode 100644 index 000000000..68543a505 --- /dev/null +++ b/modules/common/test_execution_lock.py @@ -0,0 +1,101 @@ +"""Replay and shadow must compete for one kernel-owned numerical worker slot.""" +import io +import os +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from common.biometrics import execution_lock, replay_worker, shadow_process + + +def test_replay_and_shadow_contend_for_same_inode(tmp_path, monkeypatch): + monkeypatch.setattr(sys, 'platform', 'linux') + path = tmp_path / 'execution.lock' + assert execution_lock.EXECUTION_LOCK_PATH == shadow_process.SHADOW_LOCK_PATH + first = shadow_process.acquire_shadow_lock(path) + try: + with pytest.raises(BlockingIOError): + replay_worker.acquire_execution_lock(path) + finally: + os.close(first) + second = replay_worker.acquire_execution_lock(path) + try: + with pytest.raises(BlockingIOError): + shadow_process.acquire_shadow_lock(path) + finally: + os.close(second) + + +def test_replay_contention_fails_before_limits_imports_or_input(monkeypatch): + def busy(): + raise BlockingIOError('private path') + monkeypatch.setattr(replay_worker, 'acquire_execution_lock', busy) + monkeypatch.setattr(replay_worker, 'configure_process_limits', lambda *_: pytest.fail('limits before lock')) + monkeypatch.setattr(sys, 'argv', ['worker', '--retained-directory', '/private']) + monkeypatch.setattr(sys, 'stdin', None) + stderr = io.StringIO() + monkeypatch.setattr(sys, 'stderr', stderr) + assert replay_worker.main() == 1 + assert stderr.getvalue() == 'Retained waveform worker failed\n' + + +@pytest.mark.parametrize('failed', [False, True]) +def test_replay_keeps_lock_through_processing_and_closes_on_every_exit(monkeypatch, failed): + calls = [] + monkeypatch.setattr(sys, 'argv', ['worker', '--retained-directory', '/retained']) + monkeypatch.setattr(sys, 'stdin', SimpleNamespace(buffer='source')) + monkeypatch.setattr(sys, 'stdout', SimpleNamespace(buffer='sink')) + monkeypatch.setattr(sys, 'stderr', io.StringIO()) + monkeypatch.setattr(replay_worker, 'acquire_execution_lock', lambda: (calls.append('lock'), 123)[1]) + monkeypatch.setattr(replay_worker, 'configure_process_limits', lambda *_: calls.append('limits')) + monkeypatch.setattr(replay_worker.os, 'close', lambda fd: calls.append(('close', fd))) + def run(*args): + assert calls == ['lock', 'limits'] + calls.append('run') + if failed: + raise ValueError('failed') + monkeypatch.setattr(replay_worker, 'run', run) + assert replay_worker.main() == (1 if failed else 0) + assert calls == ['lock', 'limits', 'run', ('close', 123)] + + +@pytest.mark.skipif(sys.platform != 'linux', reason='Real Linux signal/lock ownership') +def test_real_shared_lock_survives_stop_and_releases_on_kill(tmp_path): + path = tmp_path / 'execution.lock' + code = ('from common.biometrics.execution_lock import acquire_execution_lock; ' + 'import sys,time; fd=acquire_execution_lock(sys.argv[1]); ' + 'print("owned",flush=True); time.sleep(60)') + child = subprocess.Popen([sys.executable, '-c', code, str(path)], stdout=subprocess.PIPE, + env={**os.environ, 'PYTHONPATH': str(Path(__file__).resolve().parent.parent)}) + try: + import select + import signal + assert select.select([child.stdout], [], [], 5)[0] + assert child.stdout.readline() == b'owned\n' + child.send_signal(signal.SIGSTOP) + with pytest.raises(BlockingIOError): + shadow_process.acquire_shadow_lock(path) + child.kill() + child.wait(timeout=5) + fd = shadow_process.acquire_shadow_lock(path) + os.close(fd) + finally: + if child.poll() is None: + child.kill() + child.wait(timeout=5) + child.stdout.close() + + +def test_build_probe_does_not_claim_numerical_execution_slot(monkeypatch): + from common.biometrics import build_identity + monkeypatch.setattr(sys, 'argv', ['worker', '--describe-build']) + monkeypatch.setattr(replay_worker, 'acquire_execution_lock', lambda: pytest.fail('probe claimed slot')) + monkeypatch.setattr(replay_worker, 'configure_process_limits', lambda *_: None) + monkeypatch.setattr(build_identity, 'replay_build_identity', lambda: {'hash': 'fixture'}) + output = io.StringIO() + monkeypatch.setattr(sys, 'stdout', output) + assert replay_worker.main() == 0 + assert output.getvalue() == '{"hash":"fixture"}\n' diff --git a/modules/common/test_inspection_worker.py b/modules/common/test_inspection_worker.py new file mode 100644 index 000000000..abbd3a196 --- /dev/null +++ b/modules/common/test_inspection_worker.py @@ -0,0 +1,71 @@ +import gzip +from io import BytesIO +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest + +from common.biometrics.inspection_worker import run + + +def request(root, after=None, payload=None): + output = BytesIO() + run(BytesIO(payload if payload is not None else json.dumps({ + 'version': 'raw-inspection-work-v1', 'after_name': after}).encode()), output, root) + return json.loads(output.getvalue()) + + +def write_archive(root, name): + payload = b'\xa1\x64type\x61x' + (root / name).write_bytes(gzip.compress(b'\xa2\x63seq\x01\x64data\x48' + payload, mtime=0)) + + +def test_inspects_one_archive_at_a_time_without_retention(tmp_path): + write_archive(tmp_path, 'b.RAW.gz') + write_archive(tmp_path, 'a.RAW.gz') + before = {p.name: p.read_bytes() for p in tmp_path.iterdir()} + first = request(tmp_path) + assert first['status'] == 'inspected' + assert first['nextCursor'] == 'a.RAW.gz' + assert first['inspection']['retained'] is False + assert first['inspection']['inventory']['decoded_records'] == 1 + assert first['inspection']['artifact']['sha256'] == first['inspection']['inventory']['artifact_sha256'] + assert request(tmp_path, first['nextCursor'])['nextCursor'] == 'b.RAW.gz' + assert request(tmp_path, 'b.RAW.gz') == {'status': 'complete', 'nextCursor': None} + assert {p.name: p.read_bytes() for p in tmp_path.iterdir()} == before + + +def test_missing_directory_and_corruption_have_explicit_results(tmp_path): + assert request(tmp_path / 'missing') == {'status': 'archive_unavailable', 'nextCursor': None} + assert not (tmp_path / 'missing').exists() + (tmp_path / 'a.RAW.gz').write_bytes(b'not gzip') + write_archive(tmp_path, 'b.RAW.gz') + assert request(tmp_path) == {'status': 'source_unavailable', 'nextCursor': 'a.RAW.gz'} + assert request(tmp_path, 'a.RAW.gz')['status'] == 'inspected' + + +@pytest.mark.parametrize('payload', [ + b'{}', b'[]', b'x' * 1025, + b'{"version":"raw-inspection-work-v1","after_name":null,"after_name":null}', + b'{"version":"raw-inspection-work-v1","after_name":"../a.RAW.gz"}', + b'{"version":"raw-inspection-work-v1","after_name":0}', + b'{"version":"raw-inspection-work-v1","after_name":null,"path":"/tmp"}', +]) +def test_rejects_invalid_work_before_filesystem_access(tmp_path, payload): + with pytest.raises(ValueError): + request(tmp_path / 'missing', payload=payload) + + +def test_fixed_cli_returns_metadata_and_private_failure(tmp_path): + write_archive(tmp_path, 'a.RAW.gz') + command = [sys.executable, '-B', '-m', 'common.biometrics.inspection_worker', '--archive-directory', str(tmp_path)] + env = {**os.environ, 'PYTHONPATH': str(Path(__file__).resolve().parents[1])} + result = subprocess.run(command, input=json.dumps({'version': 'raw-inspection-work-v1', 'after_name': None}).encode(), capture_output=True, env=env, timeout=10) + assert result.returncode == 0 and not result.stderr + assert json.loads(result.stdout)['inspection']['source_name'] == 'a.RAW.gz' + failed = subprocess.run(command, input=b'private malformed input', capture_output=True, env=env, timeout=10) + assert failed.returncode == 1 and not failed.stdout + assert failed.stderr == b'RAW inspection worker failed\n' diff --git a/modules/common/test_raw_artifacts.py b/modules/common/test_raw_artifacts.py new file mode 100644 index 000000000..12a340485 --- /dev/null +++ b/modules/common/test_raw_artifacts.py @@ -0,0 +1,203 @@ +"""Retained RAW bytes survive rolling archive removal, with bounded reads.""" +from concurrent.futures import ThreadPoolExecutor +from dataclasses import replace +import gzip +import hashlib +import os +from pathlib import Path + +import pytest + +from common.biometrics.raw_artifacts import RawArtifactStore, RawArtifact, RawLimits + + +def fixture(tmp_path, payload=b'synthetic waveform bytes' * 100): + archive = tmp_path / 'archive' + archive.mkdir() + compressed = gzip.compress(payload, mtime=0) + (archive / '1.RAW.gz').write_bytes(compressed) + store = RawArtifactStore(tmp_path / 'retained', RawLimits(4096, 65536, 8192, 1)) + return archive, store, compressed + + +def read_all(store, descriptor): + chunks = [] + with store.open_raw(descriptor) as stream: + while True: + chunk = stream.read(1024) + if not chunk: + break + chunks.append(chunk) + return b''.join(chunks) + + +def test_retains_private_immutable_bytes_and_reuses_identical_content(tmp_path): + payload = b'original samples' * 100 + archive, store, compressed = fixture(tmp_path, payload) + descriptor = store.retain(archive, '1.RAW.gz') + assert descriptor.sha256 == hashlib.sha256(compressed).hexdigest() + assert descriptor.compressed_bytes == len(compressed) + assert descriptor.expanded_bytes == len(payload) + assert store.retain(archive, '1.RAW.gz') == descriptor + (archive / '1.RAW.gz').unlink() + assert read_all(store, descriptor) == payload + assert (store.directory / descriptor.name).stat().st_mode & 0o777 == 0o600 + assert store.directory.stat().st_mode & 0o777 == 0o700 + assert len(list(store.directory.glob('*.RAW.gz'))) == 1 + assert not list(store.directory.glob('.part-*')) + + +def test_rejects_paths_symlinks_and_non_regular_archive_inputs(tmp_path): + archive, store, _ = fixture(tmp_path) + for name in ['../1.RAW.gz', '/etc/passwd', '1.RAW', '']: + with pytest.raises(ValueError): + store.retain(archive, name) + (archive / '2.RAW.gz').symlink_to(archive / '1.RAW.gz') + with pytest.raises(OSError): + store.retain(archive, '2.RAW.gz') + os.mkfifo(archive / '3.RAW.gz') + with pytest.raises(ValueError, match='regular'): + store.retain(archive, '3.RAW.gz') + with pytest.raises(ValueError): + RawArtifact('../file', 'x', 1, 1) + + +def test_compressed_expanded_and_free_space_limits_leave_no_partial_artifact(tmp_path): + archive, store, compressed = fixture(tmp_path, b'a' * 10000) + for limits, message in [ + (RawLimits(len(compressed) - 1, 65536, 8192, 1), 'compressed'), + (RawLimits(4096, 9999, 8192, 1), 'expanded'), + (RawLimits(4096, 65536, 8192, 2**62), 'free space'), + ]: + store.limits = limits + with pytest.raises(ValueError, match=message): + store.retain(archive, '1.RAW.gz') + assert not list(store.directory.glob('*.RAW.gz')) + assert not list(store.directory.glob('.part-*')) + + +def test_corruption_and_wrong_descriptors_fail_before_reading(tmp_path): + archive, store, _ = fixture(tmp_path) + descriptor = store.retain(archive, '1.RAW.gz') + with pytest.raises(ValueError, match='size mismatch'): + read_all(store, replace(descriptor, compressed_bytes=descriptor.compressed_bytes + 1)) + with pytest.raises(ValueError, match='expanded size mismatch'): + read_all(store, replace(descriptor, expanded_bytes=descriptor.expanded_bytes + 1)) + with pytest.raises(ValueError, match='expanded byte limit'): + read_all(store, replace(descriptor, expanded_bytes=descriptor.expanded_bytes - 1)) + path = store.directory / descriptor.name + corrupted = bytearray(path.read_bytes()) + corrupted[-1] ^= 1 + path.write_bytes(corrupted) + with pytest.raises(ValueError, match='checksum'): + read_all(store, descriptor) + with pytest.raises(ValueError, match='checksum'): + store.retain(archive, '1.RAW.gz') + + +def test_serializes_quota_publication_and_recovers_crashed_staging_file(tmp_path): + archive, store, _ = fixture(tmp_path, os.urandom(1024)) + (archive / '2.RAW.gz').write_bytes(gzip.compress(os.urandom(1024), mtime=0)) + store.limits = RawLimits(4096, 65536, 1500, 1) + (store.directory / ('.part-' + 'a' * 32)).write_bytes(b'crashed copy') + def retain(name): + try: + return store.retain(archive, name) + except ValueError as error: + return str(error) + with ThreadPoolExecutor(max_workers=2) as pool: + results = list(pool.map(retain, ['1.RAW.gz', '2.RAW.gz'])) + assert sum(isinstance(row, RawArtifact) for row in results) == 1 + assert any(isinstance(row, str) and 'quota' in row for row in results) + assert len(list(store.directory.glob('*.RAW.gz'))) == 1 + assert not list(store.directory.glob('.part-*')) + + +def test_rejects_unbounded_reads_and_invalid_gzip(tmp_path): + archive, store, _ = fixture(tmp_path) + descriptor = store.retain(archive, '1.RAW.gz') + with store.open_raw(descriptor) as stream: + with pytest.raises(ValueError, match='bounded'): + stream.read() + assert stream.read(0) == b'' + (archive / '2.RAW.gz').write_bytes(b'not gzip') + with pytest.raises(OSError): + store.retain(archive, '2.RAW.gz') + assert not list(store.directory.glob('.part-*')) + + +def test_reuses_identical_content_at_exact_publication_quota(tmp_path): + archive, store, compressed = fixture(tmp_path) + store.limits = RawLimits(4096, 65536, len(compressed), 1) + first = store.retain(archive, '1.RAW.gz') + assert store.retain(archive, '1.RAW.gz') == first + assert store._usage() == (len(compressed), 1) + + +def test_changed_archive_is_not_published(tmp_path, monkeypatch): + archive, store, _ = fixture(tmp_path) + original_read = os.read + changed = False + def read(fd, count): + nonlocal changed + data = original_read(fd, count) + if data and not changed: + changed = True + (archive / '1.RAW.gz').write_bytes(gzip.compress(b'replaced source', mtime=0)) + return data + monkeypatch.setattr(os, 'read', read) + with pytest.raises(ValueError, match='changed during retention'): + store.retain(archive, '1.RAW.gz') + assert not list(store.directory.glob('*.RAW.gz')) + assert not list(store.directory.glob('.part-*')) + + +def test_count_budget_applies_even_to_tiny_files_but_allows_idempotent_retry(tmp_path): + archive, store, _ = fixture(tmp_path, b'one') + store.limits = replace(store.limits, max_artifacts=1) + first = store.retain(archive, '1.RAW.gz') + assert store.retain(archive, '1.RAW.gz') == first + (archive / '2.RAW.gz').write_bytes(gzip.compress(b'two', mtime=0)) + with pytest.raises(ValueError, match='artifact count'): + store.retain(archive, '2.RAW.gz') + assert read_all(store, first) == b'one' + assert not list(store.directory.glob('.part-*')) + + +def test_read_only_store_does_not_create_or_change_storage_and_rejects_mutations(tmp_path): + archive, store, _ = fixture(tmp_path) + descriptor = store.retain(archive, '1.RAW.gz') + os.chmod(store.directory, 0o750) + reader = RawArtifactStore(store.directory, store.limits, read_only=True) + assert store.directory.stat().st_mode & 0o777 == 0o750 + assert read_all(reader, descriptor) == b'synthetic waveform bytes' * 100 + with pytest.raises(ValueError, match='read-only'): + reader.retain(archive, '1.RAW.gz') + with pytest.raises(ValueError, match='read-only'): + reader.collect_released(None, now_ms=1000) + missing = tmp_path / 'missing' + with pytest.raises(ValueError): + RawArtifactStore(missing, read_only=True) + assert not missing.exists() + (store.directory / '.lock').unlink() + with pytest.raises(FileNotFoundError): + read_all(reader, descriptor) + assert not (store.directory / '.lock').exists() + + os.mkfifo(store.directory / '.lock') + with pytest.raises(ValueError, match='regular file'): + read_all(reader, descriptor) + + +def test_reuses_exact_retained_bytes_without_the_original_archive_and_checks_expanded_size(tmp_path): + archive, store, _ = fixture(tmp_path) + descriptor = store.retain(archive, '1.RAW.gz') + (archive / '1.RAW.gz').unlink() + archive.rmdir() + seen = [] + assert store.retain(archive, '1.RAW.gz', expected=descriptor, publish=seen.append) == descriptor + assert seen == [descriptor] + with pytest.raises(ValueError, match='expanded size mismatch'): + store.retain(archive, '1.RAW.gz', expected=replace(descriptor, expanded_bytes=descriptor.expanded_bytes + 1)) + with pytest.raises(ValueError, match='expanded byte limit'): + store.retain(archive, '1.RAW.gz', expected=replace(descriptor, expanded_bytes=descriptor.expanded_bytes - 1)) diff --git a/modules/common/test_raw_collection.py b/modules/common/test_raw_collection.py new file mode 100644 index 000000000..de75588de --- /dev/null +++ b/modules/common/test_raw_collection.py @@ -0,0 +1,269 @@ +"""Released files cannot race verified reads or reference publication.""" +from concurrent.futures import ThreadPoolExecutor +from contextlib import closing +import gzip +from pathlib import Path +import sqlite3 +from threading import Event + +import pytest + +from common.biometrics.raw_artifacts import RawArtifactStore, RawLimits + + +def fixture(tmp_path): + archive = tmp_path / 'archive' + archive.mkdir() + (archive / '1.RAW.gz').write_bytes(gzip.compress(b'samples' * 100, mtime=0)) + store = RawArtifactStore(tmp_path / 'retained', RawLimits(4096, 65536, 8192, 1)) + descriptor = store.retain(archive, '1.RAW.gz') + database = tmp_path / 'references.db' + connection = sqlite3.connect(database) + connection.execute('PRAGMA foreign_keys = ON') + connection.execute('CREATE TABLE biometric_datasets (id TEXT PRIMARY KEY)') + connection.execute("INSERT INTO biometric_datasets VALUES ('dataset')") + migration = Path(__file__).parents[2] / 'src/db/biometrics-migrations/0028_retained_raw_artifacts.sql' + connection.executescript(migration.read_text()) + connection.execute('INSERT INTO biometric_raw_artifacts VALUES (?, ?, ?, ?, ?, ?)', + (descriptor.sha256, descriptor.name, descriptor.compressed_bytes, + descriptor.expanded_bytes, 1000, 2000)) + connection.commit() + return archive, store, descriptor, database, connection + + +def test_collects_only_released_unreferenced_bytes_after_grace(tmp_path): + _, store, descriptor, _, db = fixture(tmp_path) + try: + db.execute('INSERT INTO biometric_dataset_raw_artifacts VALUES (?, ?, ?, ?, ?)', + ('link', 'dataset', descriptor.sha256, 0, None)) + db.commit() + assert store.collect_released(db, now_ms=10000, minimum_age_ms=1000) == [] + db.execute('UPDATE biometric_dataset_raw_artifacts SET released_at_ms = 2000') + db.commit() + assert store.collect_released(db, now_ms=2999, minimum_age_ms=1000) == [] + assert store.collect_released(db, now_ms=3000, minimum_age_ms=1000) == [descriptor.name] + assert not (store.directory / descriptor.name).exists() + assert store.collect_released(db, now_ms=4000, minimum_age_ms=1000) == [] + # Metadata/links remain as tombstones, including their original identity. + assert db.execute('SELECT released_at_ms FROM biometric_raw_artifacts').fetchone() == (2000,) + assert db.execute('SELECT artifact_hash FROM biometric_dataset_raw_artifacts').fetchone() == (descriptor.sha256,) + finally: + db.close() + + +def test_preserves_unknown_files_and_rejects_unbounded_or_nested_collection(tmp_path): + _, store, descriptor, _, db = fixture(tmp_path) + try: + unknown = store.directory / ('f' * 64 + '.RAW.gz') + unknown.write_bytes(b'unknown future artifact') + assert store.collect_released(db, now_ms=2000, minimum_age_ms=0) == [descriptor.name] + assert unknown.exists() + db.execute('BEGIN') + with pytest.raises(ValueError, match='independent transaction'): + store.collect_released(db, now_ms=3000) + db.rollback() + for kwargs in [{'now_ms': -1}, {'now_ms': True}, {'now_ms': 3000, 'minimum_age_ms': -1}, + {'now_ms': 3000, 'maximum_entries': 0}]: + with pytest.raises(ValueError, match='bounds'): + store.collect_released(db, **kwargs) + with pytest.raises(ValueError, match='entry limit'): + store.collect_released(db, now_ms=3000, maximum_entries=1) + finally: + db.close() + + +def test_rejects_released_symlinks_and_metadata_identity_mismatch(tmp_path): + _, store, descriptor, _, db = fixture(tmp_path) + try: + path = store.directory / descriptor.name + path.unlink() + victim = tmp_path / 'unrelated' + victim.write_text('keep') + path.symlink_to(victim) + with pytest.raises(ValueError, match='regular file'): + store.collect_released(db, now_ms=3000, minimum_age_ms=0) + assert victim.read_text() == 'keep' + assert not db.in_transaction + db.execute('UPDATE biometric_raw_artifacts SET content_hash = ?', ('f' * 64,)) + db.commit() + with pytest.raises(ValueError, match='identity mismatch'): + store.collect_released(db, now_ms=3000, minimum_age_ms=0) + finally: + db.close() + + +def test_reader_holds_file_until_stream_is_closed(tmp_path): + _, store, descriptor, database, db = fixture(tmp_path) + db.close() + started = Event() + + def collect(): + with closing(sqlite3.connect(database)) as connection: + started.set() + return store.collect_released(connection, now_ms=3000, minimum_age_ms=0) + + with ThreadPoolExecutor(max_workers=1) as pool: + with store.open_raw(descriptor) as stream: + future = pool.submit(collect) + assert started.wait(2) + assert not future.done() + assert stream.read(7) == b'samples' + assert future.result(timeout=2) == [descriptor.name] + + +def test_reference_publication_completes_before_collection_can_check(tmp_path): + archive, store, descriptor, database, db = fixture(tmp_path) + started = Event() + pending = [] + + def collect(): + with closing(sqlite3.connect(database)) as connection: + started.set() + return store.collect_released(connection, now_ms=3000, minimum_age_ms=0) + + with ThreadPoolExecutor(max_workers=1) as pool: + def publish(actual): + assert actual == descriptor + pending.append(pool.submit(collect)) + assert started.wait(2) + assert not pending[0].done() + with db: + db.execute('UPDATE biometric_raw_artifacts SET released_at_ms = NULL') + db.execute('INSERT INTO biometric_dataset_raw_artifacts VALUES (?, ?, ?, ?, ?)', + ('link', 'dataset', descriptor.sha256, 0, None)) + + assert store.retain(archive, '1.RAW.gz', expected=descriptor, publish=publish) == descriptor + assert pending[0].result(timeout=2) == [] + db.close() + assert (store.directory / descriptor.name).exists() + + +def test_failed_publisher_leaves_valid_bytes_and_releases_file_lock(tmp_path): + archive, store, descriptor, _, db = fixture(tmp_path) + try: + # Exercise fresh publication, not just the deduplicated path. + (store.directory / descriptor.name).unlink() + + def fail(actual): + assert actual == descriptor + raise RuntimeError('publication failed') + + with pytest.raises(ValueError, match='inspected descriptor'): + store.retain(archive, '1.RAW.gz', publish=fail) + with pytest.raises(RuntimeError, match='publication failed'): + store.retain(archive, '1.RAW.gz', expected=descriptor, publish=fail) + with store.open_raw(descriptor) as stream: + assert stream.read(7) == b'samples' + assert not list(store.directory.glob('.part-*')) + assert store.collect_released(db, now_ms=3000, minimum_age_ms=0) == [descriptor.name] + finally: + db.close() + + +def test_collects_only_old_unpublished_copies_and_parts(tmp_path): + import os + _, store, descriptor, _, db = fixture(tmp_path) + try: + orphan = store.directory / ('a' * 64 + '.RAW.gz') + part = store.directory / ('.part-' + 'b' * 32) + fresh = store.directory / ('c' * 64 + '.RAW.gz') + unknown = store.directory / 'unrecognized' + for path in [orphan, part, fresh, unknown]: + path.write_bytes(b'private fixture') + os.utime(path, ns=(1_000_000_000, 1_000_000_000)) + os.utime(fresh, ns=(3_000_000_000, 3_000_000_000)) + assert store.collect_orphans(db, now_ms=1999, minimum_age_ms=1000) == [] + assert store.collect_orphans(db, now_ms=2000, minimum_age_ms=1000) == sorted([orphan.name, part.name]) + assert fresh.exists() and unknown.exists() + assert (store.directory / descriptor.name).exists() + assert db.execute('SELECT count(*) FROM biometric_raw_artifacts').fetchone()[0] == 1 + finally: + db.close() + + +def test_orphan_inventory_and_symlink_errors_delete_nothing(tmp_path): + import os + _, store, _, _, db = fixture(tmp_path) + try: + orphan = store.directory / ('a' * 64 + '.RAW.gz') + orphan.write_bytes(b'old') + os.utime(orphan, ns=(0, 0)) + with pytest.raises(ValueError, match='entry limit'): + store.collect_orphans(db, now_ms=10000, maximum_entries=1) + assert orphan.exists() + symlink = store.directory / ('b' * 64 + '.RAW.gz') + symlink.symlink_to(orphan) + with pytest.raises(ValueError, match='regular file'): + store.collect_orphans(db, now_ms=10000, minimum_age_ms=0) + assert orphan.exists() and symlink.is_symlink() + assert not db.in_transaction + finally: + db.close() + + +def test_orphan_collector_preserves_metadata_even_when_unreferenced_or_misnamed(tmp_path): + import os + _, store, descriptor, _, db = fixture(tmp_path) + try: + path = store.directory / descriptor.name + os.utime(path, ns=(0, 0)) + db.execute('UPDATE biometric_raw_artifacts SET released_at_ms = NULL, name = ?', ('inconsistent-name',)) + db.commit() + assert store.collect_orphans(db, now_ms=10000, minimum_age_ms=0) == [] + assert path.exists() + finally: + db.close() + + +def test_orphan_collection_waits_for_publication_metadata_commit(tmp_path): + archive, store, descriptor, database, db = fixture(tmp_path) + db.execute('DELETE FROM biometric_raw_artifacts') + db.commit() + started = Event() + pending = [] + + def collect(): + with closing(sqlite3.connect(database)) as connection: + started.set() + return store.collect_orphans(connection, now_ms=9_000_000_000_000, minimum_age_ms=0) + + try: + with ThreadPoolExecutor(max_workers=1) as pool: + def publish(actual): + pending.append(pool.submit(collect)) + assert started.wait(2) + assert not pending[0].done() + with db: + db.execute('INSERT INTO biometric_raw_artifacts VALUES (?, ?, ?, ?, ?, ?)', + (actual.sha256, actual.name, actual.compressed_bytes, + actual.expanded_bytes, 1000, None)) + + store.retain(archive, '1.RAW.gz', expected=descriptor, publish=publish) + assert pending[0].result(timeout=2) == [] + assert (store.directory / descriptor.name).exists() + finally: + db.close() + + +def test_collection_worker_runs_both_collectors_without_exposing_filenames(tmp_path): + from io import BytesIO + import json + import os + from common.biometrics.collection_worker import run + _, store, descriptor, database, db = fixture(tmp_path) + try: + orphan = store.directory / ('a' * 64 + '.RAW.gz') + orphan.write_bytes(b'private fixture') + os.utime(orphan, ns=(0, 0)) + output = BytesIO() + run(BytesIO(b'{"version":"raw-collection-work-v1"}'), output, database, store.directory) + assert json.loads(output.getvalue()) == {'version': 'raw-collection-result-v1', 'released': 1, 'orphans': 1} + assert not orphan.exists() and not (store.directory / descriptor.name).exists() + assert db.execute('SELECT count(*) FROM biometric_raw_artifacts').fetchone()[0] == 1 + with pytest.raises(ValueError, match='envelope'): + run(BytesIO(b'{"version":"raw-collection-work-v1","path":"/tmp"}'), BytesIO(), database, store.directory) + missing = tmp_path / 'missing' + run(BytesIO(b'{"version":"raw-collection-work-v1"}'), BytesIO(), database, missing) + assert not missing.exists() + finally: + db.close() diff --git a/modules/common/test_raw_records.py b/modules/common/test_raw_records.py new file mode 100644 index 000000000..48b7e1c39 --- /dev/null +++ b/modules/common/test_raw_records.py @@ -0,0 +1,164 @@ +"""Bounded CBOR decoding against captured firmware and malformed archives.""" +import base64 +from dataclasses import replace +import gzip +import importlib.util +from pathlib import Path + +import pytest +import cbor2 + +from common.biometrics import raw_records +from common.biometrics.raw_records import decode_record, DecodeLimits, iter_retained_records +from common.biometrics.raw_artifacts import RawArtifactStore, RawLimits +from common.nats_capture_fixtures import CAPTURE_FIXTURES + +# Other follower tests stub the outer parser during collection. Use its actual +# source for this integration test, matching test_cbor_raw's isolation strategy. +spec = importlib.util.spec_from_file_location('_retained_raw_parser', Path(__file__).with_name('cbor_raw.py')) +parser = importlib.util.module_from_spec(spec) +spec.loader.exec_module(parser) + + +@pytest.fixture(autouse=True) +def real_outer_parser(monkeypatch): + monkeypatch.setattr(raw_records, 'read_raw_record', parser.read_raw_record) + + +@pytest.mark.parametrize('fixture', list(CAPTURE_FIXTURES.values())) +def test_decodes_actual_firmware_indefinite_maps(fixture): + payload = base64.b64decode(fixture['payload_b64']) + record = decode_record(payload) + assert record == cbor2.loads(payload) + assert record['type'] == fixture['type'] + assert isinstance(record['ts'], (int, float)) + if record['type'] == 'piezo-dual': + assert len(record['left1']) == len(record['right1']) == 2000 + + +def test_rejects_huge_declared_containers_without_allocating_them(): + for prefix in [b'\x9b', b'\xbb']: + with pytest.raises(ValueError, match='container budget'): + decode_record(prefix + b'\xff' * 8) + with pytest.raises(ValueError, match='string budget'): + decode_record(b'\x5b' + b'\xff' * 8) + + +@pytest.mark.parametrize('payload', [ + b'\xbf\x64type\x61x\x64type\x61y\xff', # Duplicate indefinite key. + b'\xa2\x64type\x61x\x64type\x61y', # Duplicate definite key. + b'\xa1\x01\x61x', # Non-text key. + b'\xa1\x64type\x61x\x00', # Trailing second value. + b'\xa1\x64type\xc0\x61x', # Semantic tag. + b'\xbf\x64type\x61x', # Missing break. + b'\xa1\x64type\x7f\x61\xc3\x61\xa9\xff', # Invalid UTF-8 chunks. +]) +def test_rejects_ambiguous_or_malformed_records(payload): + with pytest.raises(ValueError): + decode_record(payload) + + +def test_depth_nodes_bytes_and_indefinite_chunk_budgets(): + with pytest.raises(ValueError, match='nesting'): + decode_record(b'\x81' * 18 + b'\x00') + with pytest.raises(ValueError, match='node budget'): + decode_record(b'\x9f' + b'\x00' * 8 + b'\xff', replace(DecodeLimits(), nodes=5)) + with pytest.raises(ValueError, match='byte budget'): + decode_record(b'x' * 10, replace(DecodeLimits(), record_bytes=9)) + with pytest.raises(ValueError, match='string budget'): + decode_record(b'\x5f\x43abc\x43def\xff', replace(DecodeLimits(), string_bytes=5)) + + +def test_numeric_and_chunked_values_preserve_wire_semantics(): + payload = (b'\xbf\x64type\x61x\x61v\x9f\x20\xf5\xf4\xf6' + b'\xfa\x3f\xc0\x00\x00\xf9\x3e\x00\xff' + b'\x61b\x5f\x42ab\x41c\xff\xff') + assert decode_record(payload) == {'type': 'x', 'v': [-1, True, False, None, 1.5, 1.5], 'b': b'abc'} + + +def envelope(payload): + return b'\xa2\x63seq\x01\x64data\x5a' + len(payload).to_bytes(4, 'big') + payload + + +def retain(tmp_path, payload): + archive = tmp_path / 'archive' + archive.mkdir(exist_ok=True) + (archive / '1.RAW.gz').write_bytes(gzip.compress(payload, mtime=0)) + store = RawArtifactStore(tmp_path / 'retained', RawLimits(4096, 65536, 8192, 1)) + return store, store.retain(archive, '1.RAW.gz') + + +def test_streams_verified_records_and_counts_placeholders_against_budget(tmp_path): + payload = b'\xa1\x64type\x61x' + store, descriptor = retain(tmp_path, envelope(b'') + envelope(payload) + envelope(payload)) + assert list(iter_retained_records(store, descriptor)) == [{'type': 'x'}, {'type': 'x'}] + with pytest.raises(ValueError, match='record count'): + list(iter_retained_records(store, descriptor, max_records=2)) + + +@pytest.mark.parametrize('tail', [b'\xa2', b'\xa2\x63seq\x01', envelope(b'abc')[:-1]]) +def test_rejects_truncated_retained_outer_record_instead_of_silent_eof(tmp_path, tail): + store, descriptor = retain(tmp_path, tail) + with pytest.raises(ValueError, match='Truncated retained RAW framing'): + list(iter_retained_records(store, descriptor)) + + +def test_inventory_uses_retained_frames_without_inventing_missing_context(tmp_path): + payload = base64.b64decode(CAPTURE_FIXTURES['raw.sens.piezo']['payload_b64']) + store, descriptor = retain(tmp_path, envelope(payload) + envelope(payload)) + result = raw_records.inventory_retained(store, descriptor) + assert result['ordered_waveform_available'] is True + assert result['usable_dual_frames'] == 2 + assert result['duplicate_timestamps'] == 1 + assert result['first_event_ms'] == result['last_event_ms'] + assert result['channels']['left1']['samples'] == 1000 + assert result['channels']['right1']['samples'] == 1000 + assert result['pump_records'] == 0 + assert result['artifact_sha256'] == descriptor.sha256 + + +def test_inventory_exposes_missing_times_and_does_not_claim_replayability(tmp_path): + # Definite map: type=piezo-dual, no ts or sample channels. + payload = b'\xa1\x64type\x6apiezo-dual' + store, descriptor = retain(tmp_path, envelope(payload)) + result = raw_records.inventory_retained(store, descriptor) + assert result['invalid_timestamps'] == 1 + assert result['empty_channel_frames'] == 1 + assert result['ordered_waveform_available'] is False + assert result['first_event_ms'] is None + + +def piezo_record(timestamp, channel=b'\x01\x00\x00\x00'): + encoded_time = b'\x1a' + timestamp.to_bytes(4, 'big') + encoded_channel = b'\x59' + len(channel).to_bytes(2, 'big') + channel + return (b'\xa4\x64type\x6apiezo-dual\x62ts' + encoded_time + + b'\x65left1' + encoded_channel + b'\x66right1' + encoded_channel) + + +def test_inventory_reports_reversals_lost_markers_and_truncated_sample_bytes(tmp_path): + channel = b'\x01\x00\x00\x00\xff\xff\xff\x7f\x03\x00\x00\x00x' + store, descriptor = retain(tmp_path, envelope(piezo_record(100, channel)) + envelope(piezo_record(99, channel))) + result = raw_records.inventory_retained(store, descriptor) + assert result['timestamp_reversals'] == 1 + assert result['ordered_waveform_available'] is False + assert (result['first_event_ms'], result['last_event_ms']) == (99000, 100000) + assert result['channels']['left1'] == {'samples': 6, 'lost_markers': 2, 'truncated_tail_bytes': 2} + + +def test_inventory_rejects_channels_too_large_for_the_replay_driver(tmp_path): + store, descriptor = retain(tmp_path, envelope(piezo_record(100, bytes(20004)))) + result = raw_records.inventory_retained(store, descriptor) + assert result['oversize_channel_frames'] == 1 + assert result['ordered_waveform_available'] is False + + +def test_shared_budget_counts_empty_placeholders_across_artifact_streams(tmp_path): + store, descriptor = retain(tmp_path, envelope(b'') + envelope(b'')) + budget = raw_records.RawRecordBudget(3) + assert list(iter_retained_records(store, descriptor, budget=budget)) == [] + assert budget.consumed == 2 + with pytest.raises(ValueError, match='record count'): + list(iter_retained_records(store, descriptor, budget=budget)) + assert budget.consumed == 4 + with pytest.raises(ValueError, match='shared RAW'): + list(iter_retained_records(store, descriptor, budget=3)) diff --git a/modules/common/test_raw_sources.py b/modules/common/test_raw_sources.py new file mode 100644 index 000000000..7f04d34b4 --- /dev/null +++ b/modules/common/test_raw_sources.py @@ -0,0 +1,66 @@ +"""Inspection cannot substitute mutable archive names for frozen dataset bytes.""" +import gzip +import importlib.util +from pathlib import Path + +import pytest + +from common.biometrics import raw_records +from common.biometrics.raw_sources import ArchiveSource, archive_names +from common.biometrics.raw_artifacts import RawArtifactStore, RawLimits + +spec = importlib.util.spec_from_file_location('_source_raw_parser', Path(__file__).with_name('cbor_raw.py')) +parser = importlib.util.module_from_spec(spec) +spec.loader.exec_module(parser) + + +def archive(tmp_path, monkeypatch): + monkeypatch.setattr(raw_records, 'read_raw_record', parser.read_raw_record) + root = tmp_path / 'archive' + root.mkdir() + payload = b'\xa1\x64type\x61x' + encoded = b'\xa2\x63seq\x01\x64data\x48' + payload + (root / '1.RAW.gz').write_bytes(gzip.compress(encoded, mtime=0)) + return root + + +def test_inspects_without_retention_then_freezes_exact_selected_bytes(tmp_path, monkeypatch): + root = archive(tmp_path, monkeypatch) + source = ArchiveSource(root, '1.RAW.gz') + inspection = source.inspect() + assert inspection['retained'] is False + assert inspection['inventory']['decoded_records'] == 1 + assert sorted(p.name for p in tmp_path.iterdir()) == ['archive'] + store = RawArtifactStore(tmp_path / 'retained', RawLimits(4096, 65536, 8192, 1)) + saved = store.retain(root, '1.RAW.gz', expected=inspection['artifact']) + assert saved == inspection['artifact'] + (root / '1.RAW.gz').unlink() + assert list(raw_records.iter_retained_records(store, saved)) == [{'type': 'x'}] + + +def test_source_replacement_requires_reinspection_and_publishes_nothing(tmp_path, monkeypatch): + root = archive(tmp_path, monkeypatch) + source = ArchiveSource(root, '1.RAW.gz') + descriptor = source.inspect()['artifact'] + (root / '1.RAW.gz').write_bytes(gzip.compress(b'new archive', mtime=0)) + with pytest.raises(ValueError, match='changed since inspection'): + with source.open_raw(descriptor): + pass + store = RawArtifactStore(tmp_path / 'retained', RawLimits(4096, 65536, 8192, 1)) + with pytest.raises(ValueError, match='changed since inspection'): + store.retain(root, '1.RAW.gz', expected=descriptor) + assert not list(store.directory.glob('*.RAW.gz')) + assert not list(store.directory.glob('.part-*')) + + +def test_discovery_is_bounded_and_excludes_symlinks_and_nonarchives(tmp_path, monkeypatch): + root = archive(tmp_path, monkeypatch) + (root / '2.RAW.gz').symlink_to(root / '1.RAW.gz') + (root / 'partial.tmp').write_bytes(b'ignored') + assert archive_names(root) == ['1.RAW.gz'] + with pytest.raises(ValueError, match='entry limit'): + archive_names(root, maximum_entries=2) + with pytest.raises(ValueError): + ArchiveSource(root, '../outside.RAW.gz') + with pytest.raises(OSError): + ArchiveSource(root, '2.RAW.gz').inspect() diff --git a/modules/common/test_replay_resources.py b/modules/common/test_replay_resources.py new file mode 100644 index 000000000..3ff509354 --- /dev/null +++ b/modules/common/test_replay_resources.py @@ -0,0 +1,52 @@ +import builtins +import io +import sys +from types import SimpleNamespace + +import pytest + +from common.biometrics import replay_worker + + +def test_linux_peak_is_kernel_high_water_mark_in_bytes_without_resource(monkeypatch): + monkeypatch.setattr(sys, 'platform', 'linux') + opened = [] + + def read(path, mode, encoding): + opened.append((path, mode, encoding)) + return io.StringIO('Name:\tpython\nVmHWM:\t 12345 kB\nVmRSS:\t 100 kB\n') + + monkeypatch.setattr(builtins, 'open', read) + assert replay_worker.peak_rss_bytes() == 12345 * 1024 + assert opened == [('/proc/self/status', 'r', 'ascii')] + + +@pytest.mark.parametrize('text', ['', 'VmRSS: 12 kB\n', 'VmHWM: 0 kB\n', + 'VmHWM: 12 bytes\n', 'VmHWM: 12 kB\nVmHWM: 13 kB\n', + 'VmHWM: 99999999999999999999999 kB\n', 'x' * 16385]) +def test_missing_or_malformed_peak_is_unknown(monkeypatch, text): + monkeypatch.setattr(sys, 'platform', 'linux') + monkeypatch.setattr(builtins, 'open', lambda *args, **kwargs: io.StringIO(text)) + assert replay_worker.peak_rss_bytes() is None + + +def test_missing_procfs_is_unknown(monkeypatch): + monkeypatch.setattr(sys, 'platform', 'linux') + + def fail(*args, **kwargs): + raise FileNotFoundError() + + monkeypatch.setattr(builtins, 'open', fail) + assert replay_worker.peak_rss_bytes() is None + + +def test_darwin_peak_is_already_bytes(monkeypatch): + monkeypatch.setattr(sys, 'platform', 'darwin') + monkeypatch.setitem(sys.modules, 'resource', SimpleNamespace(RUSAGE_SELF=0, + getrusage=lambda who: SimpleNamespace(ru_maxrss=12345))) + assert replay_worker.peak_rss_bytes() == 12345 + + +def test_unknown_platform_does_not_guess_units(monkeypatch): + monkeypatch.setattr(sys, 'platform', 'other') + assert replay_worker.peak_rss_bytes() is None diff --git a/modules/common/test_shadow_build.py b/modules/common/test_shadow_build.py new file mode 100644 index 000000000..a1a4c5b9c --- /dev/null +++ b/modules/common/test_shadow_build.py @@ -0,0 +1,58 @@ +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest + +from common.biometrics import build_identity, shadow_build +from common.biometrics.config import config_hash + + +def test_shadow_identity_covers_shared_and_prospective_sources(): + result = shadow_build.shadow_build_identity() + assert result == shadow_build.shadow_build_identity() + assert result['hash'] == config_hash(result['identity']) + assert result['hash'] != build_identity.replay_build_identity()['hash'] + assert result['identity']['version'] == 'shadow-python-build-v1' + root = Path(shadow_build.__file__).resolve().parents[2] + assert set(result['identity']['sources']) == set(build_identity.SOURCE_FILES + shadow_build.SHADOW_SOURCE_FILES) + for name in shadow_build.SHADOW_SOURCE_FILES: + assert result['identity']['sources'][name] == hashlib.sha256((root / name).read_bytes()).hexdigest() + + +@pytest.mark.parametrize('name', shadow_build.SHADOW_SOURCE_FILES) +def test_each_shadow_source_change_invalidates_identity(name, monkeypatch): + before = shadow_build.shadow_build_identity()['hash'] + original = build_identity._digest_file + monkeypatch.setattr(build_identity, '_digest_file', lambda path: 'a' * 64 if str(path).endswith(name) else original(path)) + assert shadow_build.shadow_build_identity()['hash'] != before + + +def test_probe_emits_only_identity_json_and_never_imports_processor(): + env = {**os.environ, 'PYTHONPATH': str(Path(shadow_build.__file__).resolve().parents[2])} + result = subprocess.run([sys.executable, '-B', '-m', 'common.biometrics.shadow_build'], + capture_output=True, text=True, check=True, timeout=10, env=env) + assert json.loads(result.stdout) == shadow_build.shadow_build_identity() + assert result.stderr == '' + code = "from common.biometrics.shadow_build import shadow_build_identity; import sys; shadow_build_identity(); assert 'common.biometrics.shadow' not in sys.modules; assert 'numpy' not in sys.modules" + subprocess.run([sys.executable, '-B', '-c', code], check=True, timeout=10, env=env) + + +def test_missing_dependency_file_listing_fails_closed(monkeypatch): + original = shadow_build.importlib.metadata.distribution + + class WithoutFiles: + files = [] + + def __init__(self, name): + self.original = original(name) + + def __getattr__(self, name): + return getattr(self.original, name) + + monkeypatch.setattr(shadow_build.importlib.metadata, 'distribution', WithoutFiles) + with pytest.raises(ValueError, match='paths unavailable'): + shadow_build.shadow_build_identity() diff --git a/modules/common/test_shadow_process.py b/modules/common/test_shadow_process.py new file mode 100644 index 000000000..e52b0dfec --- /dev/null +++ b/modules/common/test_shadow_process.py @@ -0,0 +1,329 @@ +"""Kernel-limit startup must fail closed before numerical imports or input.""" +import builtins +import io +import subprocess +import sys +from types import SimpleNamespace + +import pytest + +from common.biometrics import shadow_process + + +def test_installs_address_limit_after_shared_cpu_limits(monkeypatch): + calls = [] + monkeypatch.setattr(sys, 'platform', 'linux') + monkeypatch.setattr(shadow_process, 'configure_process_limits', lambda cpu: calls.append(('cpu', cpu))) + monkeypatch.setitem(sys.modules, 'resource', SimpleNamespace(RLIMIT_AS=9, + setrlimit=lambda kind, values: calls.append(('as', kind, values)))) + shadow_process.configure_shadow_limits(31, 123456789) + assert calls == [('cpu', 31), ('as', 9, (123456789, 123456789))] + + +@pytest.mark.parametrize('budget', [0, -1, True, 1.5, '123', 9_007_199_254_740_992]) +def test_invalid_address_budget_cannot_start(monkeypatch, budget): + monkeypatch.setattr(sys, 'platform', 'linux') + monkeypatch.setattr(shadow_process, 'configure_process_limits', lambda cpu: pytest.fail('limits already changed')) + with pytest.raises(ValueError): + shadow_process.configure_shadow_limits(1, budget) + + +def test_unsupported_host_does_not_claim_enforcement(monkeypatch): + monkeypatch.setattr(sys, 'platform', 'darwin') + with pytest.raises(RuntimeError, match='Linux'): + shadow_process.configure_shadow_limits(1, 123) + + +def test_yocto_uses_fixed_prlimit_and_propagates_refusal(monkeypatch): + monkeypatch.setattr(sys, 'platform', 'linux') + monkeypatch.setattr(shadow_process, 'configure_process_limits', lambda cpu: None) + original = builtins.__import__ + def without_resource(name, *args, **kwargs): + if name == 'resource': + raise ModuleNotFoundError(name='resource') + return original(name, *args, **kwargs) + monkeypatch.setattr(builtins, '__import__', without_resource) + calls = [] + def execute(command, **kwargs): + calls.append((command, kwargs)) + monkeypatch.setattr(subprocess, 'run', execute) + shadow_process.configure_shadow_limits(1, 123456789) + assert calls == [(['/usr/bin/prlimit', '--pid', str(shadow_process.os.getpid()), '--as=123456789:123456789'], + dict(check=True, timeout=5, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL))] + def refuse(*args, **kwargs): + raise subprocess.CalledProcessError(1, args[0]) + monkeypatch.setattr(subprocess, 'run', refuse) + with pytest.raises(subprocess.CalledProcessError): + shadow_process.configure_shadow_limits(1, 123456789) + + +def test_failed_limits_never_import_processor_or_read_input(monkeypatch): + original = builtins.__import__ + def guard(name, *args, **kwargs): + if name.endswith('shadow_worker'): + pytest.fail('numerical processor imported before limits succeeded') + return original(name, *args, **kwargs) + monkeypatch.setattr(builtins, '__import__', guard) + def fail(*args): + raise OSError('private-path-or-frame') + monkeypatch.setattr(shadow_process, 'configure_parent_death', lambda pid: None) + monkeypatch.setattr(shadow_process, 'acquire_shadow_lock', lambda: 123) + monkeypatch.setattr(shadow_process, 'configure_wall_deadline', lambda ms: None) + monkeypatch.setattr(shadow_process.os, 'close', lambda fd: None) + monkeypatch.setattr(shadow_process, 'configure_shadow_limits', fail) + stderr = io.StringIO() + monkeypatch.setattr(sys, 'stderr', stderr) + monkeypatch.setattr(sys, 'stdin', None) + assert shadow_process.main(['--parent-pid', '123', '--wall-ms', '10000', '--cpu-seconds', '1', '--address-space-bytes', '123']) == 1 + assert stderr.getvalue() == 'Shadow process failed\n' + + +def test_successful_limits_precede_processor_import_and_run(monkeypatch): + calls = [] + monkeypatch.setattr(shadow_process, 'configure_parent_death', lambda pid: calls.append('parent')) + monkeypatch.setattr(shadow_process, 'acquire_shadow_lock', lambda: (calls.append('lock'), 123)[1]) + monkeypatch.setattr(shadow_process, 'configure_wall_deadline', lambda ms: calls.append(('wall', ms))) + monkeypatch.setattr(shadow_process.os, 'close', lambda fd: calls.append(('close', fd))) + monkeypatch.setattr(shadow_process, 'configure_shadow_limits', lambda *args: calls.append('limits')) + original = builtins.__import__ + def guarded(name, *args, **kwargs): + if name == 'shadow_worker': + assert calls == ['parent', 'lock', ('wall', 10000), 'limits'] + calls.append('import') + return SimpleNamespace(run=lambda source, destination: calls.append((source, destination))) + return original(name, *args, **kwargs) + monkeypatch.setattr(builtins, '__import__', guarded) + monkeypatch.setattr(sys, 'stdin', SimpleNamespace(buffer='source')) + monkeypatch.setattr(sys, 'stdout', SimpleNamespace(buffer='destination')) + assert shadow_process.main(['--parent-pid', '123', '--wall-ms', '10000', '--cpu-seconds', '1', '--address-space-bytes', '123']) == 0 + assert calls == ['parent', 'lock', ('wall', 10000), 'limits', 'import', ('source', 'destination'), ('close', 123)] + + +@pytest.mark.skipif(sys.platform != 'linux', reason='Kernel enforcement is Linux-only') +def test_real_linux_kernel_refuses_allocation_and_terminates_busy_worker(): + import os + from pathlib import Path + import signal + environment = {**os.environ, 'PYTHONPATH': str(Path(__file__).resolve().parent.parent)} + prefix = 'from common.biometrics.shadow_process import configure_shadow_limits\nconfigure_shadow_limits(1, 96 * 1024 * 1024)\n' + memory = subprocess.run([sys.executable, '-B', '-c', prefix + ''' +import resource +assert resource.getrlimit(resource.RLIMIT_AS) == (96 * 1024 * 1024,) * 2 +try: + bytearray(128 * 1024 * 1024) +except MemoryError: + print('allocation refused') +else: + raise AssertionError('allocation was not bounded') +'''], env=environment, capture_output=True, timeout=10) + assert memory.returncode == 0, memory.stderr + assert memory.stdout == b'allocation refused\n' + cpu = subprocess.run([sys.executable, '-B', '-c', prefix + 'while True: pass\n'], + env=environment, capture_output=True, timeout=10) + assert cpu.returncode == -signal.SIGXCPU + + +@pytest.mark.parametrize('parent', [0, -1, True, 1.5, '123', 2_147_483_648]) +def test_invalid_parent_pid_refused(monkeypatch, parent): + monkeypatch.setattr(sys, 'platform', 'linux') + with pytest.raises(ValueError): + shadow_process.configure_parent_death(parent) + + +@pytest.mark.parametrize('parents, result', [([122], 0), ([123, 122], 0), ([123], -1)]) +def test_parent_loss_and_kernel_refusal_fail_closed(monkeypatch, parents, result): + import ctypes + monkeypatch.setattr(sys, 'platform', 'linux') + values = iter(parents) + monkeypatch.setattr(shadow_process.os, 'getppid', lambda: next(values)) + def prctl(*args): + assert args == (1, shadow_process.signal.SIGKILL, 0, 0, 0) + return result + monkeypatch.setattr(ctypes, 'CDLL', lambda *args, **kwargs: SimpleNamespace(prctl=prctl)) + with pytest.raises((RuntimeError, OSError)): + shadow_process.configure_parent_death(123) + + +@pytest.mark.skipif(sys.platform != 'linux', reason='Parent-death signal is Linux-only') +def test_real_parent_death_kills_detached_shadow_worker(): + import os + import signal + import time + from pathlib import Path + environment = {**os.environ, 'PYTHONPATH': str(Path(__file__).resolve().parent.parent)} + child_code = "from common.biometrics.shadow_process import configure_parent_death; import os,sys,time; configure_parent_death(int(sys.argv[1])); print(os.getpid(), flush=True); time.sleep(60)" + parent_code = "import subprocess,sys,os,time; subprocess.Popen([sys.executable,'-B','-c',sys.argv[1],str(os.getpid())],start_new_session=True); time.sleep(60)" + parent = subprocess.Popen([sys.executable, '-B', '-c', parent_code, child_code], + env=environment, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + worker_pid = None + try: + import select + ready, _, _ = select.select([parent.stdout], [], [], 10) + assert ready, 'Worker did not arm parent-death protection' + worker_pid = int(parent.stdout.readline()) + parent.kill() + parent.wait(timeout=5) + deadline = time.monotonic() + 5 + while time.monotonic() < deadline: + try: + state = Path(f'/proc/{worker_pid}/stat').read_text().rsplit(')', 1)[1].split()[0] + except (FileNotFoundError, ProcessLookupError): + # The task can vanish after procfs open but before read (ESRCH). + break + if state == 'Z': + break # dead; reaping belongs to the container init/subreaper + time.sleep(0.02) + else: + raise AssertionError('Worker survived host death') + worker_pid = None + finally: + if parent.poll() is None: + parent.kill() + parent.wait(timeout=5) + if worker_pid is not None: + try: + os.kill(worker_pid, signal.SIGKILL) + except ProcessLookupError: + pass + parent.stdout.close() + parent.stderr.close() + + +def test_parent_protection_failure_prevents_limits_and_processing(monkeypatch): + def fail(pid): + raise OSError('private parent protection detail') + monkeypatch.setattr(shadow_process, 'configure_parent_death', fail) + monkeypatch.setattr(shadow_process, 'configure_shadow_limits', lambda *args: pytest.fail('limits reached after parent protection failed')) + monkeypatch.setattr(sys, 'stdin', None) + stderr = io.StringIO() + monkeypatch.setattr(sys, 'stderr', stderr) + assert shadow_process.main(['--parent-pid', '123', '--wall-ms', '10000', '--cpu-seconds', '1', '--address-space-bytes', '123']) == 1 + assert stderr.getvalue() == 'Shadow process failed\n' + + +@pytest.mark.parametrize('kind', ['symlink', 'hardlink', 'public', 'content']) +def test_unsafe_lock_files_refused(tmp_path, monkeypatch, kind): + import os + monkeypatch.setattr(sys, 'platform', 'linux') + path = tmp_path / 'worker.lock' + if kind == 'symlink': + target = tmp_path / 'target' + target.touch(mode=0o600) + path.symlink_to(target) + else: + path.touch(mode=0o600) + if kind == 'hardlink': + os.link(path, tmp_path / 'alias') + if kind == 'public': + path.chmod(0o644) + if kind == 'content': + path.write_text('not a lock') + with pytest.raises((OSError, RuntimeError)): + shadow_process.acquire_shadow_lock(path) + + +def test_lock_contention_and_release_do_not_delete_runtime_file(tmp_path, monkeypatch): + import os + monkeypatch.setattr(sys, 'platform', 'linux') + path = tmp_path / 'worker.lock' + first = shadow_process.acquire_shadow_lock(path) + try: + with pytest.raises(BlockingIOError): + shadow_process.acquire_shadow_lock(path) + finally: + os.close(first) + before = path.stat().st_ino + second = shadow_process.acquire_shadow_lock(path) + os.close(second) + assert path.stat().st_ino == before + assert path.read_bytes() == b'' + + +@pytest.mark.skipif(sys.platform != 'linux', reason='Kernel execution lock is Linux-only') +def test_real_linux_lock_survives_pause_and_releases_after_worker_death(): + import os + import select + import signal + import tempfile + from pathlib import Path + from common.biometrics.shadow_process import acquire_shadow_lock + environment = {**os.environ, 'PYTHONPATH': str(Path(__file__).resolve().parent.parent)} + with tempfile.TemporaryDirectory() as directory: + path = str(Path(directory) / 'worker.lock') + code = "from common.biometrics.shadow_process import acquire_shadow_lock; import sys,time; fd=acquire_shadow_lock(sys.argv[1]); print('locked',flush=True); time.sleep(60)" + worker = subprocess.Popen([sys.executable, '-B', '-c', code, path], env=environment, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + try: + ready, _, _ = select.select([worker.stdout], [], [], 10) + assert ready and worker.stdout.readline() == 'locked\n' + os.kill(worker.pid, signal.SIGSTOP) + try: + fd = acquire_shadow_lock(path) + except BlockingIOError: + pass + else: + os.close(fd) + raise AssertionError('Paused worker lost exclusive execution lock') + worker.kill() + worker.wait(timeout=5) + fd = acquire_shadow_lock(path) + os.close(fd) + finally: + if worker.poll() is None: + worker.kill() + worker.wait(timeout=5) + worker.stdout.close() + worker.stderr.close() + + +def test_lock_contention_prevents_numerical_startup(monkeypatch): + monkeypatch.setattr(shadow_process, 'configure_parent_death', lambda pid: None) + def busy(): + raise BlockingIOError('another worker owns the slot') + monkeypatch.setattr(shadow_process, 'acquire_shadow_lock', busy) + monkeypatch.setattr(shadow_process, 'configure_shadow_limits', lambda *args: pytest.fail('limits reached without execution slot')) + monkeypatch.setattr(sys, 'stdin', None) + stderr = io.StringIO() + monkeypatch.setattr(sys, 'stderr', stderr) + assert shadow_process.main(['--parent-pid', '123', '--wall-ms', '10000', '--cpu-seconds', '1', '--address-space-bytes', '123']) == 1 + assert stderr.getvalue() == 'Shadow process failed\n' + + +@pytest.mark.parametrize('budget', [0, -1, True, 1.5, '1000', 7 * 86_400_000 + 300_001]) +def test_invalid_worker_wall_budget_refused(monkeypatch, budget): + monkeypatch.setattr(sys, 'platform', 'linux') + with pytest.raises(ValueError): + shadow_process.configure_wall_deadline(budget) + + +def test_worker_wall_timer_uses_default_termination_signal(monkeypatch): + monkeypatch.setattr(sys, 'platform', 'linux') + calls = [] + monkeypatch.setattr(shadow_process.signal, 'signal', lambda *args: calls.append(args)) + monkeypatch.setattr(shadow_process.signal, 'pthread_sigmask', lambda *args: calls.append(args)) + monkeypatch.setattr(shadow_process.signal, 'setitimer', lambda *args: calls.append(args)) + shadow_process.configure_wall_deadline(1234) + assert calls == [(shadow_process.signal.SIGALRM, shadow_process.signal.SIG_DFL), + (shadow_process.signal.SIG_UNBLOCK, {shadow_process.signal.SIGALRM}), + (shadow_process.signal.ITIMER_REAL, 1.234)] + + +@pytest.mark.skipif(sys.platform != 'linux', reason='Worker wall timer is Linux-only') +def test_real_worker_wall_timer_terminates_idle_process_and_releases_slot(): + import os + import signal + import tempfile + import time + from pathlib import Path + from common.biometrics.shadow_process import acquire_shadow_lock + environment = {**os.environ, 'PYTHONPATH': str(Path(__file__).resolve().parent.parent)} + with tempfile.TemporaryDirectory() as directory: + path = str(Path(directory) / 'worker.lock') + code = "from common.biometrics.shadow_process import acquire_shadow_lock,configure_wall_deadline; import sys,time,signal; fd=acquire_shadow_lock(sys.argv[1]); signal.pthread_sigmask(signal.SIG_BLOCK,{signal.SIGALRM}); configure_wall_deadline(100); time.sleep(60)" + started = time.monotonic() + result = subprocess.run([sys.executable, '-B', '-c', code, path], env=environment, + capture_output=True, timeout=5) + assert result.returncode == -signal.SIGALRM + assert time.monotonic() - started < 5 + fd = acquire_shadow_lock(path) + os.close(fd) diff --git a/modules/common/test_worker_limits.py b/modules/common/test_worker_limits.py new file mode 100644 index 000000000..e922c5856 --- /dev/null +++ b/modules/common/test_worker_limits.py @@ -0,0 +1,81 @@ +import builtins +import subprocess +import sys +from types import SimpleNamespace + +import pytest + +from common.biometrics import replay_worker + + +def without_resource(monkeypatch): + original = builtins.__import__ + + def load(name, *args, **kwargs): + if name == 'resource': + raise ModuleNotFoundError("No module named 'resource'", name='resource') + return original(name, *args, **kwargs) + + monkeypatch.setattr(builtins, '__import__', load) + monkeypatch.setattr(replay_worker.sys, 'platform', 'linux') + monkeypatch.setattr(replay_worker.os, 'nice', lambda value: None) + + +def test_yocto_worker_sets_same_limits_on_its_own_pid(monkeypatch): + without_resource(monkeypatch) + calls = [] + monkeypatch.setattr(replay_worker.subprocess, 'run', lambda *args, **kwargs: calls.append((args, kwargs))) + replay_worker.configure_process_limits() + assert calls == [((['/usr/bin/prlimit', '--pid', str(replay_worker.os.getpid()), + '--core=0:0', '--cpu=120:121', '--nofile=64:64'],), { + 'check': True, 'timeout': 5, 'stdin': subprocess.DEVNULL, + 'stdout': subprocess.DEVNULL, 'stderr': subprocess.DEVNULL})] + + +@pytest.mark.parametrize('failure', [FileNotFoundError(), + subprocess.CalledProcessError(1, 'prlimit'), subprocess.TimeoutExpired('prlimit', 5)]) +def test_missing_or_failed_limit_utility_fails_closed(monkeypatch, failure): + without_resource(monkeypatch) + + def fail(*args, **kwargs): + raise failure + + monkeypatch.setattr(replay_worker.subprocess, 'run', fail) + with pytest.raises(type(failure)): + replay_worker.configure_process_limits() + + +def test_missing_resource_on_other_platform_does_not_use_linux_utility(monkeypatch): + without_resource(monkeypatch) + monkeypatch.setattr(replay_worker.sys, 'platform', 'darwin') + with pytest.raises(ModuleNotFoundError): + replay_worker.configure_process_limits() + + +@pytest.mark.parametrize('cpu_seconds', [120, 1800, 3600]) +def test_standard_resource_extension_sets_limits_without_external_command(monkeypatch, cpu_seconds): + limits = [] + monkeypatch.setitem(sys.modules, 'resource', SimpleNamespace(RLIMIT_CORE=4, + RLIMIT_CPU=0, RLIMIT_NOFILE=7, setrlimit=lambda key, value: limits.append((key, value)))) + monkeypatch.setattr(replay_worker.os, 'nice', lambda value: None) + + def unexpected(*args, **kwargs): + pytest.fail('The standard resource path must not spawn a utility') + + monkeypatch.setattr(replay_worker.subprocess, 'run', unexpected) + replay_worker.configure_process_limits(cpu_seconds) + assert limits == [(4, (0, 0)), (0, (cpu_seconds, cpu_seconds + 1)), (7, (64, 64))] + + +def test_configured_cpu_limit_reaches_yocto_utility(monkeypatch): + without_resource(monkeypatch) + calls = [] + monkeypatch.setattr(replay_worker.subprocess, 'run', lambda args, **kwargs: calls.append(args)) + replay_worker.configure_process_limits(1800) + assert '--cpu=1800:1801' in calls[0] + + +@pytest.mark.parametrize('value', [True, 0, -1, 1.5, 3601, float('inf'), None]) +def test_invalid_cpu_limit_fails_before_changing_process_limits(value): + with pytest.raises(ValueError, match='CPU budget'): + replay_worker.configure_process_limits(value) diff --git a/modules/piezo-processor/fixtures/default-processing-v1.json b/modules/piezo-processor/fixtures/default-processing-v1.json new file mode 100644 index 000000000..37f13a3aa --- /dev/null +++ b/modules/piezo-processor/fixtures/default-processing-v1.json @@ -0,0 +1,27 @@ +{ + "sourceCommit": "3e9027ed", + "numpy": "2.2.6", + "scipy": "1.15.3", + "seed": 20261012, + "sampleRateHz": 500, + "hr": [ + 66.51884700665188, + 0.9522735382682044 + ], + "breathing": 14.936519790888722, + "hrvIndex": 12.937515617181978, + "tracker": [ + 60, + 61, + 60.0, + 62, + 65.0, + 65, + 50.0, + 50.0, + 50.0, + 50.5, + null, + 50 + ] +} diff --git a/modules/piezo-processor/main.py b/modules/piezo-processor/main.py index 27015c5a9..7a55ada15 100644 --- a/modules/piezo-processor/main.py +++ b/modules/piezo-processor/main.py @@ -36,7 +36,6 @@ import threading from pathlib import Path from datetime import datetime, timezone -from collections import deque from dataclasses import dataclass from typing import Callable, Optional @@ -48,7 +47,23 @@ from common.side_mode import SingleSleeperMode from common.bed_presence import BedPresence import numpy as np -from scipy.signal import butter, sosfiltfilt, hilbert, find_peaks +from runtime_adapter import RuntimeCoordinator, RunContext +from common.biometrics.frames import decode_int32_samples as _int32_samples, LOST_SAMPLE +from common.biometrics.processing import ( + StatefulSideProcessor, HR_WINDOW_S, BREATHING_WINDOW_S, HRV_WINDOW_S, +) +from common.biometrics.routing import SingleSleeperRouter, VITALS_INTERVAL_S +from common.biometrics.gates import ( + FrzHealthPumpState as _FrzHealthPumpState, PumpGate as _PumpGate, + PresenceDetector, PUMP_ENERGY_MULTIPLIER, PUMP_CORRELATION_MIN, + PUMP_GUARD_S, PUMP_ACTIVE_RPM_MIN, PUMP_OFF_GUARD_S, + PUMP_COUPLING_STD_FACTOR, PUMP_COUPLING_ACR_THRESHOLD, +) +from common.biometrics.signal import ( + _bandpass, _autocorr_quality, _compute_autocorr, subharmonic_summation_hr, + HRTracker, compute_breathing_rate, compute_hrv, + ProcessingParameters, DEFAULT_PARAMETERS, WindowEstimator, +) # --------------------------------------------------------------------------- # Configuration @@ -62,26 +77,6 @@ "SLEEP_DETECTOR_STATE_PATH", str(BIOMETRICS_DB.parent / "sleep-detector-state.json"))) SAMPLE_RATE = 500 # Hz — piezo sensor sample rate -VITALS_INTERVAL_S = 60 # write a vitals row every N seconds -HR_WINDOW_S = 30 # seconds of data for heart rate calculation -BREATHING_WINDOW_S = 60 # seconds of data for breathing rate calculation -HRV_WINDOW_S = 300 # seconds of data for HRV (5-minute RMSSD) - -# Pump gating -PUMP_ENERGY_MULTIPLIER = 10.0 -PUMP_CORRELATION_MIN = 0.5 -PUMP_GUARD_S = 5.0 - -# Per-side pump activity threshold (frzHealth pump RPM). PumpGate catches -# broadband-symmetric energy spikes but misses two steady-state coupling -# modes: asymmetric (one pump on, observed 2026-05-02 Pod 3 — pump@2000RPM -# on right with left idle drove phantom HR/HRV/BR) and symmetric (both -# pumps on, observed 2026-05-05 Pod 5 — 1940/2004 rpm beat at 64/min lands -# in the cardiac band). Both are caught by the guard below. -PUMP_ACTIVE_RPM_MIN = 50 # pump considered "running" above this -PUMP_OFF_GUARD_S = 5.0 # trailing guard after own-side pump-off -PUMP_COUPLING_STD_FACTOR = 4.0 # raise enter_threshold by this factor -PUMP_COUPLING_ACR_THRESHOLD = 0.6 # require strong autocorr to enter # HR band: 0.8 Hz preserves fundamental of 48+ BPM; 8.5 Hz per PMC7582983 HR_BAND = (0.8, 8.5) @@ -172,7 +167,8 @@ def write_vitals(holder: "DBHolder", side: str, ts: datetime, breathing_rate: Optional[float], quality_score: float, flags: Optional[list] = None, - hr_raw: Optional[float] = None) -> bool: + hr_raw: Optional[float] = None, + run_context: Optional[RunContext] = None) -> bool: """Insert one vitals row + paired vitals_quality row. Logs and swallows sqlite3 errors so the main loop survives transient WAL/disk issues (#325). After _DB_RECONNECT_THRESHOLD consecutive failures, the holder's @@ -187,10 +183,21 @@ def write_vitals(holder: "DBHolder", side: str, ts: datetime, flags_json = json.dumps(flags) if flags else None try: with conn: - cur = conn.execute( - "INSERT INTO vitals (side, timestamp, heart_rate, hrv, breathing_rate) VALUES (?, ?, ?, ?, ?)", - (side, ts_unix, heart_rate, hrv, breathing_rate), - ) + if run_context is not None: + # Keep the profile and source channel that computed the value, + # even if single-sleeper mode changed while this row was held. + if side != run_context.side: + raise sqlite3.IntegrityError("Vitals routing disagrees with processing provenance") + run_context.ensure(conn) + cur = conn.execute( + "INSERT INTO vitals (side, timestamp, heart_rate, hrv, breathing_rate, processing_run_id) VALUES (?, ?, ?, ?, ?, ?)", + (side, ts_unix, heart_rate, hrv, breathing_rate, run_context.id), + ) + else: + cur = conn.execute( + "INSERT INTO vitals (side, timestamp, heart_rate, hrv, breathing_rate) VALUES (?, ?, ?, ?, ?)", + (side, ts_unix, heart_rate, hrv, breathing_rate), + ) conn.execute( "INSERT INTO vitals_quality (vitals_id, side, timestamp, quality_score, flags, hr_raw, created_at) " "VALUES (?, ?, ?, ?, ?, ?, ?)", @@ -219,91 +226,25 @@ class VitalsCandidate: quality_score: float flags: Optional[list] hr_raw: Optional[float] + run_context: Optional[RunContext] = None def write(self, holder: "DBHolder", side: Optional[str] = None) -> bool: - return write_vitals(holder, side or self.side, self.ts, self.heart_rate, + # Provenance captures the logical side at computation, not at retry. + destination = self.run_context.side if self.run_context else side or self.side + kwargs = {"run_context": self.run_context} if self.run_context else {} + return write_vitals(holder, destination, self.ts, self.heart_rate, self.hrv, self.breathing_rate, quality_score=self.quality_score, flags=self.flags, - hr_raw=self.hr_raw) - + hr_raw=self.hr_raw, **kwargs) -class SingleSleeperVitals: - """Routes vitals rows when one side is in away mode (single sleeper). - - A solo sleeper who rolls onto the away side is picked up by that side's - piezo too — the same heartbeat, within a few bpm. Each cycle's - candidates from the two sides are paired and - only the higher-quality one is written, under the home side, so the - home session shows one heart-rate series with no gaps or duplicates. - A candidate whose partner doesn't arrive within VITALS_INTERVAL_S (the - other side saw no one) is written alone under the home side. - - Outside single-sleeper mode rows are written under their own side. - """ +class SingleSleeperVitals(SingleSleeperRouter[VitalsCandidate]): + """Bind shared arbitration to the live database/provenance writer.""" def __init__(self, holder: "DBHolder", mode: SingleSleeperMode, clock: Callable[[], float] = time.monotonic): self._holder = holder - self._mode = mode - self._clock = clock - self._pending: Optional[VitalsCandidate] = None - self._retry: Optional[VitalsCandidate] = None - self._retry_home: Optional[str] = None - self._pending_home: Optional[str] = None - self._pending_at = 0.0 - - def submit(self, cand: VitalsCandidate) -> bool: - """Accept one side's candidate. True means the side may advance its - write cursor: the row was written, or is held (for pairing or a - write retry). A held candidate is only dropped once written.""" - # Completed intervals must never pair with a later candidate. - if self._retry is not None and not self.flush(): - return False - home = self._mode.home_side() - if not self._mode.active_sides(): - self._pending = self._retry = None - return True - if home is None: - if not self.flush(): - return False - return cand.write(self._holder) - pending = self._pending - if pending is not None and pending.side != cand.side and self._pending_home == home: - best = cand if cand.quality_score > pending.quality_score else pending - self._pending = best - self.flush() - return True - # A same-side repeat means its partner never arrived: write the - # older one first. If that fails, refuse the new candidate so its - # side keeps it and resubmits, rather than dropping the older row. - if not self.flush(): - return False - self._pending, self._pending_home, self._pending_at = cand, home, self._clock() - return True - - def tick(self) -> None: - """Write a held candidate whose partner didn't arrive in time.""" - if not self._mode.active_sides(): - self._pending = self._retry = None - return - held = self._pending is not None or self._retry is not None - if held and self._clock() - self._pending_at >= VITALS_INTERVAL_S: - self.flush() - - def flush(self) -> bool: - """Write the held candidate, if any. On failure it stays held and - idle retries wait VITALS_INTERVAL_S; a new submission retries first. - True when nothing is left held.""" - if self._retry is None: - self._retry, self._retry_home = self._pending, self._pending_home - self._pending = None - if self._retry is None: - return True - if self._retry.write(self._holder, side=self._retry_home): - self._retry = None - return True - self._pending_at = self._clock() - return False + super().__init__(mode, lambda candidate, side: candidate.write(holder, side=side), + clock=clock) def report_health(status: str, message: str) -> None: @@ -326,735 +267,37 @@ def report_health(status: str, message: str) -> None: except Exception as e: log.warning("Could not write health status: %s", e) -# --------------------------------------------------------------------------- -# Signal processing — filters -# --------------------------------------------------------------------------- - -def _bandpass(sig: np.ndarray, lo: float, hi: float, fs: float, - order: int = 4) -> np.ndarray: - """Bandpass filter using SOS form for numerical stability.""" - sos = butter(order, [lo, hi], btype='band', fs=fs, output='sos') - return sosfiltfilt(sos, sig) - # --------------------------------------------------------------------------- # Pump gating — streaming adaptation (Shin et al. IEEE TBME 2009) # --------------------------------------------------------------------------- -class FrzHealthPumpState: - """Tracks per-side pump RPM from frzHealth records. - - Used by SideProcessor to suppress phantom presence detection when - only the same side's pump is running (asymmetric pump-coupling - artifact). Trailing guard period mirrors the post-pump-off ringing - in the piezo channel. - """ - - def __init__(self): - self._pump_rpm = {"left": 0.0, "right": 0.0} - # monotonic seconds when each side's pump last turned off; -inf = - # never (0.0 would falsely assert the guard for the first - # PUMP_OFF_GUARD_S when time.monotonic() has a near-zero origin) - self._pump_off_at = {"left": float("-inf"), "right": float("-inf")} - self._was_active = {"left": False, "right": False} - - def update(self, record: dict) -> None: - if record.get("type") != "frzHealth": - return - for side in ("left", "right"): - data = record.get(side) - if not isinstance(data, dict): - continue - rpm = 0.0 - for key in ("pumpRpm", "pump_rpm", "pumpRPM", "rpm"): - val = data.get(key) - if val is not None: - try: - rpm = float(val) - except (TypeError, ValueError): - rpm = 0.0 - break - # New NATS firmware nests the same reading at side.pump.rpm. - pump = data.get("pump") - if rpm == 0 and isinstance(pump, dict): - val = pump.get("rpm") - if val is not None: - try: - rpm = float(val) - except (TypeError, ValueError): - rpm = 0.0 - if rpm == 0: - duty = next((data.get(key) for key in - ("pumpDuty", "pump_duty", "duty") - if data.get(key) is not None), None) - if duty is None and isinstance(pump, dict): - duty = next((pump.get(key) for key in ("duty", "power") - if pump.get(key) is not None), None) - if duty is not None: - try: - rpm = PUMP_ACTIVE_RPM_MIN + 1.0 if float(duty) > 0 else 0.0 - except (TypeError, ValueError): - rpm = 0.0 - - now_active = rpm >= PUMP_ACTIVE_RPM_MIN - if self._was_active[side] and not now_active: - self._pump_off_at[side] = time.monotonic() - self._pump_rpm[side] = rpm - self._was_active[side] = now_active - - def is_side_pump_active(self, side: str) -> bool: - """Pump active = currently running OR within the trailing guard window.""" - if self._pump_rpm[side] >= PUMP_ACTIVE_RPM_MIN: - return True - return time.monotonic() - self._pump_off_at[side] < PUMP_OFF_GUARD_S - - def is_asymmetric_for(self, side: str) -> bool: - """Own pump active, opposite pump idle (and outside its own guard). - This is the configuration that produces phantom presence: pump - vibration on the powered side without competing signal on the - other side to balance/reject.""" - other = "right" if side == "left" else "left" - return self.is_side_pump_active(side) and not self.is_side_pump_active(other) - - def is_symmetric_active(self) -> bool: - """Both pumps active. RPM mismatch (Pod 5 live: 1940/2004) produces - a beat frequency in the cardiac band, generating bilateral phantom - vitals. PumpGate catches symmetric spikes but not steady-state.""" - return self.is_side_pump_active("left") and self.is_side_pump_active("right") - - -class PumpGate: - """Detects pump activity from dual-channel energy spikes. - - In streaming mode, each incoming record's L+R chunks are checked. - If a pump is detected, a guard period is set during which all - subsequent records are dropped (not ingested into side buffers). - """ - - def __init__(self, fs: float = SAMPLE_RATE): - self._fs = fs - self._baseline: Optional[float] = None - self._pump_until: float = 0.0 # monotonic seconds when guard expires - - def is_pump_active(self) -> bool: - return time.monotonic() < self._pump_until - - def check(self, left_chunk: np.ndarray, right_chunk: np.ndarray) -> bool: - """Check a single record's L+R chunks for pump activity. - - Returns True if the record should be DROPPED (pump detected or - guard period active), False if the record is clean. - """ - # If we're still in the guard period, drop - if self.is_pump_active(): - return True - - # Compute energy for both channels - le = float(np.mean(left_chunk.astype(np.float64) ** 2)) - re = float(np.mean(right_chunk.astype(np.float64) ** 2)) - - if le == 0 or re == 0: - return False - - ratio = min(le, re) / max(le, re) - - if self._baseline is not None and self._baseline > 0: - spike = max(le, re) / self._baseline - if ratio > PUMP_CORRELATION_MIN and spike > PUMP_ENERGY_MULTIPLIER: - # Pump detected — set guard period - self._pump_until = time.monotonic() + PUMP_GUARD_S - log.debug("Pump detected (spike=%.1f, ratio=%.2f), " - "guard until +%.0fs", spike, ratio, PUMP_GUARD_S) - return True - - # Update baseline with exponential moving average (only on clean data). - # Guard: if first record has unusually high energy (module started during - # pump), don't initialize baseline from it — wait for a quieter sample. - avg = (le + re) / 2 - if self._baseline is None: - self._baseline = avg - self._baseline_samples = 1 - elif avg < self._baseline * 3: - self._baseline = 0.95 * self._baseline + 0.05 * avg - self._baseline_samples = getattr(self, '_baseline_samples', 0) + 1 - # If early samples are settling, allow baseline to drop faster - if self._baseline_samples < 10: - self._baseline = min(self._baseline, avg) - - return False - -# --------------------------------------------------------------------------- -# Presence detection — hysteresis state machine (Paalasmaa et al. 2012) -# --------------------------------------------------------------------------- - -def _autocorr_quality(sig: np.ndarray, fs: float = SAMPLE_RATE) -> float: - """Peak height of best autocorrelation peak in cardiac range. - - Person on bed produces periodic signal with high peak. - Empty bed produces noise with low peak. - """ - try: - filtered = _bandpass(sig, 0.8, 8.5, fs) - seg = filtered[:int(min(len(filtered), 10 * fs))] - seg = seg - np.mean(seg) - norm = np.std(seg) - if norm < 1e-10: - return 0.0 - seg = seg / norm - - n = len(seg) - fft_size = 1 - while fft_size < 2 * n: - fft_size *= 2 - fft_f = np.fft.rfft(seg, n=fft_size) - acr = np.fft.irfft(fft_f * np.conj(fft_f), n=fft_size)[:n] - if acr[0] == 0: - return 0.0 - acr = acr / acr[0] +class FrzHealthPumpState(_FrzHealthPumpState): + """Live adapter; replay imports the shared gate with its event clock.""" + def __init__(self, *, clock=None): + super().__init__(clock=clock if clock is not None else lambda: time.monotonic()) - min_lag = int(fs * 60 / 150) - max_lag = min(int(fs * 60 / 40), len(acr) - 1) - if max_lag <= min_lag: - return 0.0 - return float(max(np.max(acr[min_lag:max_lag + 1]), 0.0)) - except Exception: - return 0.0 +class PumpGate(_PumpGate): + """Live adapter retaining the historical no-argument constructor.""" + def __init__(self, fs: float = SAMPLE_RATE, *, clock=None): + super().__init__(fs, clock=clock if clock is not None else lambda: time.monotonic()) -class PresenceDetector: - """Hysteresis-based presence detection with autocorrelation quality. - - Uses median std of 1-10 Hz filtered signal in 5-second windows as - the primary feature, with autocorrelation quality as secondary. - Requires 3 consecutive low windows to exit PRESENT state. - """ - ABSENT = 0 - PRESENT = 1 - - def __init__(self): - self.state = self.ABSENT - self.consecutive_low = 0 - self.enter_threshold = 400_000 - self.exit_threshold = 150_000 - self.exit_count = 3 # 3 consecutive low windows to declare absent - self.acr_threshold = 0.45 - - def update(self, window_std: float, acr_qual: float) -> bool: - """Update state and return True if user is present.""" - if self.state == self.ABSENT: - if window_std > self.enter_threshold or acr_qual > self.acr_threshold: - self.state = self.PRESENT - self.consecutive_low = 0 - return True - return False - else: - if window_std < self.exit_threshold and acr_qual < self.acr_threshold * 0.5: - self.consecutive_low += 1 - if self.consecutive_low >= self.exit_count: - self.state = self.ABSENT - self.consecutive_low = 0 - return False - return True # still present (hysteresis) - else: - self.consecutive_low = 0 - return True - -# --------------------------------------------------------------------------- -# Heart rate — subharmonic summation autocorrelation (Hermes 1988; Bruser 2011) -# --------------------------------------------------------------------------- - -def _compute_autocorr(filtered: np.ndarray, fs: float) -> Optional[np.ndarray]: - """Compute normalised autocorrelation via FFT.""" - n = len(filtered) - fft_size = 1 - while fft_size < 2 * n: - fft_size *= 2 - fft_f = np.fft.rfft(filtered, n=fft_size) - acr = np.fft.irfft(fft_f * np.conj(fft_f), n=fft_size)[:n] - if acr[0] == 0: - return None - return acr / acr[0] - - -def subharmonic_summation_hr(samples: np.ndarray, fs: float = SAMPLE_RATE, - bpm_range: tuple = (45, 120), - n_harmonics: int = 3): - """Subharmonic summation for robust fundamental HR detection. - - Only scores autocorrelation PEAKS (not every lag) to avoid edge effects - where long lags accumulate noise. For each candidate peak lag L, - score = weighted sum of autocorr at L, L/2, L/3. - - Hermes 1988 (JASA), adapted for BCG by Bruser et al. 2011. - - Returns (heart_rate_bpm, score) or (None, 0.0) on failure. - """ - filtered = _bandpass(samples, HR_BAND[0], HR_BAND[1], fs) - acr = _compute_autocorr(filtered, fs) - if acr is None: - return None, 0.0 - - min_lag = int(fs * 60 / bpm_range[1]) - max_lag = int(fs * 60 / bpm_range[0]) - max_lag = min(max_lag, len(acr) - 1) - - if max_lag <= min_lag: - return None, 0.0 - - # Find actual peaks in autocorrelation (not every lag) - search = acr[min_lag:max_lag + 1] - peaks, _props = find_peaks(search, height=0.02, distance=int(fs * 0.15)) - if len(peaks) == 0: - return None, 0.0 - - candidate_lags = peaks + min_lag - weights = [1.0, 0.8, 0.6][:n_harmonics] - - best_lag = None - best_score = 0.0 - - for lag in candidate_lags: - score = 0.0 - for k in range(n_harmonics): - exact_lag = lag / (k + 1) - lag_lo = int(exact_lag) - lag_hi = lag_lo + 1 - if lag_hi >= len(acr): - continue - frac = exact_lag - lag_lo - val = acr[lag_lo] * (1 - frac) + acr[lag_hi] * frac - score += weights[k] * max(val, 0) - if score > best_score: - best_score = score - best_lag = lag - - if best_lag is None or best_score < 0.1: - return None, best_score - - hr = 60.0 * fs / best_lag - return hr, best_score - - -class HRTracker: - """Inter-window HR tracking with Gaussian consistency constraint. - - HR cannot jump > max_delta BPM between consecutive windows at rest. - Catches residual harmonic escapes via half/double correction. - Bruser et al. 2011. - """ - - # After this many consecutive uncorrectable readings, re-seed history at - # the new level. A lone outlier must not poison the median, but a - # sustained shift (e.g. different occupant) previously froze the tracker: - # the stale median rejected every new reading forever. - FALLBACK_RESET_STREAK = 3 - - def __init__(self, max_delta: float = 15.0, history_len: int = 5): - # Bounded deque: only the last history_len entries are ever consulted, - # so retaining the full list would leak ~1440 floats/day (#325). - self.history: deque = deque(maxlen=history_len) - self.max_delta = max_delta - self.history_len = history_len - self._fallback_streak = 0 - - def update(self, hr_candidate: Optional[float], - score: float) -> Optional[float]: - if hr_candidate is None: - return None - - if not self.history: - self.history.append(hr_candidate) - return hr_candidate - - recent = float(np.median(list(self.history))) - delta = abs(hr_candidate - recent) - - # Gaussian consistency weight - consistency = np.exp(-0.5 * (delta / self.max_delta) ** 2) - if consistency > 0.3: - self._fallback_streak = 0 - self.history.append(hr_candidate) - return hr_candidate - - # Try half (harmonic correction) - half_hr = hr_candidate / 2 - if 40 <= half_hr <= 120: - half_delta = abs(half_hr - recent) - half_consistency = np.exp(-0.5 * (half_delta / self.max_delta) ** 2) - if half_consistency > 0.3: - self._fallback_streak = 0 - self.history.append(half_hr) - return half_hr - - # Try double (sub-harmonic correction) - double_hr = hr_candidate * 2 - if 40 <= double_hr <= 180: - double_delta = abs(double_hr - recent) - double_consistency = np.exp( - -0.5 * (double_delta / self.max_delta) ** 2) - if double_consistency > 0.3: - self._fallback_streak = 0 - self.history.append(double_hr) - return double_hr - - # Accept anyway but keep a lone outlier out of history. A sustained - # streak of uncorrectable readings means the tracked level itself is - # stale (e.g. different occupant) — previously the frozen median then - # rejected every reading forever, so re-seed at the new level. - self._fallback_streak += 1 - if self._fallback_streak >= self.FALLBACK_RESET_STREAK: - self.history.clear() - self.history.append(hr_candidate) - self._fallback_streak = 0 - return hr_candidate - -# --------------------------------------------------------------------------- -# Breathing rate — Hilbert envelope (PMC9354426) -# --------------------------------------------------------------------------- - -def compute_breathing_rate(samples: np.ndarray, - fs: float = SAMPLE_RATE) -> Optional[float]: - """Breathing rate via Hilbert envelope of the cardiac band. - - Pipeline: bandpass 0.8-10 Hz -> Hilbert transform -> amplitude envelope - -> bandpass 0.1-0.7 Hz -> peak counting. - """ - try: - cardiac = _bandpass(samples, 0.8, 10.0, fs) - envelope = np.abs(hilbert(cardiac)) - resp = _bandpass(envelope, 0.1, 0.7, fs) - peaks, _ = find_peaks(resp, distance=int(2.0 * fs)) - if len(peaks) < 3: - return None - intervals = np.diff(peaks) / fs - med = np.median(intervals) - valid = intervals[(intervals > med * 0.5) & (intervals < med * 1.5)] - if len(valid) < 2: - return None - br = 60.0 / float(np.mean(valid)) - return br if 6 <= br <= 30 else None - except Exception as e: - log.debug("Breathing rate computation failed: %s", e) - return None - -# --------------------------------------------------------------------------- -# HR variability index — window-level IBI with harmonic gate -# -# NOTE: This is NOT clinical RMSSD. It computes successive differences of -# window-level IBI estimates (~59 per 5 min with 10s windows), not beat-to-beat -# intervals (~300+ per 5 min). The output measures HR stability across windows, -# not vagal beat-to-beat modulation. See #221 for the beat-level upgrade path. -# --------------------------------------------------------------------------- - -# Harmonic tolerance for IBI gating (18% per DSP expert recommendation) -_HRV_HARMONIC_TOL = 0.18 -# Sub-window length for IBI estimation (10s per DSP expert recommendation — -# gives ~59 windows per 5 min vs ~19 with 30s windows) -_HRV_WINDOW_S = 10 -# Minimum accepted IBI estimates after filtering -_HRV_MIN_IBIS = 10 -# Maximum gap between consecutive accepted windows before skipping the diff -_HRV_MAX_GAP_WINDOWS = 3 - - -def _harmonic_gate(ibi_ms: float, tracker_ibi_ms: float) -> Optional[float]: - """Reject or correct IBIs that are harmonic multiples of the expected value. - - Returns the accepted/corrected IBI in ms, or None to discard. - """ - if tracker_ibi_ms <= 0: - return ibi_ms # no prior yet — accept as-is - - for multiplier in [1.0, 2.0, 0.5, 3.0, 1.0 / 3.0]: - candidate = tracker_ibi_ms * multiplier - if candidate > 0 and abs(ibi_ms - candidate) / candidate < _HRV_HARMONIC_TOL: - if multiplier == 1.0: - return ibi_ms # fundamental — accept - else: - return tracker_ibi_ms # harmonic — substitute tracker value - return None # outside all harmonic relationships — discard - - -def compute_hrv(samples: np.ndarray, - fs: float = SAMPLE_RATE) -> Optional[float]: - """HR variability index via window-level IBI with harmonic gate. - - Pipeline: - 1. Bandpass 0.8-8.5 Hz - 2. 10s sub-windows with 50% overlap (~59 windows per 5 min) - 3. SHS autocorrelation per window → raw IBI - 4. Harmonic gate: reject/correct IBIs using trimmed-mean tracker - 5. Hampel filter on accepted IBI series - 6. Gap-aware RMSSD (skip diffs across rejected-window gaps) - 7. Range gate: 5-100 ms - - NOT clinical RMSSD — see module docstring and #221. - """ - try: - filtered = _bandpass(samples, HR_BAND[0], HR_BAND[1], fs) - sub_window = int(_HRV_WINDOW_S * fs) - step = sub_window // 2 - min_lag = int(fs * 60 / 150) - max_lag = int(fs * 60 / 40) - - # --- Pass 1: SHS autocorrelation per window → raw IBIs --- - raw_ibis: list = [] # (window_index, ibi_ms) - for idx, start in enumerate( - range(0, len(filtered) - sub_window + 1, step)): - chunk = filtered[start:start + sub_window] - acr = _compute_autocorr(chunk, fs) - if acr is None: - continue - search = acr[min_lag:min(max_lag + 1, len(acr))] - if len(search) == 0: - continue - - # SHS scoring — find peaks first, then score - peaks, _props = find_peaks(search, height=0.02, - distance=int(fs * 0.15)) - if len(peaks) == 0: - continue - - candidate_lags = peaks + min_lag - weights = [1.0, 0.8, 0.6] - best_lag, best_score = None, 0.0 - for lag in candidate_lags: - score = 0.0 - for k in range(3): - exact = lag / (k + 1) - lo_i = int(exact) - hi_i = lo_i + 1 - if hi_i >= len(acr): - continue - frac = exact - lo_i - val = acr[lo_i] * (1 - frac) + acr[hi_i] * frac - score += weights[k] * max(val, 0) - if score > best_score: - best_score = score - best_lag = lag - - if best_lag is None or best_score < 0.1: - continue - ibi_ms = best_lag / fs * 1000 - if 400 <= ibi_ms <= 1500: - raw_ibis.append((idx, ibi_ms)) - - if len(raw_ibis) < _HRV_MIN_IBIS: - return None - - # --- Pass 2: Harmonic gate with trimmed-mean tracker --- - tracker_history: list = [] - gated: list = [] # (window_index, ibi_ms) - - for win_idx, ibi_ms in raw_ibis: - # Compute tracker IBI as trimmed mean of last 5 accepted - if len(tracker_history) >= 3: - sorted_h = sorted(tracker_history[-5:]) - # Drop min+max if we have enough - if len(sorted_h) >= 4: - trimmed = sorted_h[1:-1] - else: - trimmed = sorted_h - tracker_ibi = sum(trimmed) / len(trimmed) - elif tracker_history: - tracker_ibi = sum(tracker_history) / len(tracker_history) - else: - tracker_ibi = 0 # no prior — accept first value - - accepted = _harmonic_gate(ibi_ms, tracker_ibi) - if accepted is not None: - gated.append((win_idx, accepted)) - tracker_history.append(accepted) - - if len(gated) < _HRV_MIN_IBIS: - return None - - # --- Pass 3: Hampel filter on gated IBI series --- - ibi_values = np.array([ibi for _, ibi in gated]) - win_indices = [idx for idx, _ in gated] - clean_mask = np.ones(len(ibi_values), dtype=bool) - - for i in range(len(ibi_values)): - lo_i = max(0, i - 3) - hi_i = min(len(ibi_values), i + 4) - local = ibi_values[lo_i:hi_i] - med = np.median(local) - mad = max(np.median(np.abs(local - med)), 1e-6) - if abs(ibi_values[i] - med) > 3.0 * 1.4826 * mad: - clean_mask[i] = False - - clean_ibis = ibi_values[clean_mask] - clean_indices = [win_indices[i] for i in range(len(win_indices)) - if clean_mask[i]] - - if len(clean_ibis) < _HRV_MIN_IBIS: - return None - - # --- Pass 4: Gap-aware RMSSD --- - sq_diffs: list = [] - for i in range(1, len(clean_ibis)): - gap = clean_indices[i] - clean_indices[i - 1] - if gap >= _HRV_MAX_GAP_WINDOWS: - continue # skip diff across rejected-window gap - diff = clean_ibis[i] - clean_ibis[i - 1] - sq_diffs.append(diff ** 2) - - if len(sq_diffs) < 5: - return None - - hrv_index = float(np.sqrt(np.mean(sq_diffs))) - # Range gate: 5-100 ms. Window-level HRV above 100 ms is artifact - # even after harmonic correction (cardiologist recommendation). - return hrv_index if 5 <= hrv_index <= 100 else None - except Exception as e: - log.debug("HRV computation failed: %s", e) - return None # --------------------------------------------------------------------------- # Per-side processor # --------------------------------------------------------------------------- -class SideProcessor: +class SideProcessor(StatefulSideProcessor): + """Live adapter binds shared state to wall time and canonical DB writes.""" def __init__(self, side: str, db_holder: DBHolder, - pump_state: Optional[FrzHealthPumpState] = None): - self.side = side + pump_state: Optional[FrzHealthPumpState] = None, + parameters: ProcessingParameters = DEFAULT_PARAMETERS): self.db_holder = db_holder - self._hr_buf: deque = deque(maxlen=HR_WINDOW_S * SAMPLE_RATE) - self._hrv_buf: deque = deque(maxlen=HRV_WINDOW_S * SAMPLE_RATE) - self._br_buf: deque = deque(maxlen=BREATHING_WINDOW_S * SAMPLE_RATE) - self._last_write = 0.0 - self._presence = PresenceDetector() - self._hr_tracker = HRTracker() - self._other: Optional['SideProcessor'] = None # set after both sides created - self._last_med_std: float = 0.0 # cached for cross-channel comparison - self._last_acr_qual: float = 0.0 - self._pump_state = pump_state - # Where computed vitals go; None writes them under this side. - # main() routes both sides through SingleSleeperVitals.submit. - self.sink: Optional[Callable[[VitalsCandidate], bool]] = None - # The bed's own answer to "is anyone here" (sleep-detector presence): - # False vetoes piezo presence, None (unknown) leaves it alone. - self.bed_occupied: Optional[Callable[[], Optional[bool]]] = None - - def ingest(self, samples: np.ndarray) -> None: - self._hr_buf.extend(samples) - self._hrv_buf.extend(samples) - self._br_buf.extend(samples) - self._maybe_write() - - def _maybe_write(self) -> None: - now = time.time() - if now - self._last_write < VITALS_INTERVAL_S: - return - - hr_arr = np.array(self._hr_buf) - if len(hr_arr) < int(10 * SAMPLE_RATE): - return # Not enough data yet - - # --- Presence detection (hysteresis + autocorrelation quality) --- - filt = _bandpass(hr_arr, 1.0, 10.0, SAMPLE_RATE) - w = int(5 * SAMPLE_RATE) - stds = [np.std(filt[j:j + w]) - for j in range(0, len(filt) - w + 1, w)] - med_std = float(np.median(stds)) if stds else 0.0 - acr_qual = _autocorr_quality(hr_arr) - - # Cross-channel rejection: if the other side has stronger or similar - # signal AND strong autocorrelation (someone is actually there), this - # side's signal is likely vibration coupling, not a person. - # A real person produces ~3-5x stronger signal on their side. - # - # Note: due to sequential processing in main() (left before right), - # left sees right's values from the previous cycle (~60s stale). - # This is acceptable — coupling produces similar energy regardless - # of which side checks first. - if self._other is not None and self._other._last_med_std > 0: - other_std = self._other._last_med_std - other_acr = self._other._last_acr_qual - # Only suppress if: other side has real periodicity (someone there), - # other side's energy is >= ours, and our energy is below threshold - if (other_acr > 0.3 - and other_std > med_std * 0.7 - and med_std < self._presence.enter_threshold): - acr_qual = 0.0 - - # Cache AFTER suppression so the other side sees post-suppression values - self._last_med_std = med_std - self._last_acr_qual = acr_qual - - # Pump-coupling guard. PumpGate handles symmetric spikes (ramps) - # but misses two steady-state modes: asymmetric (own-pump-on with - # opposite idle — vibration couples into same-side piezo, no - # competing signal for cross-channel rejection) and symmetric - # (both pumps on at mismatched RPMs — beat frequency lands in the - # cardiac band, both sides look "real"). In both, require elevated - # energy AND strong autocorrelation to enter PRESENT; coupling - # rarely produces both, only a real person does. - if self._pump_state is not None and self._presence.state == PresenceDetector.ABSENT: - symmetric = self._pump_state.is_symmetric_active() - asymmetric = self._pump_state.is_asymmetric_for(self.side) - if symmetric or asymmetric: - std_threshold = self._presence.enter_threshold * PUMP_COUPLING_STD_FACTOR - if not (med_std > std_threshold and acr_qual > PUMP_COUPLING_ACR_THRESHOLD): - log.debug( - "%s: pump-coupling guard suppressed presence " - "(med_std=%.0f, acr=%.2f, mode=%s)", - self.side, med_std, acr_qual, - "symmetric" if symmetric else "asymmetric", - ) - return - - present = self._presence.update(med_std, acr_qual) - # Piezo presence is vibration energy and rhythm; a prime or the pump - # shaking an empty bed passes it. Trust the capacitance-based - # presence when the sleep-detector says nobody is there. - if present and self.bed_occupied is not None and self.bed_occupied() is False: - present = False - - if not present: - # Reset the interval cursor so a return from extended absence - # doesn't trigger burst processing on every ingest until the - # first write completes (#325). - self._last_write = now - return # No user detected — skip - - # --- Heart rate (subharmonic summation + tracking) --- - hr_raw, hr_score = subharmonic_summation_hr(hr_arr) - hr = self._hr_tracker.update(hr_raw, hr_score) - - # --- Breathing rate (Hilbert envelope) --- - br = compute_breathing_rate(np.array(self._br_buf)) - - # --- HRV (sub-window autocorrelation IBI) --- - hrv_arr = np.array(self._hrv_buf) - hrv = compute_hrv(hrv_arr) if len(hrv_arr) >= int( - HRV_WINDOW_S * SAMPLE_RATE) else None - - if hr is not None or hrv is not None or br is not None: - ts = datetime.now(timezone.utc) - snr = max(0.0, min(1.0, acr_qual)) - hr_conf = max(0.0, min(1.0, hr_score)) if hr is not None else 0.0 - quality = round(0.55 * snr + 0.45 * hr_conf, 3) - flags = [] - if hr is None: - flags.append("no_hr") - if hrv is None: - flags.append("no_hrv") - if br is None: - flags.append("no_br") - if med_std < self._presence.enter_threshold: - flags.append("low_signal") - cand = VitalsCandidate(self.side, ts, hr, hrv, br, quality, flags or None, hr_raw) - wrote = self.sink(cand) if self.sink is not None else cand.write(self.db_holder) - log.info("vitals %s — HR=%.1f HRV=%.1f BR=%.1f q=%.2f", self.side, - hr or 0, hrv or 0, br or 0, quality) - # Only advance the downsample cursor when the write actually - # committed — otherwise a transient sqlite error would skip - # VITALS_INTERVAL_S of valid samples. - if wrote: - self._last_write = now - else: - # No metric to write — advance so we don't recompute on every ingest. - self._last_write = now + super().__init__(side, clock=lambda: time.time(), + writer=lambda candidate: candidate.write(db_holder), + candidate_factory=VitalsCandidate, pump_state=pump_state, + parameters=parameters, + autocorr_quality=lambda samples: _autocorr_quality(samples)) # --------------------------------------------------------------------------- # Main loop @@ -1065,35 +308,6 @@ def _maybe_write(self) -> None: # in a few percent of records. Taken as data it is a spike ~450x any real # signal: the pump gate drops the record (and 5 s after it), and the beat # tracker restarts. -LOST_SAMPLE = np.iinfo(np.int32).max - - -def _int32_samples(buf) -> np.ndarray: - """Decode an int32 sample buffer, tolerating truncated tails and - repairing lost-sample markers. - - A RAW record cut mid-write can carry a payload whose length is not a - multiple of 4; np.frombuffer would raise ValueError and kill the module. - Truncate to whole samples instead. A LOST_SAMPLE marker is replaced by - linear interpolation between its neighbours (the nearest valid sample at - a record edge); a record with no valid sample decodes as empty. - """ - if not isinstance(buf, (bytes, bytearray, memoryview)): - return np.empty(0, dtype=np.int32) - usable = len(buf) - (len(buf) % 4) - if usable == 0: - return np.empty(0, dtype=np.int32) - samples = np.frombuffer(buf[:usable], dtype=np.int32) - lost = samples == LOST_SAMPLE - if not lost.any(): - return samples - if lost.all(): - return np.empty(0, dtype=np.int32) - idx = np.arange(samples.size) - repaired = samples.astype(np.float64) - repaired[lost] = np.interp(idx[lost], idx[~lost], repaired[~lost]) - return np.round(repaired).astype(np.int32) - def main() -> None: log.info("Starting piezo-processor v2 (biometrics.db=%s)", BIOMETRICS_DB) @@ -1119,6 +333,9 @@ def main() -> None: bed = BedPresence(SLEEP_DETECTOR_STATE) left.bed_occupied = lambda: bed.occupied(bed_mode.home_side() or "left") right.bed_occupied = lambda: bed.occupied(bed_mode.home_side() or "right") + left.home_side = right.home_side = bed_mode.home_side + runtime = RuntimeCoordinator(db_holder, {"left": left, "right": right}) + runtime_problem = None # Source selected once at startup: NatsFollower on new-firmware pods (NATS # reachable), else the unchanged .RAW tailer. Same decoded-record contract. follower = create_follower(RAW_DATA_DIR, _shutdown, poll_interval=0.01) @@ -1157,8 +374,16 @@ def main() -> None: if pump_gate.check(l_samples, r_samples): continue + frame_time_ms = int(time.time() * 1000) + runtime.before_frame(bed_mode.home_side(), bed.occupied, frame_time_ms) left.ingest(l_samples) right.ingest(r_samples) + runtime.after_frame(frame_time_ms) + if runtime.problem != runtime_problem: + runtime_problem = runtime.problem + report_health("degraded" if runtime_problem else "healthy", + "Biometric configuration fallback: " + runtime_problem if runtime_problem + else "Biometric runtime configuration available") except Exception as e: log.exception("Fatal error in main loop: %s", e) diff --git a/modules/piezo-processor/manifest.json b/modules/piezo-processor/manifest.json index e44925d26..5c3b3e651 100644 --- a/modules/piezo-processor/manifest.json +++ b/modules/piezo-processor/manifest.json @@ -2,8 +2,17 @@ "name": "piezo-processor", "version": "1.0.0", "description": "Heart rate, HRV, and breathing rate from piezoelectric sensors", - "provides": ["vitals.heartRate", "vitals.hrv", "vitals.breathingRate"], - "writes": ["vitals"], + "provides": [ + "vitals.heartRate", + "vitals.hrv", + "vitals.breathingRate" + ], + "writes": [ + "vitals", + "vitals_quality", + "biometric_processing_runs", + "biometric_runtime_receipts" + ], "service": "sleepypod-piezo-processor.service", "language": "python", "minVersion": "1.0.0" diff --git a/modules/piezo-processor/runtime_adapter.py b/modules/piezo-processor/runtime_adapter.py new file mode 100644 index 000000000..aa0276cec --- /dev/null +++ b/modules/piezo-processor/runtime_adapter.py @@ -0,0 +1,185 @@ +"""Live configuration adoption; no hardware control or desired-state writes.""" +from dataclasses import dataclass +import json +import logging +import sqlite3 +import time +from uuid import uuid4 + +from common.biometrics.config import config_hash, manifest +from common.biometrics.runtime import BoundarySelection, RuntimeSnapshot, read_latest_snapshot +from common.biometrics.signal import ProcessingParameters + +log = logging.getLogger(__name__) +COMPONENTS = ("heart_rate", "breathing_rate", "hrv_index") + + +@dataclass(frozen=True) +class RunContext: + id: str + side: str + source_side: str + bundle_json: str + config_hash: str + started_at_ms: int + + def ensure(self, connection): + """Called inside the observation transaction, including after DB reset.""" + connection.execute( + "INSERT OR IGNORE INTO biometric_processing_runs " + "(id, side, kind, bundle, config_hash, calibration_hash, input_identity, started_at_ms) " + "VALUES (?, ?, 'live', ?, ?, NULL, ?, ?)", + (self.id, self.side, self.bundle_json, self.config_hash, + json.dumps({"sourceSide": self.source_side, "timestampBasis": "processing_clock", + "rawIdentity": "not_retained", "sensorCalibration": "not_consumed"}), + self.started_at_ms)) + + +def make_context(side, source_side, snapshot, now_ms): + shipped = manifest() + resolved = snapshot.resolved if snapshot else shipped["defaults"] + bundle = {"schemaVersion": 1, "manifestHash": config_hash(shipped), + "resolved": resolved, "generation": snapshot.generation if snapshot else 0, + "revisionId": snapshot.revision_id if snapshot else None, + "components": shipped["components"], "sourceHashes": shipped["sourceHashes"]} + return RunContext(str(uuid4()), side, source_side, json.dumps(bundle), + config_hash(resolved), now_ms) + + +def restore_applied(connection, side): + """Only restore a projection acknowledged by all three numerical consumers.""" + rows = connection.execute( + "SELECT generation, applied_hash FROM biometric_runtime_receipts " + "WHERE side = ? AND state = 'applied' AND effective_sample_time_ms IS NOT NULL " + "AND component IN ('heart_rate', 'breathing_rate', 'hrv_index') " + "GROUP BY generation, applied_hash HAVING COUNT(DISTINCT component) = 3 " + "ORDER BY generation DESC LIMIT 1", (side,)).fetchone() + if rows is None: + return None + row = connection.execute( + "SELECT side, generation, resolved_hash, boundary, " + "CASE WHEN length(CAST(payload AS BLOB)) <= 65536 THEN payload ELSE NULL END " + "FROM biometric_runtime_configs WHERE side = ? AND generation = ?", + (side, rows[0])).fetchone() + if row is None: + raise ValueError("Applied runtime projection is missing") + snapshot = RuntimeSnapshot.parse(row[4]) + if (snapshot.side, snapshot.generation, snapshot.resolved_hash, snapshot.boundary) != tuple(row[:4]): + raise ValueError("Applied runtime projection envelope mismatch") + if (snapshot.side, snapshot.generation, snapshot.resolved_hash) != (side, rows[0], rows[1]): + raise ValueError("Applied runtime projection disagrees with receipts") + return snapshot + + +class RuntimeCoordinator: + def __init__(self, holder, processors, clock=time.monotonic, wall=time.time): + self.holder = holder + self.processors = processors + self.clock, self.wall = clock, wall + self.selections = {side: BoundarySelection(side) for side in ("left", "right")} + self.initialized = False + self.enabled = False + self.last_poll = float("-inf") + self.problem = None + self.effective_times = {} + self._frame_sides = set() + self._receipted = {} + + def _problem(self, error): + problem = str(error) + if problem != self.problem: + log.warning("Biometric runtime configuration unavailable; retaining actual compatible configuration: %s", problem) + self.problem = problem + + def _poll(self): + if self.clock() - self.last_poll < 5: + return + self.last_poll = self.clock() + try: + conn = self.holder.conn + # Activation requires both acknowledgment and provenance schema. + conn.execute("SELECT processing_run_id FROM vitals LIMIT 0") + conn.execute("SELECT id FROM biometric_processing_runs LIMIT 0") + conn.execute("SELECT component FROM biometric_runtime_receipts LIMIT 0") + problems = [] + for side, selection in self.selections.items(): + try: + if not self.initialized: + restored = restore_applied(conn, side) + selection = BoundarySelection(side, restored) + self.selections[side] = selection + snapshot = read_latest_snapshot(conn, side) + if snapshot: + selection.offer(snapshot) + except ValueError as error: + selection.pending = None + problems.append(f"{side}: {error}") + self.initialized = True + self.enabled = True + if problems: + self._problem("; ".join(problems)) + else: + self.problem = None + except sqlite3.Error as error: + for selection in self.selections.values(): + selection.pending = None + self._problem(error) + + def before_frame(self, home, occupied, now_ms): + """Install before accepting either physical channel's first new sample.""" + self._poll() + self._frame_sides = set() + if not self.enabled: + return + for side, selection in self.selections.items(): + if home is not None and side != home: + continue + ready = selection.ready(occupied(side)) + snapshot = ready or selection.applied + for source_side, processor in self.processors.items(): + logical_side = home or source_side + if logical_side != side: + continue + key = (side, snapshot) + if getattr(processor, "runtime_key", None) != key: + parameters = ProcessingParameters.from_config(snapshot.resolved if snapshot else manifest()["defaults"]) + processor.install_runtime(parameters, make_context(side, source_side, snapshot, now_ms), + reset_side=getattr(processor, "runtime_key", (None,))[0] != side) + processor.runtime_key = key + if ready: + selection.commit(ready) + self.effective_times[side] = now_ms + elif side not in self.effective_times: + # Restored config starts a new warmup after daemon restart. + self.effective_times[side] = now_ms + self._frame_sides.add(side) + + def after_frame(self, now_ms): + """Receipt only after both channels have accepted this frame successfully.""" + if not self.enabled: + return + try: + completed = {} + with self.holder.conn: + for side in self._frame_sides: + snapshot = self.selections[side].applied + if snapshot is None: + continue + prior = self._receipted.get(side) + identity = (snapshot.generation, snapshot.resolved_hash, self.effective_times[side]) + if prior and prior[0] == identity and self.clock() - prior[1] < 30: + continue + for component in COMPONENTS: + self.holder.conn.execute( + "INSERT INTO biometric_runtime_receipts " + "(side, component, generation, applied_hash, effective_sample_time_ms, state, error, heartbeat_at_ms) " + "VALUES (?, ?, ?, ?, ?, 'applied', NULL, ?) " + "ON CONFLICT(side, component, generation) DO UPDATE SET " + "applied_hash=excluded.applied_hash, effective_sample_time_ms=excluded.effective_sample_time_ms, " + "state='applied', error=NULL, heartbeat_at_ms=excluded.heartbeat_at_ms", + (side, component, snapshot.generation, snapshot.resolved_hash, + self.effective_times[side], now_ms)) + completed[side] = (identity, self.clock()) + self._receipted.update(completed) + except sqlite3.Error as error: + self._problem(error) diff --git a/modules/piezo-processor/test_default_golden.py b/modules/piezo-processor/test_default_golden.py new file mode 100644 index 000000000..785c0b383 --- /dev/null +++ b/modules/piezo-processor/test_default_golden.py @@ -0,0 +1,29 @@ +"""Pre-refactor baseline captured from dev 3e9027ed with the locked environment. + +A synthetic fixture establishes deterministic numerical parity, not clinical +accuracy. Existing artifact/presence/pump fixtures cover gating separately. +""" +import json +from pathlib import Path + +import numpy as np +import pytest + +from main import HRTracker, compute_breathing_rate, compute_hrv, subharmonic_summation_hr + + +def test_default_processing_golden(): + fixture = json.loads(Path(__file__).with_name("fixtures").joinpath("default-processing-v1.json").read_text()) + fs = fixture["sampleRateHz"] + t = np.arange(fs * 300) / fs + rng = np.random.default_rng(fixture["seed"]) + phase = 2 * np.pi * (1.1 * t + .2 * np.sin(2 * np.pi * t / 40)) + samples = ((1 + .2 * np.sin(2 * np.pi * .25 * t)) + * (np.sin(phase) + .5 * np.sin(2 * phase) + .25 * np.sin(3 * phase)) + + rng.normal(0, .08, len(t))) + assert subharmonic_summation_hr(samples[-30 * fs:]) == pytest.approx(fixture["hr"], abs=1e-9) + assert compute_breathing_rate(samples[-60 * fs:]) == pytest.approx(fixture["breathing"], abs=1e-9) + assert compute_hrv(samples) == pytest.approx(fixture["hrvIndex"], abs=1e-9) + tracker = HRTracker() + actual = [tracker.update(hr, 1.0) for hr in [60, 61, 120, 62, 130, 65, 100, 100, 100, 101, None, 50]] + assert actual == fixture["tracker"] diff --git a/modules/piezo-processor/test_main.py b/modules/piezo-processor/test_main.py index 4a60dbcf1..4b4f63430 100644 --- a/modules/piezo-processor/test_main.py +++ b/modules/piezo-processor/test_main.py @@ -40,6 +40,7 @@ "common.bed_presence", Path(__file__).resolve().parent.parent / "common" / "bed_presence.py") _stubs["common.bed_presence"] = importlib.util.module_from_spec(_spec) _spec.loader.exec_module(_stubs["common.bed_presence"]) +_stubs["common"].__path__ = [str(Path(__file__).resolve().parent.parent / "common")] sys.modules.update(_stubs) from main import ( # noqa: E402 @@ -52,6 +53,7 @@ FrzHealthPumpState, PUMP_OFF_GUARD_S, PUMP_ACTIVE_RPM_MIN, + PUMP_COUPLING_STD_FACTOR, HRTracker, PresenceDetector, PumpGate, @@ -1088,6 +1090,222 @@ def close(self): assert rows == [("right",)] +class TestPresenceDetectorRelaxedReentry: + """With the bed confirmed occupied by the capacitance sensor, re-entry + uses the exit criteria instead of the (much higher) enter threshold, so a + sleeper who went quiet resumes vitals without a hard movement.""" + + def _absent_after_quiet(self): + pd = PresenceDetector() + pd.update(1_000_000, 0.8) + for _ in range(pd.exit_count): + pd.update(50_000, 0.05) + assert pd.state == PresenceDetector.ABSENT + return pd + + def test_moderate_signal_does_not_reenter_by_default(self): + pd = self._absent_after_quiet() + # Between exit (150k) and enter (400k), autocorr below 0.45. + assert pd.update(250_000, 0.3) is False + assert pd.state == PresenceDetector.ABSENT + + def test_moderate_signal_reenters_when_relaxed(self): + pd = self._absent_after_quiet() + assert pd.update(250_000, 0.3, relaxed=True) is True + assert pd.state == PresenceDetector.PRESENT + assert pd.consecutive_low == 0 + + def test_relaxed_reentry_on_autocorrelation_alone(self): + pd = self._absent_after_quiet() + # Energy still weak, but rhythm above the exit bar (0.225). + assert pd.update(50_000, 0.3, relaxed=True) is True + + def test_weak_window_stays_absent_even_when_relaxed(self): + pd = self._absent_after_quiet() + assert pd.update(100_000, 0.1, relaxed=True) is False + assert pd.state == PresenceDetector.ABSENT + + def test_relaxed_flag_does_not_change_exit_behaviour(self): + pd = PresenceDetector() + pd.update(1_000_000, 0.8, relaxed=True) + for _ in range(pd.exit_count - 1): + assert pd.update(50_000, 0.05, relaxed=True) is True + assert pd.update(50_000, 0.05, relaxed=True) is False + assert pd.state == PresenceDetector.ABSENT + + def test_first_entry_from_cold_start_is_also_relaxed(self): + pd = PresenceDetector() + assert pd.update(200_000, 0.1, relaxed=True) is True + + +class TestPumpCouplingGuardWithBedPresence: + """Both pumps running (Pod 5 steady state) raise the piezo re-entry bar + to PUMP_COUPLING_STD_FACTOR x enter and acr > 0.6. That guard exists to + tell a person from pump vibration; when the capacitance sensor already + says a person is in bed it must step aside. Regression for the 05:57-07:01 + blackout on 2026-10-11 (sleeper went quiet, cap-sense occupied all along, + pumps at 1941/2008 rpm, piezo stayed ABSENT until a hard movement).""" + + def _pumps_both_on(self): + ps = FrzHealthPumpState() + ps.update({"type": "frzHealth", + "left": {"pumpRpm": 1941}, "right": {"pumpRpm": 2008}}) + assert ps.is_symmetric_active() + return ps + + def _proc(self, bed, acr): + import main + conn = TestWriteVitalsResilience()._make_db() + proc = main.SideProcessor("left", main.DBHolder(conn), pump_state=self._pumps_both_on()) + seen = [] + proc.sink = lambda cand: seen.append(cand) or True + if bed != "unset": + proc.bed_occupied = lambda: bed + return proc, seen + + def _moderate_signal(self, proc): + """A BCG signal whose filtered energy clears the normal enter + threshold but not the pump-coupling bar.""" + import main + np.random.seed(7) + sig = make_bcg_signal(62, 15, noise_level=0.05) + filt = main._bandpass(sig, 1.0, 10.0, FS) + w = int(5 * FS) + med = float(np.median([np.std(filt[j:j + w]) for j in range(0, len(filt) - w + 1, w)])) + enter = proc._presence.enter_threshold + assert enter < med < enter * PUMP_COUPLING_STD_FACTOR, med + return sig + + def _run(self, bed, acr=0.5): + import main + proc, seen = self._proc(bed, acr) + sig = self._moderate_signal(proc) + # Autocorrelation above the normal bar (0.45) but below the coupling bar (0.6). + with patch("main._autocorr_quality", return_value=acr), \ + patch("main.time.time", return_value=1_000_000.0): + proc.ingest(sig) + return proc, seen + + def test_guard_suppresses_moderate_signal_when_bed_presence_unknown(self): + proc, seen = self._run(None) + assert proc._presence.state == PresenceDetector.ABSENT + assert seen == [] + + def test_guard_suppresses_when_no_bed_presence_configured(self): + proc, seen = self._run("unset") + assert proc._presence.state == PresenceDetector.ABSENT + assert seen == [] + + def test_bed_occupied_bypasses_guard_and_resumes_vitals(self): + proc, seen = self._run(True) + assert proc._presence.state == PresenceDetector.PRESENT + assert len(seen) == 1 + assert seen[0].heart_rate is not None + + def test_bed_occupied_relaxes_reentry_below_the_normal_enter_bar(self): + import main + proc, seen = self._proc(True, 0.3) + np.random.seed(7) + sig = make_bcg_signal(62, 15, noise_level=0.05) * 0.6 + filt = main._bandpass(sig, 1.0, 10.0, FS) + w = int(5 * FS) + med = float(np.median([np.std(filt[j:j + w]) for j in range(0, len(filt) - w + 1, w)])) + assert proc._presence.exit_threshold < med < proc._presence.enter_threshold, med + with patch("main._autocorr_quality", return_value=0.3), \ + patch("main.time.time", return_value=1_000_000.0): + proc.ingest(sig) + assert proc._presence.state == PresenceDetector.PRESENT + assert len(seen) == 1 + + def test_bed_empty_never_relaxes_the_guard(self): + proc, seen = self._run(False, acr=0.9) + assert proc._presence.state == PresenceDetector.ABSENT + assert seen == [] + + def test_weak_signal_in_an_occupied_bed_writes_nothing(self): + import main + proc, seen = self._proc(True, 0.05) + sig = make_noise(15, amplitude=20_000) + with patch("main._autocorr_quality", return_value=0.05), \ + patch("main.time.time", return_value=1_000_000.0): + proc.ingest(sig) + assert proc._presence.state == PresenceDetector.ABSENT + assert seen == [] + + def test_cross_channel_rejected_side_is_not_relaxed_into_presence(self): + """Single-sleeper mode: both sides read the home side as occupied, + so the away side (coupling from the sleeper, weaker energy, rejected + by the other side's stronger signal) must stay ABSENT even though + its energy alone clears the relaxed re-entry bar.""" + import main + proc, seen = self._proc(True, 0.3) + other = main.SideProcessor("right", main.DBHolder(TestWriteVitalsResilience()._make_db())) + other._last_med_std = proc._presence.enter_threshold * 2 + other._last_acr_qual = 0.8 + proc._other = other + np.random.seed(7) + sig = make_bcg_signal(62, 15, noise_level=0.05) * 0.6 + with patch("main._autocorr_quality", return_value=0.3), \ + patch("main.time.time", return_value=1_000_000.0): + proc.ingest(sig) + assert proc._last_acr_qual == 0.0 + assert proc._presence.state == PresenceDetector.ABSENT + assert seen == [] + + def test_away_side_inherits_home_occupancy_but_not_relaxation(self): + """Single sleeper on the left: both processors read the left zone as + occupied. The right (away) side's own pump coupling, in the band the + guard exists for, must stay behind the guard — only the home side + may treat the merged occupancy as its own.""" + import main + proc, seen = self._proc(True, 0.5) + proc.side = "right" + proc.home_side = lambda: "left" + sig = self._moderate_signal(proc) + with patch("main._autocorr_quality", return_value=0.5), \ + patch("main.time.time", return_value=1_000_000.0): + proc.ingest(sig) + assert proc._presence.state == PresenceDetector.ABSENT + assert seen == [] + + def test_home_side_in_single_sleeper_mode_is_relaxed(self): + proc, seen = self._proc(True, 0.5) + proc.home_side = lambda: "left" + sig = self._moderate_signal(proc) + with patch("main._autocorr_quality", return_value=0.5), \ + patch("main.time.time", return_value=1_000_000.0): + proc.ingest(sig) + assert proc._presence.state == PresenceDetector.PRESENT + assert len(seen) == 1 + + def test_later_cross_channel_rejection_ends_relaxed_presence(self): + """Relaxed entry before the sibling's values arrive, then the sibling + shows the stronger signal: rejection must count as weak windows so + presence exits through the normal hysteresis instead of latching on + energy alone.""" + import main + proc, seen = self._proc(True, 0.3) + proc._pump_state = None + other = main.SideProcessor("right", main.DBHolder(TestWriteVitalsResilience()._make_db())) + proc._other = other + np.random.seed(7) + sig = make_bcg_signal(62, 15, noise_level=0.05) * 0.6 + states = [] + with patch("main._autocorr_quality", return_value=0.3): + for cycle in range(1 + proc._presence.exit_count): + if cycle == 1: + other._last_med_std = proc._presence.enter_threshold * 2 + other._last_acr_qual = 0.8 + with patch("main.time.time", return_value=1_000_000.0 + cycle * 61): + proc.ingest(sig) + states.append(proc._presence.state) + assert states[0] == PresenceDetector.PRESENT + assert states[-1] == PresenceDetector.ABSENT + # Only the window before rejection produced vitals; the rejected + # windows advance the exit count but are never submitted. + assert len(seen) == 1 + + class TestSideProcessorAbsenceThrottle: """_maybe_write must update _last_write on 'no user' so return from extended absence doesn't trigger burst processing until the first write diff --git a/modules/piezo-processor/test_raw_publication.py b/modules/piezo-processor/test_raw_publication.py new file mode 100644 index 000000000..725d53bef --- /dev/null +++ b/modules/piezo-processor/test_raw_publication.py @@ -0,0 +1,238 @@ +"""Real migration/SQLite tests for complete-batch waveform publication.""" +from concurrent.futures import ThreadPoolExecutor +from copy import deepcopy +import gzip +import json +from pathlib import Path +import sqlite3 +import sys +from threading import Event +import uuid + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from common.biometrics.raw_artifacts import RawArtifactStore +from common.biometrics.raw_publication import publish_waveform_dataset +from test_retained_replay import fixture + + +def prepare(tmp_path): + source, inputs, _ = fixture(tmp_path) + archive = tmp_path / 'archive' + selections = [] + for index, item in enumerate(inputs['artifacts']): + name = f'{index}.RAW.gz' + (archive / name).write_bytes((source.directory / item['artifact']['name']).read_bytes()) + selections.append({'source_name': name, 'artifact': item['artifact']}) + store = RawArtifactStore(tmp_path / 'published', source.limits) + database = tmp_path / 'biometrics.db' + db = sqlite3.connect(database) + db.execute('PRAGMA foreign_keys = ON') + migrations = Path(__file__).resolve().parents[2] / 'src/db/biometrics-migrations' + for migration in sorted(migrations.glob('*.sql')): + db.executescript(migration.read_text()) + prepared = {'id': str(uuid.uuid4()), 'side': 'left', 'subjectId': 'fixture', 'idempotencyKey': 'waveform-one', + 'requestHash': 'a' * 64, 'contentHash': 'b' * 64, 'createdAtMs': 1000, + 'manifest': {'version': 'waveform-dataset-manifest-v1', 'splitProtocol': {'assignments': [ + {'cohortId': 'night', 'startMs': inputs['score_start_ms'], 'endMs': inputs['score_end_ms']}]}}, + 'snapshot': {'version': 'waveform-dataset-v1', 'side': 'left', 'subjectId': 'fixture', + 'sessions': [], 'references': [], 'processingRuns': [], 'waveformInputs': [{'cohortId': 'night', 'input': inputs}]}} + return store, db, database, archive, selections, prepared + + +def counts(db): + return [db.execute(f'SELECT count(*) FROM {name}').fetchone()[0] for name in + ('biometric_datasets', 'biometric_raw_artifacts', 'biometric_dataset_raw_artifacts', 'vitals')] + + +def test_publishes_exact_snapshot_with_all_ordered_references_and_idempotent_retry(tmp_path): + store, db, _, archive, selections, prepared = prepare(tmp_path) + try: + result = publish_waveform_dataset(store, db, archive, selections, prepared) + assert result['id'] == prepared['id'] and result['available'] + assert counts(db) == [1, 2, 2, 0] + snapshot = db.execute('SELECT snapshot FROM biometric_datasets').fetchone()[0] + assert json.loads(snapshot) == prepared['snapshot'] + assert [row[0] for row in db.execute('SELECT artifact_hash FROM biometric_dataset_raw_artifacts ORDER BY position')] == [item['artifact']['sha256'] for item in selections] + for name in archive.iterdir(): + name.unlink() + retry = {**prepared, 'id': str(uuid.uuid4()), 'createdAtMs': 2000} + assert publish_waveform_dataset(store, db, archive, selections, retry) == result + with pytest.raises(ValueError, match='idempotency key changed'): + publish_waveform_dataset(store, db, archive, selections, {**retry, 'requestHash': 'c' * 64}) + assert counts(db) == [1, 2, 2, 0] + finally: + db.close() + + +def test_failure_on_last_reference_rolls_back_every_database_write(tmp_path): + store, db, _, archive, selections, prepared = prepare(tmp_path) + try: + db.executescript("CREATE TRIGGER fail_last BEFORE INSERT ON biometric_dataset_raw_artifacts WHEN NEW.position = 1 BEGIN SELECT RAISE(ABORT, 'injected failure'); END;") + with pytest.raises(sqlite3.IntegrityError, match='injected failure'): + publish_waveform_dataset(store, db, archive, selections, prepared) + assert counts(db) == [0, 0, 0, 0] + assert len(list(store.directory.glob('*.RAW.gz'))) == 2 + assert not db.in_transaction + db.execute('DROP TRIGGER fail_last') + assert publish_waveform_dataset(store, db, archive, selections, prepared)['available'] + assert counts(db) == [1, 2, 2, 0] + finally: + db.close() + + +def test_changed_second_archive_never_publishes_a_partial_dataset(tmp_path): + store, db, _, archive, selections, prepared = prepare(tmp_path) + try: + (archive / selections[1]['source_name']).write_bytes(gzip.compress(b'changed archive')) + with pytest.raises(ValueError, match='changed since inspection'): + publish_waveform_dataset(store, db, archive, selections, prepared) + assert counts(db) == [0, 0, 0, 0] + assert len(list(store.directory.glob('*.RAW.gz'))) == 1 + assert not list(store.directory.glob('.part-*')) + finally: + db.close() + + +def test_released_retry_stays_released_and_a_new_dataset_can_retain_shared_bytes(tmp_path): + store, db, _, archive, selections, prepared = prepare(tmp_path) + try: + first = publish_waveform_dataset(store, db, archive, selections, prepared) + with db: + db.execute('UPDATE biometric_datasets SET released_at_ms = 2000, snapshot = ?', ('{"released":true}',)) + db.execute('UPDATE biometric_dataset_raw_artifacts SET released_at_ms = 2000') + db.execute('UPDATE biometric_raw_artifacts SET released_at_ms = 2000') + assert not publish_waveform_dataset(store, db, archive, selections, prepared)['available'] + assert db.execute('SELECT count(*) FROM biometric_raw_artifacts WHERE released_at_ms IS NULL').fetchone()[0] == 0 + for file in archive.iterdir(): + file.unlink() + archive.rmdir() + second = publish_waveform_dataset(store, db, archive, selections, {**prepared, + 'id': str(uuid.uuid4()), 'idempotencyKey': 'waveform-two', 'createdAtMs': 3000}) + assert second['id'] != first['id'] and second['available'] + assert counts(db) == [2, 2, 4, 0] + assert db.execute('SELECT count(*) FROM biometric_raw_artifacts WHERE released_at_ms IS NULL AND created_at_ms = 1000').fetchone()[0] == 2 + assert db.execute('SELECT count(*) FROM biometric_dataset_raw_artifacts WHERE released_at_ms = 2000').fetchone()[0] == 2 + finally: + db.close() + + +def test_rejects_mismatched_cohorts_and_sources_without_copies(tmp_path): + store, db, _, archive, selections, prepared = prepare(tmp_path) + try: + bad = deepcopy(prepared) + bad['snapshot']['waveformInputs'][0]['input']['score_start_ms'] += 1 + with pytest.raises(ValueError, match='whole-night cohort'): + publish_waveform_dataset(store, db, archive, selections, bad) + with pytest.raises(ValueError, match='match frozen'): + publish_waveform_dataset(store, db, archive, selections[:1], prepared) + db.execute('BEGIN') + with pytest.raises(ValueError, match='independent transaction'): + publish_waveform_dataset(store, db, archive, selections, prepared) + db.rollback() + assert counts(db) == [0, 0, 0, 0] + assert not list(store.directory.glob('*.RAW.gz')) + finally: + db.close() + + +def test_copying_does_not_hold_sqlite_write_lock_or_expose_partial_rows(tmp_path, monkeypatch): + store, db, database, archive, selections, prepared = prepare(tmp_path) + reached, proceed = Event(), Event() + original = store._retain_locked + + def delayed(root, name, expected): + if name == selections[1]['source_name']: + reached.set() + assert proceed.wait(5) + return original(root, name, expected) + + monkeypatch.setattr(store, '_retain_locked', delayed) + + def publish(): + connection = sqlite3.connect(database) + try: + connection.execute('PRAGMA foreign_keys = ON') + return publish_waveform_dataset(store, connection, archive, selections, prepared) + finally: + connection.close() + + try: + with ThreadPoolExecutor(max_workers=1) as pool: + future = pool.submit(publish) + try: + assert reached.wait(5) + assert counts(db) == [0, 0, 0, 0] + db.execute('BEGIN IMMEDIATE') + db.rollback() + finally: + proceed.set() + assert future.result(timeout=5)['available'] + assert counts(db) == [1, 2, 2, 0] + finally: + db.close() + + +def test_real_publication_entrypoint_commits_and_recovers_a_lost_reply_by_retry(tmp_path): + import subprocess + store, db, database, archive, selections, prepared = prepare(tmp_path) + payload = json.dumps({'version': 'waveform-publication-work-v1', 'prepared': prepared, 'selections': selections}).encode() + command = [sys.executable, '-B', '-m', 'common.biometrics.publication_worker', '--database', str(database), + '--archive-directory', str(archive), '--retained-directory', str(store.directory)] + env = {'PYTHONPATH': str(Path(__file__).resolve().parent.parent), 'LANG': 'C.UTF-8', 'TZ': 'UTC'} + try: + first = subprocess.run(command, input=payload, capture_output=True, env=env, cwd=tmp_path, timeout=20) + assert first.returncode == 0, first.stderr + assert first.stderr == b'' + assert json.loads(first.stdout) == {'type': 'result', 'id': prepared['id'], 'contentHash': prepared['contentHash'], 'available': True} + assert counts(db) == [1, 2, 2, 0] + for file in archive.iterdir(): + file.unlink() + retry = subprocess.run(command, input=payload, capture_output=True, env=env, cwd=tmp_path, timeout=20) + assert retry.returncode == 0 and retry.stdout == first.stdout + assert counts(db) == [1, 2, 2, 0] + finally: + db.close() + + +def test_sigkill_inside_reference_transaction_recovers_without_partial_publication(tmp_path): + import select + import subprocess + store, db, database, archive, selections, prepared = prepare(tmp_path) + request = tmp_path / 'request.json' + request.write_text(json.dumps({'prepared': prepared, 'selections': selections})) + db.executescript('CREATE TRIGGER pause_publication BEFORE INSERT ON biometric_dataset_raw_artifacts BEGIN SELECT pause_ref(); END;') + script = ''' +import json, sqlite3, sys, time +from common.biometrics.raw_artifacts import RawArtifactStore +from common.biometrics.raw_publication import publish_waveform_dataset +work=json.load(open(sys.argv[1])) +db=sqlite3.connect(sys.argv[2]); db.execute('PRAGMA foreign_keys = ON') +def pause(): + print('transaction-open', flush=True) + time.sleep(60) + return 0 +db.create_function('pause_ref', 0, pause) +publish_waveform_dataset(RawArtifactStore(sys.argv[4]), db, sys.argv[3], work['selections'], work['prepared']) +''' + child = subprocess.Popen([sys.executable, '-B', '-c', script, str(request), str(database), str(archive), str(store.directory)], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + env={'PYTHONPATH': str(Path(__file__).resolve().parent.parent), 'OPENBLAS_NUM_THREADS': '1'}) + try: + assert select.select([child.stdout], [], [], 10)[0] + assert child.stdout.readline() == b'transaction-open\n' + assert counts(db) == [0, 0, 0, 0] + child.kill() + child.communicate(timeout=5) + assert child.returncode < 0 + assert counts(db) == [0, 0, 0, 0] + assert len(list(store.directory.glob('*.RAW.gz'))) == 2 + db.execute('DROP TRIGGER pause_publication') + assert publish_waveform_dataset(store, db, archive, selections, prepared)['available'] + assert counts(db) == [1, 2, 2, 0] + finally: + if child.poll() is None: + child.kill() + child.communicate(timeout=5) + db.close() diff --git a/modules/piezo-processor/test_replay_worker.py b/modules/piezo-processor/test_replay_worker.py new file mode 100644 index 000000000..96cde2246 --- /dev/null +++ b/modules/piezo-processor/test_replay_worker.py @@ -0,0 +1,128 @@ +"""Real subprocess protocol for shipped retained waveform processing.""" +from copy import deepcopy +from io import BytesIO +import json +from pathlib import Path +import subprocess +import sys + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from common.biometrics.config import manifest +from common.biometrics import replay_worker +from test_retained_replay import fixture, run + + +def work(snapshot): + return {'version': 'retained-waveform-work-v1', 'input': snapshot, + 'configs': {side: deepcopy(manifest()['defaults']) for side in ('left', 'right')}} + + +def child(tmp_path, directory, payload): + return subprocess.run([sys.executable, '-c', + 'from common.biometrics import replay_worker; ' + 'from common.biometrics.execution_lock import acquire_execution_lock; ' + f'replay_worker.acquire_execution_lock=lambda: acquire_execution_lock({str(tmp_path / "execution.lock")!r}); ' + 'raise SystemExit(replay_worker.main())', + '--retained-directory', str(directory)], input=payload, capture_output=True, + cwd=tmp_path, env={'PYTHONPATH': str(Path(__file__).resolve().parent.parent), + 'LANG': 'C.UTF-8', 'TZ': 'UTC'}, timeout=20) + + +def test_real_worker_streams_provisional_readings_then_one_success_result(tmp_path): + store, snapshot, _ = fixture(tmp_path) + before = {entry.name: (entry.stat().st_mtime_ns, entry.stat().st_mode) for entry in store.directory.iterdir()} + expected_rows = [] + expected = run(store, snapshot, expected_rows.append) + completed = child(tmp_path, store.directory, json.dumps(work(snapshot)).encode()) + assert completed.returncode == 0, completed.stderr + assert completed.stderr == b'' + events = [json.loads(line) for line in completed.stdout.splitlines()] + assert len(events) == len(expected_rows) + 1 + from common.biometrics.build_identity import replay_build_identity + resources = events[-1]['result'].pop('resources') + assert resources['version'] == 'waveform-process-resources-v1' + assert resources['wallMs'] > 0 + assert resources['cpuMs'] > 0 + assert resources['peakRssBytes'] > 0 + assert resources['scope'] == 'python_process_until_result' + assert resources['memoryScope'] == 'process_lifetime_high_water_mark' + assert events[-1] == {'type': 'result', 'result': {**expected, 'build_hash': replay_build_identity()['hash']}} + for event, row in zip(events[:-1], expected_rows): + assert event['type'] == 'reading' + value = event['reading'] + assert value['heartRate'] == row.candidate.heart_rate + assert value['timestampMs'] == row.timestamp_ms + assert value['configHash'] == row.candidate.run_context.config_hash + assert value['hrvMetric'] == 'window_ibi_variability_v1' + assert value['side'] == row.side + assert {entry.name: (entry.stat().st_mtime_ns, entry.stat().st_mode) for entry in store.directory.iterdir()} == before + + +def test_late_missing_artifact_has_no_success_event_or_sensitive_diagnostics(tmp_path): + store, snapshot, _ = fixture(tmp_path, split_at=136) + (store.directory / snapshot['artifacts'][1]['artifact']['name']).unlink() + completed = child(tmp_path, store.directory, json.dumps(work(snapshot)).encode()) + assert completed.returncode == 1 + assert completed.stderr == b'Retained waveform worker failed\n' + assert all(json.loads(line)['type'] == 'reading' for line in completed.stdout.splitlines()) + assert b'configHash' in completed.stdout # Partial output must not be mistaken for success. + + +@pytest.mark.parametrize('payload', [b'{"version":1,"version":2}', b'{"value":NaN}', b'{"value":Infinity}', b'null', b'[]']) +def test_malformed_envelopes_fail_before_numerical_execution(tmp_path, payload): + output = BytesIO() + with pytest.raises(ValueError): + replay_worker.run(BytesIO(payload), output, tmp_path / 'missing') + assert output.getvalue() == b'' + assert not (tmp_path / 'missing').exists() + + +def test_worker_byte_limits_do_not_emit_an_oversize_input_or_result(tmp_path, monkeypatch): + store, snapshot, _ = fixture(tmp_path) + payload = json.dumps(work(snapshot)).encode() + monkeypatch.setattr(replay_worker, 'MAX_INPUT_BYTES', len(payload) - 1) + output = BytesIO() + with pytest.raises(ValueError, match='input exceeds'): + replay_worker.run(BytesIO(payload), output, store.directory) + assert output.getvalue() == b'' + monkeypatch.setattr(replay_worker, 'MAX_INPUT_BYTES', len(payload)) + monkeypatch.setattr(replay_worker, 'MAX_OUTPUT_BYTES', 10) + with pytest.raises(ValueError, match='output exceeds'): + replay_worker.run(BytesIO(payload), output, store.directory) + assert output.getvalue() == b'' + + +def test_build_probe_and_pinned_execution_reject_runtime_changes_before_reading(tmp_path): + from common.biometrics.build_identity import replay_build_identity + build = replay_build_identity() + probe = subprocess.run([sys.executable, '-B', '-m', 'common.biometrics.replay_worker', '--describe-build'], + capture_output=True, cwd=tmp_path, + env={'PYTHONPATH': str(Path(__file__).resolve().parent.parent), 'LANG': 'C.UTF-8', 'TZ': 'UTC'}, timeout=20) + assert probe.returncode == 0 and not probe.stderr + assert json.loads(probe.stdout) == build + store, snapshot, _ = fixture(tmp_path) + payload = {**work(snapshot), 'expected_build_hash': build['hash']} + success = child(tmp_path, store.directory, json.dumps(payload).encode()) + assert success.returncode == 0 + assert json.loads(success.stdout.splitlines()[-1])['result']['build_hash'] == build['hash'] + for bad in ('0' * 64, None, 42, 'bad'): + output = BytesIO() + with pytest.raises(ValueError, match='build identity changed'): + replay_worker.run(BytesIO(json.dumps({**payload, 'expected_build_hash': bad}).encode()), output, tmp_path / 'missing') + assert output.getvalue() == b'' + assert not (tmp_path / 'missing').exists() + + +def test_changed_build_during_execution_never_emits_success(tmp_path, monkeypatch): + from common.biometrics import build_identity + store, snapshot, _ = fixture(tmp_path) + before = build_identity.replay_build_identity() + identities = iter([before, {**before, 'hash': '0' * 64}]) + monkeypatch.setattr(build_identity, 'replay_build_identity', lambda: next(identities)) + output = BytesIO() + with pytest.raises(ValueError, match='changed during execution'): + replay_worker.run(BytesIO(json.dumps(work(snapshot)).encode()), output, store.directory) + events = [json.loads(line) for line in output.getvalue().splitlines()] + assert events and all(event['type'] == 'reading' for event in events) diff --git a/modules/piezo-processor/test_retained_replay.py b/modules/piezo-processor/test_retained_replay.py new file mode 100644 index 000000000..09feddcd7 --- /dev/null +++ b/modules/piezo-processor/test_retained_replay.py @@ -0,0 +1,154 @@ +"""Retained gzip -> bounded CBOR -> shared numerical replay integration.""" +from copy import deepcopy +from dataclasses import asdict +import gzip +from pathlib import Path +import struct +import sys + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from common.biometrics.config import manifest +from common.biometrics.raw_artifacts import RawArtifactStore, RawLimits +from common.biometrics.retained_replay import RetainedReplayInput, replay_retained +from test_waveform_replay import frames, START, KNOWN, make_replay + + +# Encode the small fixture subset without importing cbor2, which existing live +# adapter tests intentionally stub globally during collection. +def encode(value): + if isinstance(value, dict): + return bytes([0xa0 + len(value)]) + b''.join(encode(key) + encode(item) for key, item in value.items()) + if isinstance(value, str): + return bytes([0x60 + len(value)]) + value.encode() + if isinstance(value, bytes): + return b'\x5a' + struct.pack('>I', len(value)) + value + return b'\xfb' + struct.pack('>d', value) + + +def fixture(tmp_path, records=None, split_at=76): + records = records if records is not None else [ + (START, {'type': 'frzHealth', 'left': {'pumpRpm': 0}, 'right': {'pumpRpm': 0}}), *frames()] + archive = tmp_path / 'archive' + archive.mkdir() + store = RawArtifactStore(tmp_path / 'retained', RawLimits(1_000_000, 2_000_000, 4_000_000, 1)) + selected = [] + for index, group in enumerate([records[:split_at], records[split_at:]]): + if not group: + continue + data = b'' + for timestamp, record in group: + payload = encode({**record, 'ts': timestamp / 1000}) + data += b'\xa2\x63seq\x01\x64data\x5a' + struct.pack('>I', len(payload)) + payload + name = f'{index}.RAW.gz' + (archive / name).write_bytes(gzip.compress(data, mtime=0)) + artifact = store.retain(archive, name) + selected.append({'artifact': asdict(artifact), 'first_event_ms': group[0][0], 'last_event_ms': group[-1][0]}) + (archive / name).unlink() + snapshot = {'version': 'retained-waveform-input-v1', 'retained_start_ms': START, + 'score_start_ms': START + 60000, 'score_end_ms': START + 150000, 'artifacts': selected, + 'contexts': [{'start_ms': START, 'end_ms': START + 150000, + 'active_sides': ['left'], 'left_occupied': True, 'right_occupied': False, 'pump_context_known': True}]} + return store, snapshot, records + + +def run(store, snapshot, sink, **options): + return replay_retained(store, snapshot, + {side: deepcopy(manifest()['defaults']) for side in ('left', 'right')}, sink, **options) + + +def test_retained_files_match_direct_numerical_driver_after_original_archives_are_deleted(tmp_path): + store, snapshot, records = fixture(tmp_path) + expected, actual = [], [] + replay = make_replay(expected.append) + for timestamp, record in records: + replay.feed(timestamp, record, KNOWN) + replay.close() + result = run(store, snapshot, actual.append) + assert actual == expected + assert actual + assert result['counts'] == replay.counts + assert result['config_hashes'] == replay.hashes + assert result['decoded_records'] == 151 + assert result['artifact_hashes'] == [item['artifact']['sha256'] for item in snapshot['artifacts']] + assert result['limitations'] == sorted(replay.limitations) + again = [] + assert run(store, snapshot, again.append) == result + assert again == actual + + +def test_context_gaps_are_unknown_and_caller_mutation_cannot_change_frozen_timeline(tmp_path): + store, snapshot, _ = fixture(tmp_path) + snapshot['contexts'][0]['end_ms'] = START + 90000 + frozen = RetainedReplayInput.parse(snapshot) + snapshot['contexts'][0]['active_sides'].append('right') + assert frozen.contexts[0].context.active_sides == ('left',) + snapshot['contexts'][0]['active_sides'] = ['left'] + output = [] + result = run(store, snapshot, output.append) + assert {'missing_sleeper_mode', 'missing_occupancy_context', 'missing_pump_context'} <= set(result['limitations']) + assert any('missing_occupancy_context' in row.limitations for row in output) + + +def test_rejects_overlaps_duplicates_unknown_versions_and_bad_context_before_outputs(tmp_path): + store, snapshot, _ = fixture(tmp_path) + bad_inputs = [] + bad = deepcopy(snapshot) + bad['artifacts'][1]['first_event_ms'] = bad['artifacts'][0]['last_event_ms'] + bad_inputs.append(bad) + bad = deepcopy(snapshot) + bad['artifacts'].append(bad['artifacts'][0]) + bad_inputs.append(bad) + bad = deepcopy(snapshot) + bad['contexts'].append(bad['contexts'][0]) + bad_inputs.append(bad) + bad = deepcopy(snapshot) + bad['contexts'][0]['left_occupied'] = 1 + bad_inputs.append(bad) + bad_inputs.extend([{**snapshot, 'version': 'future'}, {**snapshot, 'extra': True}, {**snapshot, 'artifacts': []}]) + for bad in bad_inputs: + output = [] + with pytest.raises(ValueError): + run(store, bad, output.append) + assert not output + + +def test_rejects_incorrect_inspection_bounds_and_record_reversals(tmp_path): + store, snapshot, _ = fixture(tmp_path) + snapshot['artifacts'][0]['first_event_ms'] -= 1 + with pytest.raises(ValueError, match='differ from inspected'): + run(store, snapshot, lambda _: None) + snapshot['artifacts'][0]['first_event_ms'] += 2 + with pytest.raises(ValueError, match='outside inspected'): + run(store, snapshot, lambda _: None) + other = tmp_path / 'reversed' + other.mkdir() + store, snapshot, _ = fixture(other, [(START + offset, {'type': 'frzHealth'}) for offset in (0, 2000, 1000, 3000)]) + with pytest.raises(ValueError, match='reversed'): + run(store, snapshot, lambda _: None) + + +def test_cancellation_output_and_decode_budgets_fail_without_successful_result(tmp_path): + store, snapshot, _ = fixture(tmp_path) + with pytest.raises(ValueError, match='cancelled'): + run(store, snapshot, lambda _: None, cancelled=lambda: True) + with pytest.raises(ValueError, match='record.*budget|record count'): + run(store, snapshot, lambda _: None, maximum_records=100) + with pytest.raises(ValueError, match='output budget'): + run(store, snapshot, lambda _: None, maximum_readings=1) + cancelled = [False] + output = [] + def sink(row): + output.append(row) + cancelled[0] = True + with pytest.raises(ValueError, match='cancelled'): + run(store, snapshot, sink, cancelled=lambda: cancelled[0]) + assert len(output) == 1 + + +def test_missing_artifact_never_falls_back_to_a_rolling_archive(tmp_path): + store, snapshot, _ = fixture(tmp_path) + (store.directory / snapshot['artifacts'][0]['artifact']['name']).unlink() + with pytest.raises(FileNotFoundError): + run(store, snapshot, lambda _: None) diff --git a/modules/piezo-processor/test_runtime_adapter.py b/modules/piezo-processor/test_runtime_adapter.py new file mode 100644 index 000000000..8b0573ad0 --- /dev/null +++ b/modules/piezo-processor/test_runtime_adapter.py @@ -0,0 +1,201 @@ +"""Real SQLite, estimator and routing checks for live profile adoption.""" +from datetime import datetime, timezone +import json +from pathlib import Path +import sqlite3 + +import numpy as np +import pytest + +import main +from common.biometrics.config import resolve_config +from runtime_adapter import RuntimeCoordinator, restore_applied + + +class Clock: + value = 0 + def __call__(self): + return self.value + + +@pytest.fixture +def rig(): + conn = sqlite3.connect(":memory:") + conn.executescript(""" + CREATE TABLE vitals (id INTEGER PRIMARY KEY, side TEXT, timestamp INTEGER, + heart_rate REAL, hrv REAL, breathing_rate REAL); + CREATE TABLE vitals_quality (vitals_id INTEGER, side TEXT, timestamp INTEGER, + quality_score REAL, flags TEXT, hr_raw REAL, created_at INTEGER); + """) + migration = Path(__file__).parents[2] / "src/db/biometrics-migrations/0020_black_patch.sql" + conn.executescript(migration.read_text()) + holder = main.DBHolder(conn) + processors = {side: main.SideProcessor(side, holder) for side in ("left", "right")} + clock = Clock() + coordinator = RuntimeCoordinator(holder, processors, clock=clock) + return conn, holder, processors, clock, coordinator + + +def project(conn, generation=1, side="left", boundary="immediate", overrides=None): + config = resolve_config(overrides or [{"path": "heart_rate.trackingToleranceBpm", "value": 20}]) + payload = dict(schemaVersion=1, side=side, generation=generation, revisionId=None, + resolved=config["resolved"], resolvedHash=config["hash"], + manifestHash=config["manifestHash"], boundary=boundary) + with conn: + conn.execute("INSERT INTO biometric_runtime_configs VALUES (?, ?, ?, ?, ?, ?)", + (side, generation, config["hash"], json.dumps(payload), boundary, 0)) + return config + + +def test_actual_parameters_reset_before_samples_and_ack_after(rig): + conn, _, processors, _, runtime = rig + project(conn) + left = processors["left"] + left._hr_buf.extend([123] * 20) + runtime.before_frame(None, lambda _: True, 1000) + assert left.parameters.tracking_tolerance_bpm == 20 + assert len(left._hr_buf) == 0 + assert processors["right"].parameters.tracking_tolerance_bpm == 15 + assert conn.execute("SELECT * FROM biometric_runtime_receipts").fetchall() == [] + left.ingest(np.array([1, 2])) + runtime.after_frame(1000) + receipts = conn.execute("SELECT component, generation, effective_sample_time_ms, state FROM biometric_runtime_receipts").fetchall() + assert len(receipts) == 3 + assert {r[0] for r in receipts} == {"heart_rate", "breathing_rate", "hrv_index"} + assert all(r[1:] == (1, 1000, "applied") for r in receipts) + + +def test_pending_next_session_and_restart_restore(rig): + conn, holder, processors, clock, runtime = rig + project(conn, boundary="next_session") + runtime.before_frame(None, lambda _: None, 1000) + runtime.after_frame(1000) + assert processors["left"].parameters.tracking_tolerance_bpm == 15 + assert restore_applied(conn, "left") is None + runtime.before_frame(None, lambda _: False, 2000) + runtime.after_frame(2000) + assert restore_applied(conn, "left").generation == 1 + restarted = RuntimeCoordinator(holder, {s: main.SideProcessor(s, holder) for s in processors}, clock=clock) + restarted.before_frame(None, lambda _: True, 3000) + assert restarted.processors["left"].parameters.tracking_tolerance_bpm == 20 + assert restarted.processors["left"].run_context.id != processors["left"].run_context.id + + +def test_single_sleeper_uses_home_profile_on_both_channels_and_retains_source(rig): + conn, holder, processors, _, runtime = rig + config = project(conn) + runtime.before_frame("left", lambda _: True, 1000) + assert processors["right"].parameters.tracking_tolerance_bpm == 20 + context = processors["right"].run_context + candidate = main.VitalsCandidate("right", datetime.fromtimestamp(5, timezone.utc), + 70, 30, 12, .8, None, 70, context) + # A delayed retry after a mode change must preserve its original destination. + assert candidate.write(holder, side="right") + row = conn.execute("SELECT side, processing_run_id FROM vitals").fetchone() + assert row == ("left", context.id) + run = conn.execute("SELECT side, config_hash, input_identity FROM biometric_processing_runs").fetchone() + assert run[:2] == ("left", config["hash"]) + assert json.loads(run[2])["sourceSide"] == "right" + runtime.before_frame(None, lambda _: True, 2000) + assert processors["right"].parameters.tracking_tolerance_bpm == 15 + assert processors["right"].run_context.side == "right" + + +def test_stage_only_change_preserves_numerical_state(rig): + conn, _, processors, clock, runtime = rig + project(conn) + runtime.before_frame(None, lambda _: True, 1000) + left = processors["left"] + left._hr_buf.extend([7, 8]) + estimator = left._estimator + project(conn, 2, overrides=[{"path": "heart_rate.trackingToleranceBpm", "value": 20}, + {"path": "sleep_stages.ruleset.smoothIsolatedEpochs", "value": False}]) + clock.value = 5 + runtime.before_frame(None, lambda _: True, 2000) + assert left._estimator is estimator + assert list(left._hr_buf) == [7, 8] + assert json.loads(left.run_context.bundle_json)["generation"] == 2 + + +def test_invalid_new_projection_retains_actual_profile_and_reports_error(rig): + conn, _, processors, clock, runtime = rig + project(conn) + runtime.before_frame(None, lambda _: True, 1000) + runtime.after_frame(1000) + project(conn, 2) + conn.execute("UPDATE biometric_runtime_configs SET payload='{}' WHERE generation=2") + clock.value = 5 + runtime.before_frame(None, lambda _: True, 2000) + runtime.after_frame(2000) + assert processors["left"].parameters.tracking_tolerance_bpm == 20 + assert runtime.problem is not None + assert conn.execute("SELECT DISTINCT generation FROM biometric_runtime_receipts").fetchall() == [(1,)] + + +def test_observation_and_provenance_rollback_together_then_recover(rig): + conn, holder, processors, _, runtime = rig + runtime.before_frame(None, lambda _: True, 1000) + context = processors["left"].run_context + conn.execute("CREATE TRIGGER reject_quality BEFORE INSERT ON vitals_quality BEGIN SELECT RAISE(ABORT, 'full'); END") + candidate = main.VitalsCandidate("left", datetime.fromtimestamp(5, timezone.utc), 70, 30, 12, .8, None, 70, context) + assert not candidate.write(holder) + assert conn.execute("SELECT * FROM vitals").fetchall() == [] + assert conn.execute("SELECT * FROM biometric_processing_runs").fetchall() == [] + conn.execute("DROP TRIGGER reject_quality") + assert candidate.write(holder) + assert conn.execute("SELECT processing_run_id FROM vitals").fetchone()[0] == context.id + # History reset: a retained context can recreate its exact bundle. + conn.execute("DELETE FROM vitals") + conn.execute("DELETE FROM biometric_processing_runs") + conn.commit() + assert candidate.write(holder) + assert conn.execute("SELECT id FROM biometric_processing_runs").fetchone()[0] == context.id + + +def test_receipt_transaction_failure_retries_without_false_applied_state(rig): + conn, _, _, _, runtime = rig + project(conn) + runtime.before_frame(None, lambda _: True, 1000) + conn.execute("CREATE TRIGGER reject_receipt BEFORE INSERT ON biometric_runtime_receipts WHEN NEW.component='hrv_index' BEGIN SELECT RAISE(ABORT, 'full'); END") + runtime.after_frame(1000) + assert conn.execute("SELECT * FROM biometric_runtime_receipts").fetchall() == [] + assert runtime.problem is not None + conn.execute("DROP TRIGGER reject_receipt") + runtime.after_frame(2000) + assert restore_applied(conn, "left").generation == 1 + + +def test_old_schema_cannot_enable_custom_processing(rig): + conn, _, processors, _, runtime = rig + project(conn) + conn.execute("DROP TABLE biometric_runtime_receipts") + runtime.before_frame(None, lambda _: True, 1000) + assert not runtime.enabled + assert processors["left"].parameters.tracking_tolerance_bpm == 15 + assert runtime.problem is not None + + +def test_restore_rejects_corrupt_projection_envelope(rig): + conn, _, _, _, runtime = rig + project(conn) + runtime.before_frame(None, lambda _: True, 1000) + runtime.after_frame(1000) + conn.execute("UPDATE biometric_runtime_configs SET resolved_hash='corrupt'") + with pytest.raises(ValueError, match="envelope mismatch"): + restore_applied(conn, "left") + + +def test_heartbeat_is_bounded_and_repairs_missing_receipts(rig): + conn, _, _, clock, runtime = rig + project(conn) + runtime.before_frame(None, lambda _: True, 1000) + runtime.after_frame(1000) + runtime.after_frame(2000) + assert conn.execute("SELECT DISTINCT heartbeat_at_ms FROM biometric_runtime_receipts").fetchall() == [(1000,)] + conn.execute("DELETE FROM biometric_runtime_receipts") + conn.commit() + clock.value = 30 + runtime.before_frame(None, lambda _: True, 31000) + runtime.after_frame(31000) + assert restore_applied(conn, "left").generation == 1 + assert conn.execute("SELECT DISTINCT heartbeat_at_ms FROM biometric_runtime_receipts").fetchall() == [(31000,)] diff --git a/modules/piezo-processor/test_shadow.py b/modules/piezo-processor/test_shadow.py new file mode 100644 index 000000000..abb522cba --- /dev/null +++ b/modules/piezo-processor/test_shadow.py @@ -0,0 +1,146 @@ +"""Prospective adapter parity and failure boundaries using locked signal deps.""" +from copy import deepcopy +from pathlib import Path +import sys + +import numpy as np +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from common.biometrics.config import manifest +from common.biometrics.replay import RecordedContext, WaveformReplay +from common.biometrics.shadow import ProspectiveShadow + +START = 1_700_000_000_000 +CONTEXT = RecordedContext(('left',), True, False, True) + + +def configs(): + return {side: deepcopy(manifest()['defaults']) for side in ('left', 'right')} + + +def frames(): + for second in range(150): + values = (np.sin((np.arange(500) + second * 500) / 500 * 2 * np.pi * 1.2) * 1_000_000).astype(' 0 diff --git a/modules/piezo-processor/test_waveform_replay.py b/modules/piezo-processor/test_waveform_replay.py new file mode 100644 index 000000000..d9403da0c --- /dev/null +++ b/modules/piezo-processor/test_waveform_replay.py @@ -0,0 +1,173 @@ +"""Raw-frame replay checks using locked numerical dependencies and real estimates.""" +from copy import deepcopy +from dataclasses import FrozenInstanceError +from pathlib import Path +import sys + +import numpy as np +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from common.biometrics.config import manifest, config_hash +from common.biometrics.replay import WaveformReplay, RecordedContext, MAX_CHANNEL_BYTES +from common.biometrics.frames import decode_int32_samples, LOST_SAMPLE + +START = 1_700_000_000_000 +KNOWN = RecordedContext(('left',), True, False, True) + + +def make_replay(sink, start=START + 60_000, end=START + 150_000): + configs = {side: deepcopy(manifest()['defaults']) for side in ('left', 'right')} + return WaveformReplay(configs, retained_start_ms=START, + score_start_ms=start, score_end_ms=end, sink=sink) + + +def frames(): + for second in range(150): + samples = (np.sin((np.arange(500) + second * 500) / 500 * 2 * np.pi * 1.2) + * 1_000_000).astype(' 0 + assert replay.counts['scored_outputs'] == len(first) + assert replay.counts['accepted_frames'] == 150 + assert 'cold_start_without_serialized_state' in first[0].limitations + assert 'missing_pump_context' not in replay.limitations + with pytest.raises(FrozenInstanceError): + first[0].side = 'right' + with pytest.raises(ValueError, match='closed'): + replay.feed(START, {}, KNOWN) + + +def test_recorded_pump_guard_drops_frames_until_exact_event_time_boundary(): + replay = make_replay(lambda _: None, start=START, end=START + 20_000) + context = RecordedContext(('left', 'right'), False, False, True) + quiet = np.ones(500, dtype=' 0 → pump running - 2. Secondary: reference channel anomaly (|ref_delta| > 0.02 with - correlated active channel spikes = mechanical coupling) - 3. Guard period: 3 seconds trailing after pump-off (6 samples at ~2 Hz) + 1. Pump state change (frzHealth RPM start, stop or >10% step) → 3 s + guard. A steadily running pump is not gated: Pod 5 circulates + continuously while a side is on, and gating it zeroed whole nights. + 2. Reference channel anomaly (|ref_delta| > 0.02 with correlated + active channel spikes = mechanical coupling) When any gate is active, delta is forced to 0. Post-epoch processing: - Baseline subtraction: 5th percentile of trailing 30 epochs (10-min cold start) - - 3-epoch median filter for smoothing - Clamp to [0, 1000] + No median filter: a turn-over is usually one busy minute between still + ones, which a 3-epoch median zeroed. Sentinel values (-1.0 from firmware) are filtered via zero-order hold. Reference channel pair is used for common-mode rejection. @@ -70,6 +70,9 @@ from typing import Dict, List, Optional sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +# session_limits sits next to this file, also when another module's tests +# load main.py by path. Appended, so it never shadows their own modules. +sys.path.append(str(Path(__file__).resolve().parent)) import cbor2 from common.nats_follower import create_follower @@ -86,6 +89,7 @@ capsense_threshold, capsense2_channel_values, ) +from session_limits import ABSENCE_TIMEOUT_S, MAX_SESSION_S, MIN_SESSION_S # --------------------------------------------------------------------------- # Configuration @@ -95,10 +99,8 @@ BIOMETRICS_DB = Path(os.environ.get("BIOMETRICS_DATABASE_URL", "file:/persistent/sleepypod-data/biometrics.db").replace("file:", "")) SLEEPYPOD_DB = Path(os.environ.get("DATABASE_URL", "file:/persistent/sleepypod-data/sleepypod.db").replace("file:", "")) -# Seconds of continuous absence before we consider the user has left bed -ABSENCE_TIMEOUT_S = 120 -# Minimum session length to record (filters out accidental detections) -MIN_SESSION_S = 300 +# ABSENCE_TIMEOUT_S, MIN_SESSION_S and MAX_SESSION_S live in session_limits, +# shared with scripts/repair-capped-sessions.py. # Presence debounce / hysteresis: a present<->absent flip is only committed # after the new raw state has persisted for this many seconds. Without it, # brief capSense dropouts (sub-second flaps at 2 Hz while the occupant is @@ -107,12 +109,6 @@ # debug 2026-06-10). Must be < ABSENCE_TIMEOUT_S so debounced absence still # closes the session on a real bed-exit. PRESENCE_DEBOUNCE_S = 30.0 -# Hard upper bound on a single session. Belt-and-suspenders against a session -# that never closes (e.g. presence flapping that historically kept resetting -# the absence timer, producing 2736/2850min runaway durations). A continuous -# presence span beyond this is force-closed at the cap rather than persisted as -# a multi-day sleep_record. -MAX_SESSION_S = 16 * 3600 # How often to write a movement row (seconds) MOVEMENT_INTERVAL_S = 60 # How often to reload calibration profiles (seconds) @@ -137,6 +133,22 @@ BASELINE_PUBLISH_S = 15 * 60 # capSense2 profile threshold default (raw float units), as the calibrator. CAPSENSE2_PRESENCE_THRESHOLD = 6.0 +# Exit to a new empty level. The fast downward track also learns a lower +# EMPTY level — bedding pulled back before getting in (pod 88: -7.7 units +# for an hour) — and the made bed then reads above it for good: the empty +# bed stayed "occupied" until the MAX_SESSION_S cap (126 capped sessions). +# A side whose rise falls below EXIT_COLLAPSE_FRACTION of its occupied load +# (the peak of a OCCUPIED_LOAD_TAU_S average of the rise while occupied) +# and stays there for EXIT_SETTLE_S is empty: the baseline is reseeded to +# the reading and the exit dated where the collapse began. Sleepers there +# held 50-100% of their load; the empty bed read 10-20%. capSense2 only: +# capSense (Pod 3/4) has no recorded nights to set the fraction, and a +# sleeper there can hold a dip from +600 to +70 per channel for half an hour. +# Only observed time counts toward EXIT_SETTLE_S, so a sensor gap or a +# restart can't confirm a collapse it didn't see. +EXIT_COLLAPSE_FRACTION = 0.25 +EXIT_SETTLE_S = 10 * 60 +OCCUPIED_LOAD_TAU_S = 5 * 60 # In-progress session state survives restarts/reboots via this file. Without # it a reboot or service restart mid-session silently dropped the whole night. STATE_PATH = Path(os.environ.get( @@ -156,8 +168,11 @@ MIN_VALID_WALL_CLOCK_TS = 1577836800.0 # 2020-01-01 00:00:00 UTC # Pump gating for movement scoring (#230) -# Guard period after pump-off: 3 seconds = ~6 samples at 2 Hz capSense rate +# Guard period after a pump state change: 3 seconds = ~6 samples at 2 Hz PUMP_GUARD_S = 3.0 +# A running pump whose RPM moves by more than this fraction (e.g. the +# 1940 -> 3100 boost) counts as a state change. +PUMP_RPM_CHANGE_FRACTION = 0.1 # Reference channel anomaly threshold (capSense2 units) REF_ANOMALY_THRESHOLD = 0.02 # Pod 3/4 capSense movement. The integer channels change by several counts @@ -187,11 +202,6 @@ BASELINE_COLD_START_EPOCHS = 10 # ~10 minutes at 60s epochs # Percentile for baseline (5th percentile) BASELINE_PERCENTILE = 5 -# Median filter window (epochs). Not applied to capSense (Pod 3/4): there a -# turn-over is usually one busy minute between still ones, which the median -# zeroed, and the noise margin and level memory already reject the sensor's -# artifacts. -MEDIAN_FILTER_WINDOW = 3 # --------------------------------------------------------------------------- # Logging @@ -543,6 +553,30 @@ def usable_presence_input(record: dict, side: str) -> bool: for ch in ("out", "cen", "in")) +def _opt_float(v) -> Optional[float]: + return float(v) if v is not None else None + + +def _saved_collapse(state: dict) -> tuple: + """The saved (occupied_load, collapse_since, collapse_dwell_s), or no + collapse if any is unusable. An infinite load or a NaN dwell would confirm + a collapse on the next low sample; the rest of the baseline is still good.""" + try: + load = _opt_float(state.get("occupied_load")) + since = _opt_float(state.get("collapse_since")) + dwell = float(state.get("collapse_dwell_s") or 0.0) + except (TypeError, ValueError): + load = since = dwell = math.nan + if ((load is not None and not (math.isfinite(load) and load > 0)) + or (since is not None and not math.isfinite(since)) + or not (math.isfinite(dwell) and dwell >= 0)): + log.warning("Ignoring invalid saved collapse state (load %r, since %r, dwell %r)", + state.get("occupied_load"), state.get("collapse_since"), + state.get("collapse_dwell_s")) + return None, None, 0.0 + return load, since, dwell + + class AdaptiveBaseline: """Self-adjusting empty-bed level for one side's capacitance channels. @@ -556,6 +590,9 @@ class AdaptiveBaseline: with time constant BASELINE_UP_TAU_S. - Contamination: any reading below baseline pulls it down with BASELINE_DOWN_TAU_S — a body only adds capacitance. + - Exit to a new empty level: an occupied rise that collapses below + EXIT_COLLAPSE_FRACTION of the occupied load for EXIT_SETTLE_S reseeds + the baseline (see observe_load). - Seeding: saved state, else the calibration profile (including a manual recalibration, adopted whenever a newer one appears), else the first sample. @@ -566,7 +603,9 @@ class AdaptiveBaseline: def __init__(self, fmt: str, means: dict, threshold: float, ref_mean: Optional[float] = None, source: str = "bootstrap", - profile_seen_at: Optional[float] = None, absence_ready: bool = False): + profile_seen_at: Optional[float] = None, absence_ready: bool = False, + occupied_load: Optional[float] = None, collapse_since: Optional[float] = None, + collapse_dwell_s: float = 0.0): self.fmt = fmt self.means = {ch: float(v) for ch, v in means.items()} self.threshold = float(threshold) @@ -584,6 +623,14 @@ def __init__(self, fmt: str, means: dict, threshold: float, # deliberately skipped, so it isn't re-adopted every reload. self.profile_seen_at = profile_seen_at self._last_track_ts: Optional[float] = None + # Exit to a new empty level (observe_load). Both persist, so a restart + # after the bed emptied still recognises the collapse. + self.occupied_load = occupied_load + self.collapse_since = collapse_since + self.collapse_dwell_s = collapse_dwell_s + self._load_avg: Optional[float] = None + self._load_obs_s = 0.0 + self._load_last_ts: Optional[float] = None @property def exit_threshold(self) -> float: @@ -618,17 +665,25 @@ def from_state(cls, state: Optional[dict]): if not isinstance(state, dict): return None try: - return cls(str(state["format"]), dict(state["means"]), float(state["threshold"]), + fmt, means, threshold = str(state["format"]), dict(state["means"]), float(state["threshold"]) + except (KeyError, TypeError, ValueError): + return None + occupied_load, collapse_since, collapse_dwell_s = _saved_collapse(state) + try: + return cls(fmt, means, threshold, ref_mean=state.get("ref_mean"), source="state", profile_seen_at=state.get("profile_seen_at"), - absence_ready=state.get("absence_ready") is True) + absence_ready=state.get("absence_ready") is True, + occupied_load=occupied_load, collapse_since=collapse_since, + collapse_dwell_s=collapse_dwell_s) except (KeyError, TypeError, ValueError): return None def snapshot(self) -> dict: return {"format": self.fmt, "means": self.means, "threshold": self.threshold, "ref_mean": self.ref_mean, "profile_seen_at": self.profile_seen_at, - "absence_ready": self.absence_ready} + "absence_ready": self.absence_ready, "occupied_load": self.occupied_load, + "collapse_since": self.collapse_since, "collapse_dwell_s": self.collapse_dwell_s} def values(self, record: dict, side: str) -> Optional[dict]: """This format's per-channel values for a frame, or None if unusable.""" @@ -677,6 +732,60 @@ def observe_absence_reference(self, ts: float, values: Optional[dict]) -> None: and abs(self.deviation(values)) <= self.exit_threshold): self.absence_ready = True + def observe_load(self, ts: float, values: dict, occupied: bool) -> Optional[float]: + """Detect an exit to a new empty level. Called once per usable sample + before the presence decision; `occupied` is the side's level latch. + + Returns the ts the collapse began when the baseline was just reseeded + to this reading (the exit time), else None.""" + if self.fmt != "capSense2": + return None + dev = self.deviation(values) + if not occupied or dev <= self.exit_threshold: + # Unoccupied, or about to read absent anyway. + self.occupied_load = self.collapse_since = None + self.collapse_dwell_s = 0.0 + self._load_avg = self._load_last_ts = None + self._load_obs_s = 0.0 + return None + dt = 0.0 if self._load_last_ts is None else ts - self._load_last_ts + self._load_last_ts = ts + dt = max(0.0, min(dt, BASELINE_MAX_STEP_S)) + if self.collapse_since is None: + # Frozen during a collapse, so the drop can't lower its own bar. + # A plain mean until OCCUPIED_LOAD_TAU_S has been observed, and no + # peak before then: one transient (getting in, or the first sample + # after a restart) must not set the bar for the night. + self._load_obs_s += dt + if self._load_avg is None: + self._load_avg = dev + elif self._load_obs_s > 0: + alpha = dt / min(self._load_obs_s, OCCUPIED_LOAD_TAU_S) + self._load_avg += alpha * (dev - self._load_avg) + if self._load_obs_s >= OCCUPIED_LOAD_TAU_S: + self.occupied_load = max(self.occupied_load or 0.0, self._load_avg) + # A load at or under 4x the exit threshold leaves no reading both + # under the fraction and above the exit threshold, so the rule is off: + # such a low-gain side keeps the old exit path only. + if EXIT_COLLAPSE_FRACTION * (self.occupied_load or 0.0) <= self.exit_threshold: + self.collapse_since = None + self.collapse_dwell_s = 0.0 + return None + if dev >= EXIT_COLLAPSE_FRACTION * self.occupied_load: + self.collapse_since = None + self.collapse_dwell_s = 0.0 + return None + if self.collapse_since is None: + self.collapse_since = ts + self.collapse_dwell_s = 0.0 + return None + self.collapse_dwell_s += dt + if self.collapse_dwell_s < EXIT_SETTLE_S: + return None + since = self.collapse_since + self.reseed(values, "exit-rebase") + return since + def track(self, ts: float, values: dict, occupied: bool) -> None: """Follow the empty-bed level. Called once per sample after the presence decision; `occupied` is the committed (debounced) state.""" @@ -703,6 +812,10 @@ def reseed(self, values: dict, source: str) -> None: self._absence_reference = sum(self.means.values()) self._absence_drop_since = self._absence_rise_since = None self._absence_last_ts = None + self.occupied_load = self.collapse_since = None + self.collapse_dwell_s = 0.0 + self._load_avg = self._load_last_ts = None + self._load_obs_s = 0.0 def to_params(self) -> dict: """Calibration-profile params (the shape the calibrators write).""" @@ -757,17 +870,6 @@ def _percentile(values: List[int], pct: int) -> int: return s[idx] -def _median(values: List[int]) -> int: - """Compute the median of a list of integers.""" - if not values: - return 0 - s = sorted(values) - n = len(s) - if n % 2 == 1: - return s[n // 2] - return (s[n // 2 - 1] + s[n // 2]) // 2 - - # --------------------------------------------------------------------------- # Pump artifact gating for capSense2 movement scoring (#230) # --------------------------------------------------------------------------- @@ -798,12 +900,16 @@ def _extract_ref_delta(record: dict, side: str, class PumpGateCapSense: - """Three-signal pump artifact gate for capSense2 movement scoring. + """Pump artifact gate for capSense2 movement scoring. - Detects pump activity via: - 1. Primary: frzHealth pump RPM > 0 on either side - 2. Secondary: reference channel anomaly with correlated active channels - 3. Guard period: 3 seconds trailing after pump-off + Gates a side's movement delta: + 1. For PUMP_GUARD_S after that side's pump changes state (start, stop, + or an RPM step beyond PUMP_RPM_CHANGE_FRACTION). A steadily running + pump is not gated: on Pod 5 it circulates at ~1940 RPM for as long + as the side is on, its vibration doesn't raise the delta floor (an + empty side scored ~25/epoch with the pump off, at 1940 and at 3100 + RPM), and gating it zeroed movement for whole nights. + 2. On a reference channel anomaly with correlated active channels. When gate is active, movement delta should be forced to 0. @@ -814,16 +920,18 @@ class PumpGateCapSense: def __init__(self): # Per-side pump RPM state from frzHealth records self._pump_rpm: Dict[str, float] = {"left": 0.0, "right": 0.0} - # Per-side timestamp (monotonic) when that pump last turned off — + # Per-side timestamp (monotonic) of that pump's last state change — # for the guard period. Per-side because gating both beds on either - # pump under-counted the movement table during long pump runtimes; - # cross-side mechanical coupling is what Signal 2 (correlated - # ref-anomaly) exists to catch. -inf = never turned off (0.0 would - # falsely gate the first PUMP_GUARD_S after process start, since - # time.monotonic() has an arbitrary, possibly near-zero origin). - self._pump_off_at: Dict[str, float] = {"left": float("-inf"), "right": float("-inf")} - # Whether each pump was active on previous check (for detecting pump-off transition) - self._was_pump_active: Dict[str, bool] = {"left": False, "right": False} + # pump under-counted the movement table; cross-side mechanical + # coupling is what Signal 2 (correlated ref-anomaly) exists to catch. + # -inf = never changed (0.0 would falsely gate the first PUMP_GUARD_S + # after process start, since time.monotonic() has an arbitrary, + # possibly near-zero origin). + self._pump_change_at: Dict[str, float] = {"left": float("-inf"), "right": float("-inf")} + # Sides whose frzHealth reports a real pump RPM. Their frzTherm + # `power` is the TEC drive, not the pump; reading it as pump state + # flipped the pump "off" and back "on" every ~10 s. + self._rpm_reported: set = set() # Reference channel anomaly state self._ref_anomaly_active: bool = False @@ -851,6 +959,7 @@ def update_pump_state(self, record: dict) -> None: if val is not None: try: rpm = float(val) + self._rpm_reported.add(side) except (TypeError, ValueError): pass break @@ -861,6 +970,7 @@ def update_pump_state(self, record: dict) -> None: if val is not None: try: rpm = float(val) + self._rpm_reported.add(side) except (TypeError, ValueError): pass # Also check pumpDuty as fallback — any duty > 0 means pump is running @@ -878,6 +988,8 @@ def update_pump_state(self, record: dict) -> None: pass elif rtype == "frzTherm": + if side in self._rpm_reported: + continue # frzTherm may carry pump duty cycle for key in ("pumpDuty", "pump_duty", "duty", "pumpRpm", "pump_rpm", "power"): @@ -889,16 +1001,13 @@ def update_pump_state(self, record: dict) -> None: pass break + # Start, stop or a step in speed opens this side's guard period. + prev = self._pump_rpm[side] + if (prev > 0) != (rpm > 0) or ( + prev > 0 and abs(rpm - prev) > PUMP_RPM_CHANGE_FRACTION * prev): + self._pump_change_at[side] = time.monotonic() self._pump_rpm[side] = rpm - # Track per-side pump-off transitions for the guard period - for side in ("left", "right"): - pump_active = self._pump_rpm[side] > 0 - if self._was_pump_active[side] and not pump_active: - # This side's pump just turned off — start its guard period - self._pump_off_at[side] = time.monotonic() - self._was_pump_active[side] = pump_active - def is_gated(self, record: dict, side: str, channel_deltas: Optional[List[float]] = None, baselines: Optional[dict] = None) -> bool: @@ -914,15 +1023,10 @@ def is_gated(self, record: dict, side: str, Returns True if the delta should be suppressed. """ - # Signal 1: frzHealth pump RPM — this side's pump only. Gating both - # sides on either pump zeroed real movement on the idle side for the - # whole pump runtime; the other pump's mechanical coupling (if any) - # is caught by the correlated ref-anomaly check below. - if self._pump_rpm.get(side, 0.0) > 0: - return True - - # Signal 3: Guard period (checked before ref anomaly since it's cheap) - if time.monotonic() - self._pump_off_at.get(side, float("-inf")) < PUMP_GUARD_S: + # Signal 1: this side's pump just changed state. The other pump's + # mechanical coupling (if any) is caught by the correlated + # ref-anomaly check below. + if time.monotonic() - self._pump_change_at.get(side, float("-inf")) < PUMP_GUARD_S: return True # Signal 2: Reference channel anomaly (capSense2 only) @@ -955,6 +1059,9 @@ class SideObservation: delta: float values: Optional[dict] record: dict + # Set when this frame rebased the baseline to an empty bed at a new + # level: the ts the occupied load collapsed (AdaptiveBaseline.observe_load). + exit_at: Optional[float] = None @dataclass @@ -981,9 +1088,8 @@ class SessionTracker: _last_movement_write: float = field(default_factory=time.time) _prev_values: Optional[list] = None # previous sample's channel values _scale_factor: float = 10.0 # default for capSense2; updated on first record - # Epoch score history for baseline subtraction and median filter + # Epoch score history for baseline subtraction _epoch_scores: deque = field(default_factory=lambda: deque(maxlen=BASELINE_TRAILING_EPOCHS)) - _median_buf: deque = field(default_factory=lambda: deque(maxlen=MEDIAN_FILTER_WINDOW)) _pump_gated_samples: int = 0 # counter for logging # capSense (Pod 3/4) per-channel delta history and typical delta; see # CAPSENSE_NOISE_K. @@ -1022,6 +1128,14 @@ class SessionTracker: # evidence. A merged home-side absence requires usable evidence from both. _vitals_presence: Optional[bool] = None _vitals_evidence_ts: Optional[float] = None + # Pod 5 firmware emits an all -1.0 capSense2 frame every 5 min; the frame + # after it is still settling (channels above 28.5 read exactly 28.5), so + # it is skipped too. Its down-and-back-up delta added ~110 to one epoch + # in five on an occupied side. + _cap2_settling: bool = False + # Last sample this side's level read occupied (not persisted). Merged + # mode dates a rebase exit no earlier than the other side's. + _level_present_ts: Optional[float] = None def snapshot(self) -> dict: """JSON-serializable state needed to resume after a restart.""" @@ -1110,6 +1224,10 @@ def _sync_baseline(self, record: dict) -> Optional[AdaptiveBaseline]: if adopted is not None: log.info("%s: presence baseline from calibration profile (created %s)", self.side, created_at) + if b is not None and b.collapse_since is not None: + log.info("%s: the new profile drops a collapse pending since %s; " + "the occupied load is learned again", self.side, + datetime.fromtimestamp(b.collapse_since, tz=timezone.utc).isoformat()) b = adopted elif b is not None: b.profile_seen_at = created_at # wrong format — stop re-checking @@ -1147,21 +1265,36 @@ def observe(self, ts: float, record: dict) -> Optional["SideObservation"]: self._last_ts = ts rtype = record.get("type", "") usable = usable_presence_input(record, self.side) + if rtype == "capSense2": + data = record.get(self.side) + raw = data.get("values") if isinstance(data, dict) else None + sentinel = isinstance(raw, (list, tuple)) and CAPSENSE2_SENTINEL in raw[:8] + settling, self._cap2_settling = self._cap2_settling, sentinel + usable = usable and not settling baseline = self._sync_baseline(record) if usable else self.baseline values = baseline.values(record, self.side) if usable and baseline is not None else None present: Optional[bool] = None # unusable frame: no new evidence + exit_at: Optional[float] = None if baseline is not None: baseline.observe_absence_reference(ts, values) if values is not None: + exit_at = baseline.observe_load(ts, values, self._level_present) + if exit_at is not None: + log.warning("%s: occupied load collapsed at %s and stayed under %.0f%% for %d min — " + "presence baseline rebased to the empty bed's new level", + self.side, datetime.fromtimestamp(exit_at, tz=timezone.utc).isoformat(), + EXIT_COLLAPSE_FRACTION * 100, EXIT_SETTLE_S // 60) present = baseline.is_present(values, self._level_present) self._level_present = present + if present: + self._level_present_ts = ts # Movement's capSense2 common-mode rejection uses the baseline's ref. baselines = ({"ref": {"mean": baseline.ref_mean}} if baseline is not None and baseline.ref_mean is not None else None) # Set scale factor based on sensor type (Pod 3/4 int vs Pod 5 float). # capSense scores the excess over its noise (see CAPSENSE_NOISE_K) - # one to one, and skips the median filter (keyed on this factor). + # one to one. if rtype == "capSense" and self._scale_factor != 1.0: self._scale_factor = 1.0 elif rtype == "capSense2" and self._scale_factor != 10.0: @@ -1189,7 +1322,7 @@ def observe(self, ts: float, record: dict) -> Optional["SideObservation"]: else: # Sentinel or invalid — skip delta, keep previous (zero-order hold) delta = 0.0 - return SideObservation(ts, present, delta, values, record) + return SideObservation(ts, present, delta, values, record, exit_at) def _capsense_movement(self, channel_deltas: list, values: Optional[list] = None) -> float: @@ -1217,9 +1350,20 @@ def _capsense_movement(self, channel_deltas: list, excess[i] = 0.0 return sum(excess) - def commit(self, ts: float, present: Optional[bool], delta: float) -> bool: + def commit(self, ts: float, present: Optional[bool], delta: float, + exit_at: Optional[float] = None) -> bool: """Advance the session with one sample's presence and movement. + `exit_at`: a baseline rebase found the bed empty since then; if this + sample reads absent, the absence is dated from it so the session + ends where the occupant left, not EXIT_SETTLE_S later. Returns True if the session was just force-closed at MAX_SESSION_S.""" + if (exit_at is not None and present is False and self._debounced_present + and self._pending_state is None): + self._pending_state = False + self._pending_since = max(exit_at, self._interval_start or exit_at) + # Observed evidence of the exit: outranks the restart fallback, + # which would date it at the last presence before the restart. + self._resumed_last_present = None self._update(ts, self._debounced_present if present is None else present, delta) capped, self._cap_closed = self._cap_closed, False return capped @@ -1231,6 +1375,12 @@ def settle(self, obs: "SideObservation", reset: bool = False) -> None: if obs.values is None or self.baseline is None: return if reset: + if self.baseline.collapse_since is not None: + # Not confirmed (under EXIT_SETTLE_S observed), so not an exit. + log.info("%s: the cap discards a collapse pending since %s (%.0f s observed of the %d s needed)", + self.side, + datetime.fromtimestamp(self.baseline.collapse_since, tz=timezone.utc).isoformat(), + self.baseline.collapse_dwell_s, EXIT_SETTLE_S) self.baseline.reseed(obs.values, "cap-reset") log.warning("%s: presence baseline reset to current level after a capped session", self.side) @@ -1265,7 +1415,7 @@ def process(self, ts: float, record: dict) -> None: obs = self.observe(ts, record) if obs is None: return - capped = self.commit(ts, obs.present, obs.delta) + capped = self.commit(ts, obs.present, obs.delta, obs.exit_at) self.settle(obs, reset=capped) self.publish_vitals_presence(ts, [(self, obs)]) @@ -1391,18 +1541,21 @@ def _close_session(self, left_ts: float) -> None: log.info("%s: session too short (%ds), discarding", self.side, duration_s) wrote = True # treat discard as final — nothing to retry else: - # Close any open interval + # Close any open interval, on copies: a failed write keeps the + # session for a retry, which must not append the interval again. + present_intervals = list(self._present_intervals) + absent_intervals = list(self._absent_intervals) if self._interval_start is not None: if self._was_present: - self._present_intervals.append([self._interval_start, left_ts]) + present_intervals.append([self._interval_start, left_ts]) elif left_ts > self._interval_start: - self._absent_intervals.append([self._interval_start, left_ts]) + absent_intervals.append([self._interval_start, left_ts]) wrote = write_sleep_record( self.db, self.side, self._session_start, left_at, duration_s, self._exit_count, - self._present_intervals, self._absent_intervals, + present_intervals, absent_intervals, ) if wrote: log.info("%s: session recorded — %.1f hr, %d exits", @@ -1434,7 +1587,6 @@ def _reset_session(self) -> None: self._prev_values = None # avoid stale delta on next session start self._movement_buf = [] # discard leftover deltas from session end self._epoch_scores.clear() - self._median_buf.clear() self._pump_gated_samples = 0 self._resumed_last_present = None @@ -1471,18 +1623,9 @@ def _flush_movement(self, ts: float) -> None: # Cold start: not enough history yet, skip baseline subtraction score_after_baseline = raw_score - # Step 3: 3-epoch median filter for smoothing (not for capSense) - self._median_buf.append(score_after_baseline) - if self._scale_factor == 1.0: - filtered_score = score_after_baseline - elif len(self._median_buf) >= MEDIAN_FILTER_WINDOW: - filtered_score = _median(list(self._median_buf)) - else: - # Not enough epochs yet for median filter, pass through - filtered_score = score_after_baseline - - # Step 4: Final clamp to [0, 1000] - total = max(0, min(1000, filtered_score)) + # Step 3: Final clamp to [0, 1000]. No median filter: it zeroed a + # turn-over, usually one busy minute between still ones. + total = max(0, min(1000, score_after_baseline)) wrote = write_movement(self.db, self.side, datetime.fromtimestamp(ts, tz=timezone.utc), total) @@ -1534,7 +1677,13 @@ def process_single_sleeper(home: SessionTracker, away: SessionTracker, evidence = [o.present for o in (h, a) if o is not None and o.present is not None] present = any(evidence) if evidence else None delta = h.delta + (a.delta if a is not None else 0.0) - capped = home.commit(ts, present, delta) + exits = [o.exit_at for o in (h, a) if o is not None and o.exit_at is not None] + if exits: + # The sleeper may have stayed on the other side after this one's + # load collapsed: the merged exit is the later of the two. + exits += [t._level_present_ts for t, o in ((home, h), (away, a)) + if (o is None or o.exit_at is None) and t._level_present_ts is not None] + capped = home.commit(ts, present, delta, max(exits) if exits else None) # A capped session in merged mode may be held open by either side's load. home.settle(h, reset=capped) if a is not None: diff --git a/modules/sleep-detector/session_limits.py b/modules/sleep-detector/session_limits.py new file mode 100644 index 000000000..107cb9e52 --- /dev/null +++ b/modules/sleep-detector/session_limits.py @@ -0,0 +1,17 @@ +""" +Session limits shared by the detector (main.py) and +scripts/repair-capped-sessions.py, which must agree on them. No imports and +no side effects, so the repair script can load it without the detector's +runtime dependencies. +""" + +# Seconds of continuous absence before we consider the user has left bed +ABSENCE_TIMEOUT_S = 120 +# Minimum session length to record (filters out accidental detections) +MIN_SESSION_S = 300 +# Hard upper bound on a single session. Belt-and-suspenders against a session +# that never closes (e.g. presence flapping that historically kept resetting +# the absence timer, producing 2736/2850min runaway durations). A continuous +# presence span beyond this is force-closed at the cap rather than persisted as +# a multi-day sleep_record. +MAX_SESSION_S = 16 * 3600 diff --git a/modules/sleep-detector/test_main.py b/modules/sleep-detector/test_main.py index b4df5ed2d..99c664a91 100644 --- a/modules/sleep-detector/test_main.py +++ b/modules/sleep-detector/test_main.py @@ -226,6 +226,28 @@ def fake_open(): assert len(replaced) == 1 assert main._db_write_failures == 0 + def test_failed_close_does_not_duplicate_intervals_on_retry(self): + # The open interval was appended before the write; a failed write + # kept it, and every retry appended another copy. + import json + t = _tracker() + good = t.db.conn + base = 1_777_000_000.0 + samples = [(base, True), (base + 31, True)] + ts = base + 31 + while ts < base + main.MAX_SESSION_S - 600: + ts += 600 + samples.append((ts, True)) + _feed(t, samples) + t.db.conn = _FailingConn() + _feed(t, [(base + main.MAX_SESSION_S + 10, True), (base + main.MAX_SESSION_S + 20, True)]) + assert t._session_start is not None, "a failed write must keep the session" + t.db.conn = good + _feed(t, [(base + main.MAX_SESSION_S + 30, True)]) + rows = good.execute("SELECT present_intervals FROM sleep_records").fetchall() + assert len(rows) == 1 + assert json.loads(rows[0][0]) == [[base, base + main.MAX_SESSION_S]] + class TestSharedConnectionHolder: """Both SessionTrackers read connections from one DBHolder so reconnect @@ -248,9 +270,14 @@ def test_reconnect_swaps_holder_observed_by_both_trackers(self, monkeypatch): class TestPumpGatePerSide: """Gating both beds whenever EITHER pump ran zeroed real movement on the - idle side for the whole pump runtime, under-counting the movement table. - Signal 1 (RPM) and Signal 3 (guard period) are now per-side; cross-side - mechanical coupling remains covered by Signal 2 (correlated ref-anomaly).""" + idle side, so the gate and its guard period are per-side; cross-side + mechanical coupling remains covered by the correlated ref-anomaly check. + + Only pump state changes gate. Pod 5 circulates at ~1940 RPM for as long + as a side is on; gating the running pump zeroed 97% of a night's epochs + (pod 88, 2026-10-09).""" + + T = 1000.0 def _frz(self, left_rpm, right_rpm): return { @@ -259,55 +286,75 @@ def _frz(self, left_rpm, right_rpm): "right": {"pumpRpm": right_rpm}, } - def test_only_running_side_is_gated(self): - gate = main.PumpGateCapSense() - gate.update_pump_state(self._frz(left_rpm=3000, right_rpm=0)) - - assert gate.is_gated({}, "left") is True - assert gate.is_gated({}, "right") is False + def _at(self, gate, dt, record=None): + with patch("main.time.monotonic", return_value=self.T + dt): + if record is not None: + gate.update_pump_state(record) + return None + return gate.is_gated({}, "left"), gate.is_gated({}, "right") - def test_both_sides_gated_when_both_pumps_run(self): + def test_pump_start_gates_that_side_for_the_guard_period(self): gate = main.PumpGateCapSense() - gate.update_pump_state(self._frz(left_rpm=3000, right_rpm=2800)) + self._at(gate, 0, self._frz(left_rpm=1940, right_rpm=0)) + assert self._at(gate, 1) == (True, False) + assert self._at(gate, main.PUMP_GUARD_S + 0.1) == (False, False) - assert gate.is_gated({}, "left") is True - assert gate.is_gated({}, "right") is True + def test_steadily_running_pump_is_not_gated(self): + gate = main.PumpGateCapSense() + self._at(gate, 0, self._frz(left_rpm=1940, right_rpm=1940)) + for k in range(1, 30): # ~10 s reports, small jitter + self._at(gate, 10 * k, self._frz(left_rpm=1940 + k % 3 * 4, right_rpm=1944)) + assert self._at(gate, 10 * k + 0.5) == (False, False) - def test_guard_period_applies_per_side(self): + def test_pump_stop_gates_only_the_side_that_ran(self): gate = main.PumpGateCapSense() - gate.update_pump_state(self._frz(left_rpm=3000, right_rpm=0)) - # Left pump turns off → left enters its guard period; right never ran. - gate.update_pump_state(self._frz(left_rpm=0, right_rpm=0)) + self._at(gate, 0, self._frz(left_rpm=3000, right_rpm=0)) + self._at(gate, 60, self._frz(left_rpm=0, right_rpm=0)) + assert self._at(gate, 61) == (True, False) + assert self._at(gate, 60 + main.PUMP_GUARD_S + 0.1) == (False, False) - assert gate.is_gated({}, "left") is True, "guard period must gate the side that ran" - assert gate.is_gated({}, "right") is False, "idle side must not inherit the guard" + def test_rpm_step_gates(self): + gate = main.PumpGateCapSense() + self._at(gate, 0, self._frz(left_rpm=1940, right_rpm=0)) + self._at(gate, 60, self._frz(left_rpm=3100, right_rpm=0)) + assert self._at(gate, 61) == (True, False) def test_no_pumps_no_gate(self): gate = main.PumpGateCapSense() - gate.update_pump_state(self._frz(left_rpm=0, right_rpm=0)) - - assert gate.is_gated({}, "left") is False - assert gate.is_gated({}, "right") is False + self._at(gate, 0, self._frz(left_rpm=0, right_rpm=0)) + assert self._at(gate, 1) == (False, False) def test_captured_nats_nested_health_rpm(self): gate = main.PumpGateCapSense() - gate.update_pump_state({ + self._at(gate, 0, { "type": "frzHealth", "left": {"pump": {"mode": "pwm", "rpm": 1868, "water": True}}, "right": {"pump": {"mode": "pwm", "rpm": 0, "water": True}}, }) - assert gate.is_gated({}, "left") is True - assert gate.is_gated({}, "right") is False + assert self._at(gate, 1) == (True, False) def test_captured_nats_therm_power(self): + # Without frzHealth RPM, frzTherm power is the only pump signal. gate = main.PumpGateCapSense() - gate.update_pump_state({ + self._at(gate, 0, { "type": "frzTherm", "left": {"power": 0.024}, "right": {"power": 0.0}, }) - assert gate.is_gated({}, "left") is True - assert gate.is_gated({}, "right") is False + assert self._at(gate, 1) == (True, False) + + def test_therm_power_ignored_once_health_reports_rpm(self): + # Pod 5 frzTherm power is the TEC drive; it drops to 0 while the pump + # keeps running, and used to toggle the pump state every ~10 s. + gate = main.PumpGateCapSense() + health = {"type": "frzHealth", "left": {"pump": {"rpm": 1940}}, "right": {"pump": {"rpm": 0}}} + self._at(gate, 0, health) + for k in range(1, 10): + self._at(gate, 10 * k, {"type": "frzTherm", "left": {"power": 0.0 if k % 2 else 0.007}, + "right": {"power": 0.0}}) + assert self._at(gate, 10 * k + 0.5) == (False, False) + self._at(gate, 10 * k + 1, health) + assert self._at(gate, 10 * k + 1.5) == (False, False) def _tracker(): @@ -691,7 +738,8 @@ def test_light_load_below_enter_threshold_is_not_absorbed(self): def test_hysteresis_keeps_session_through_partial_dip(self): # Once in bed, dipping to +70/channel (+210 summed, above the 150 - # exit) must not end the session. + # exit) must not end the session, though it is under + # EXIT_COLLAPSE_FRACTION of the load: that rule is capSense2 only. t = _live_tracker() ts = _run(t, self.T0, 600, 0) ts = _run(t, ts, 3600, 600) @@ -1133,7 +1181,7 @@ def test_moving_to_a_new_position_still_counts(self): before = len(t._epoch_scores) self._feed_levels(t, ts, 5 * 60, rng, lambda k: {"out": 400, "cen": -300, "in": 250}) assert max(list(t._epoch_scores)[before:]) >= 500 - # ...and it reaches the stored minute (no median filter on capSense). + # ...and it reaches the stored minute (no median filter). around = [m for ts, m in self._minutes(t) if moved_at - 60 <= ts <= moved_at + 120] assert max(around) >= 500 @@ -1159,7 +1207,6 @@ def _flush_one(t, scale, raw, at): t._scale_factor = scale t._session_start = at - 3600 t._epoch_scores.extend([0] * 20) - t._median_buf.extend([0, 0]) t._movement_buf = [raw] t._last_movement_write = at - 61 t._flush_movement(at) @@ -1169,11 +1216,11 @@ def test_capsense_writes_a_lone_busy_epoch_unfiltered(self): self._flush_one(t, 1.0, 700.0, self.T0) assert [m for _, m in self._minutes(t)] == [700] - def test_capsense2_keeps_the_median_filter(self): - # Pod 5: a lone busy epoch between still ones is still smoothed away. + def test_capsense2_writes_a_lone_busy_epoch_unfiltered(self): + # Pod 5: the 3-epoch median zeroed a lone turn-over minute too. t = self._tracker() self._flush_one(t, 10.0, 70.0, self.T0) - assert [m for _, m in self._minutes(t)] == [0] + assert [m for _, m in self._minutes(t)] == [700] def _warm(self, t, noise=5.0): for _ in range(main.CAPSENSE_NOISE_MIN_SAMPLES): @@ -1250,3 +1297,353 @@ def test_gaps_and_invalid_samples_do_not_count_as_unloading_time(self): for ts in range(240, 271, 5): b.observe_absence_reference(ts, EMPTY) assert b.absence_ready + + +def _cap2(a, b, c, side="left", ref=1.16): + return {"type": "capSense2", side: {"values": [a, a, b, b, c, c, ref, ref]}} + + +# Pod 5 empty-bed zone means (pod 88 left side). +EMPTY2 = (16.6, 15.0, 21.6) + + +class TestExitToNewEmptyLevel: + """Pod 88 2026-10-09: bedding pulled back before bed lowered the empty + level by 7.7; the fast downward track learned it; the made bed then read + +7.5 over that baseline, above the exit threshold, so the empty bed stayed + "occupied" until the 16 h cap (126 of 782 sessions).""" + + T0 = 1_791_500_000.0 + EXIT_TS = 1791559700 # 2026-10-09 15:28:20 UTC, from the recorded frames + + @staticmethod + def _run(t, start, seconds, rise, step=5.0): + """capSense2 frames `rise` (summed) above EMPTY2, spread evenly.""" + ts = start + while ts < start + seconds: + t.process(ts, _cap2(*(v + rise / 3 for v in EMPTY2))) + ts += step + return ts + + def test_recorded_night_closes_at_the_real_exit(self): + import csv, gzip + path = Path(__file__).resolve().parent / "fixtures" / "pod5_left_bedding_shift.csv.gz" + with gzip.open(path, "rt") as f: + frames = [r for r in csv.reader(line for line in f if not line.startswith("#"))][1:] + t = _live_tracker() + for ts, a, b, c in frames: + t.process(float(ts), _cap2(float(a), float(b), float(c))) + rows = _rows_full(t) + assert t._session_start is None, "the empty bed held the session open" + night = [r for r in rows if r[0] <= self.EXIT_TS - 6 * 3600] + assert len(night) == 1 + entered, left_at, duration_s, _ = night[0] + assert abs(left_at - self.EXIT_TS) <= main.ABSENCE_TIMEOUT_S + assert duration_s < main.MAX_SESSION_S + + def test_bedding_pulled_back_before_bed_does_not_hold_the_session(self): + t = _live_tracker() + ts = self._run(t, self.T0, 3600, 0) # made bed, empty + ts = self._run(t, ts, 3600, -7.7) # covers pulled back + ts = self._run(t, ts, 6 * 3600, 30) # asleep + exit_ts = ts + ts = self._run(t, ts, 3600, 0) # up, bed made again + rows = _rows_full(t) + assert t._session_start is None + assert len(rows) == 1 + assert abs(rows[0][1] - exit_ts) <= main.ABSENCE_TIMEOUT_S + assert t.baseline.source == "exit-rebase" + + def test_still_sleeper_at_partial_load_keeps_the_session(self): + t = _live_tracker() + ts = self._run(t, self.T0, 3600, 0) + ts = self._run(t, ts, 2 * 3600, 30) + ts = self._run(t, ts, 2 * 3600, 12) # 40% of the load, still + ts = self._run(t, ts, 2 * 3600, 30) + ts = self._run(t, ts, 1800, 0) + rows = _rows_full(t) + assert len(rows) == 1 + assert rows[0][3] == 1 # only the morning exit + + def test_brief_collapse_keeps_the_session(self): + t = _live_tracker() + ts = self._run(t, self.T0, 3600, 0) + ts = self._run(t, ts, 3600, -7.7) + ts = self._run(t, ts, 3 * 3600, 30) + ts = self._run(t, ts, main.EXIT_SETTLE_S - 120, 0) # sat up, edge of bed + ts = self._run(t, ts, 3 * 3600, 30) + assert t._session_start is not None + assert _rows_full(t) == [] + + def test_collapse_survives_a_restart(self): + import json + t = _live_tracker() + ts = self._run(t, self.T0, 3600, 0) + ts = self._run(t, ts, 3600, -7.7) + ts = self._run(t, ts, 3 * 3600, 30) + exit_ts = ts + ts = self._run(t, ts, 300, 0) + t2 = _live_tracker() + t2.db = t.db + t2.restore(json.loads(json.dumps(t.snapshot())), now=ts + 30) + self._run(t2, ts + 30, 1800, 0) + rows = _rows_full(t2) + assert t2._session_start is None + assert len(rows) == 1 + assert abs(rows[0][1] - exit_ts) <= main.ABSENCE_TIMEOUT_S + + def test_one_transient_does_not_set_the_load(self): + # Two frames at 10x the load while getting in (or right after a + # restart) used to become the occupied peak, and the sleeper's real + # load then read as a collapse. + import json + t = _live_tracker() + ts = self._run(t, self.T0, 3600, 0) + ts = self._run(t, ts, 10, 300) + ts = self._run(t, ts, 2 * 3600, 30) + t2 = _live_tracker() + t2.db = t.db + t2.restore(json.loads(json.dumps(t.snapshot())), now=ts + 30) + ts = self._run(t2, ts + 30, 10, 300) + ts = self._run(t2, ts, 2 * 3600, 30) + assert t2._session_start is not None + assert _rows_full(t2) == [] + + def test_unobserved_time_does_not_settle_a_collapse(self): + # A low reading, then nothing for over EXIT_SETTLE_S, then another: + # no evidence the bed stayed empty in between. + t = _live_tracker() + ts = self._run(t, self.T0, 3600, 0) + ts = self._run(t, ts, 3600, 30) + t.process(ts, _cap2(*(v + 5 / 3 for v in EMPTY2))) + ts += main.EXIT_SETTLE_S + 1 + t.process(ts, _cap2(*(v + 5 / 3 for v in EMPTY2))) + ts = self._run(t, ts + 5, 3600, 30) + assert t.baseline.source != "exit-rebase" + assert t._session_start is not None + assert _rows_full(t) == [] + + def test_collapse_edge_outranks_the_restart_fallback(self): + # A collapse confirmed by the first frame after a restart, before any + # present frame: the exit is the collapse edge, not the last presence + # before the restart (which falls inside the collapse). + import json + t = _live_tracker() + ts = self._run(t, self.T0, 3600, 0) + ts = self._run(t, ts, 3600, -7.7) + ts = self._run(t, ts, 3 * 3600, 30) + exit_ts = ts + ts = self._run(t, ts, 300, 0) + state = json.loads(json.dumps(t.snapshot())) + state["baseline"]["collapse_dwell_s"] = main.EXIT_SETTLE_S + t2 = _live_tracker() + t2.db = t.db + t2.restore(state, now=ts + 30) + assert t2._resumed_last_present is not None + self._run(t2, ts + 30, 1800, 0) + rows = _rows_full(t2) + assert t2._session_start is None + assert len(rows) == 1 + assert abs(rows[0][1] - exit_ts) <= main.ABSENCE_TIMEOUT_S + + @pytest.mark.parametrize("field, value", [ + ("occupied_load", float("inf")), + ("occupied_load", float("nan")), + ("occupied_load", -5.0), + ("occupied_load", "heavy"), + ("collapse_since", float("inf")), + ("collapse_dwell_s", float("nan")), + ("collapse_dwell_s", -1.0), + ]) + def test_invalid_saved_collapse_is_dropped_not_the_baseline(self, field, value): + saved = {"format": "capSense2", "means": {"A": 16.6, "B": 15.0, "C": 21.6}, + "threshold": 6.0, "ref_mean": 1.16, "absence_ready": True, + "occupied_load": 30.0, "collapse_since": self.T0, "collapse_dwell_s": 120.0} + saved[field] = value + b = main.AdaptiveBaseline.from_state(saved) + assert b is not None + assert b.means == {"A": 16.6, "B": 15.0, "C": 21.6} and b.absence_ready + assert (b.occupied_load, b.collapse_since, b.collapse_dwell_s) == (None, None, 0.0) + + def test_infinite_saved_load_does_not_end_the_night(self): + # Every reading is under 25% of an infinite load, so a sleeper at + # full load would read as a collapse and close 10 minutes later. + import json + t = _live_tracker() + ts = self._run(t, self.T0, 3600, 0) + ts = self._run(t, ts, 2 * 3600, 30) + state = json.loads(json.dumps(t.snapshot())) + state["baseline"]["occupied_load"] = float("inf") + t2 = _live_tracker() + t2.db = t.db + t2.restore(json.loads(json.dumps(state)), now=ts + 30) + self._run(t2, ts + 30, 3600, 30) + assert t2.baseline.source != "exit-rebase" + assert t2._session_start is not None + assert _rows_full(t2) == [] + + def test_low_gain_load_keeps_the_old_exit_path(self): + # A load at or under 4x the exit threshold leaves no reading that is + # both a collapse and still occupied, so the rule stays off: a dip to + # just above the exit threshold holds the session (documented limit). + t = _live_tracker() + ts = self._run(t, self.T0, 3600, 0) + ts = self._run(t, ts, 2 * 3600, 10) + assert t.baseline.occupied_load <= t.baseline.exit_threshold / main.EXIT_COLLAPSE_FRACTION + ts = self._run(t, ts, 3600, 3.5) + assert t.baseline.collapse_since is None + assert t.baseline.source != "exit-rebase" + assert t._session_start is not None + self._run(t, ts, 1800, 0) + rows = _rows_full(t) + assert t._session_start is None and len(rows) == 1 + + def test_cap_logs_the_collapse_it_discards(self, caplog): + # Disputed in review: a collapse under EXIT_SETTLE_S old when the cap + # fires is unconfirmed, so the cap stands, but it must be visible. + t = _live_tracker() + ts = self._run(t, self.T0, 3600, 0) + ts = self._run(t, ts, 3600, -7.7) + start = ts + ts = self._run(t, ts, main.MAX_SESSION_S - 300, 30, step=10.0) + with caplog.at_level(logging.INFO): + self._run(t, ts, 600, 0, step=10.0) + rows = _rows_full(t) + assert len(rows) == 1 and rows[0][2] == main.MAX_SESSION_S + assert abs(rows[0][0] - start) <= 10 + assert [r for r in caplog.records + if r.levelno == logging.INFO and "cap discards a collapse" in r.getMessage()] + + def test_partial_side_frames_do_not_starve_the_baseline(self): + # Only the firmware sentinel arms the settling skip; records carrying + # one side each must not hide every frame of the other. + t = _live_tracker() + ts = self.T0 + for _ in range(20): + t.process(ts, _cap2(*EMPTY2, side="right")) + t.process(ts + 0.5, _cap2(*EMPTY2)) + ts += 1.0 + assert t.baseline is not None + + def test_merged_exit_waits_for_the_other_side(self): + # Home's load collapses while the sleeper is still on the away side: + # the session ends when they leave that side, not at the collapse. + holder = main.DBHolder(_make_db()) + main._db_write_failures = 0 + mk = lambda side: main.SessionTracker(side=side, db=holder, calibration=_Cal(), + pump_gate=main.PumpGateCapSense(), + _last_movement_write=0.0) + home, away = mk("left"), mk("right") + + def run(start, seconds, left, right): + ts = start + while ts < start + seconds: + lv = [v + left / 3 for v in EMPTY2] + rv = [v + right / 3 for v in EMPTY2] + frame = {"type": "capSense2", + "left": {"values": [x for v in lv for x in (v, v)] + [1.16, 1.16]}, + "right": {"values": [x for v in rv for x in (v, v)] + [1.16, 1.16]}} + main.process_single_sleeper(home, away, ts, frame) + ts += 5.0 + return ts + + ts = run(self.T0, 3600, 0, 0) + ts = run(ts, 3600, -7.7, 0) + ts = run(ts, 4 * 3600, 30, 0) + ts = run(ts, 500, 0, 30) # rolled onto the away side + exit_ts = ts + run(ts, 3600, 0, 0) + rows = holder.conn.execute("SELECT side, left_bed_at FROM sleep_records").fetchall() + assert len(rows) == 1 and rows[0][0] == "left" + assert abs(rows[0][1] - exit_ts) <= main.ABSENCE_TIMEOUT_S + + def test_merged_session_ends_where_the_home_side_emptied(self): + holder = main.DBHolder(_make_db()) + main._db_write_failures = 0 + mk = lambda side: main.SessionTracker(side=side, db=holder, calibration=_Cal(), + pump_gate=main.PumpGateCapSense(), + _last_movement_write=0.0) + home, away = mk("left"), mk("right") + + def run(start, seconds, rise): + ts = start + while ts < start + seconds: + vals = [v + rise / 3 for v in EMPTY2] + frame = {"type": "capSense2", + "left": {"values": [vals[0], vals[0], vals[1], vals[1], vals[2], vals[2], 1.16, 1.16]}, + "right": {"values": [v for p in EMPTY2 for v in (p, p)] + [1.16, 1.16]}} + main.process_single_sleeper(home, away, ts, frame) + ts += 5.0 + return ts + + ts = run(self.T0, 3600, 0) + ts = run(ts, 3600, -7.7) + ts = run(ts, 4 * 3600, 30) + exit_ts = ts + run(ts, 3600, 0) + rows = holder.conn.execute("SELECT side, left_bed_at FROM sleep_records").fetchall() + assert len(rows) == 1 and rows[0][0] == "left" + assert abs(rows[0][1] - exit_ts) <= main.ABSENCE_TIMEOUT_S + + +class TestPod5Movement: + """Pod 88 2026-10-09: movement was exactly 0 for 97% of a night's epochs. + The pump gate zeroed every delta while the (continuously running) pump + spun, and the 3-epoch median erased lone busy minutes.""" + + T0 = 1_791_500_000.0 + LOADED = (31.0, 19.4, 31.6) # right side, occupied + + def test_still_then_roll_with_the_pump_running(self): + import random + rng = random.Random(5) + clock = [self.T0] + gate = main.PumpGateCapSense() + holder = main.DBHolder(_make_db()) + main._db_write_failures = 0 + t = main.SessionTracker(side="left", db=holder, calibration=_Cal(), pump_gate=gate, + _last_movement_write=self.T0) + health = {"type": "frzHealth", "left": {"pump": {"rpm": 1940}}, "right": {"pump": {"rpm": 0}}} + + def feed(seconds, level, jitter=0.01, swing=0.0): + k = 0 + while clock[0] < feed.end + seconds: + ts = clock[0] + if int(ts) % 10 == 0 and ts == int(ts): + gate.update_pump_state(health) + if int(ts - self.T0) % 300 == 0 and ts == int(ts): + t.process(ts, _cap2(-1.0, -1.0, -1.0, ref=-1.0)) # firmware sentinel + clock[0] += 0.5 + continue + shift = swing * (1 if (k // 2) % 2 else -1) + vals = [v + shift + rng.gauss(0, jitter) for v in level] + t.process(ts, _cap2(*vals)) + k += 1 + clock[0] += 0.5 + feed.end = clock[0] + feed.end = self.T0 + + with patch("main.time.monotonic", side_effect=lambda: clock[0]): + feed(600, EMPTY2) # empty + feed(40 * 60, self.LOADED) # in bed, still + roll_at = clock[0] + feed(10, self.LOADED, swing=2.0) # rolls over + feed(15 * 60, self.LOADED) # still again + rows = holder.conn.execute("SELECT timestamp, total_movement FROM movement ORDER BY timestamp").fetchall() + still = [m for ts, m in rows if self.T0 + 600 + 15 * 60 <= ts < roll_at] + roll = [m for ts, m in rows if roll_at <= ts <= roll_at + 60] + assert len(still) >= 20 + assert sum(1 for m in still if m > 0) >= 0.5 * len(still), still + assert max(still) <= 50, still + assert max(roll) > 200, roll + + def test_frame_after_a_sentinel_is_skipped(self): + # The first frame after the 5-minute all -1.0 frame reads 28.5 on + # every channel above 28.5 (live pod 88 capture, 2026-10-10). + t = _live_tracker() + t.process(self.T0, _cap2(*self.LOADED)) + t.process(self.T0 + 0.5, _cap2(-1.0, -1.0, -1.0, ref=-1.0)) + obs = t.observe(self.T0 + 1.0, _cap2(28.5, 19.4, 28.5)) + assert obs.delta == 0.0 and obs.present is None + obs = t.observe(self.T0 + 1.5, _cap2(*self.LOADED)) + assert obs.delta < 0.01 diff --git a/modules/sleep-detector/test_repair_capped_sessions.py b/modules/sleep-detector/test_repair_capped_sessions.py new file mode 100644 index 000000000..3ebb78bd6 --- /dev/null +++ b/modules/sleep-detector/test_repair_capped_sessions.py @@ -0,0 +1,255 @@ +""" +Tests for scripts/repair-capped-sessions.py, the one-off repair of +sleep_records the detector force-closed at MAX_SESSION_S. Each test builds a +synthetic biometrics.db in a temp dir. +""" + +import importlib.util +import json +import sqlite3 +import sys +from pathlib import Path + +import pytest +import session_limits + +SCRIPT = Path(__file__).resolve().parents[2] / "scripts" / "repair-capped-sessions.py" +_spec = importlib.util.spec_from_file_location("repair_capped_sessions", SCRIPT) +repair = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(repair) + +CAP = session_limits.MAX_SESSION_S +T0 = 1_791_500_000 +MIN = 60 + + +def _db(path): + conn = sqlite3.connect(path) + conn.execute( + """CREATE TABLE sleep_records ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + side TEXT, entered_bed_at INTEGER, left_bed_at INTEGER, + sleep_duration_seconds INTEGER, times_exited_bed INTEGER, + present_intervals TEXT, not_present_intervals TEXT, + created_at INTEGER + )""" + ) + conn.execute("CREATE TABLE vitals (side TEXT, timestamp INTEGER, heart_rate REAL)") + return conn + + +def _record(conn, entered=T0, side="left", exits=0, present=None, absent=None): + present = json.dumps([[entered, entered + CAP]]) if present is None else present + absent = json.dumps([]) if absent is None else absent + cur = conn.execute( + """INSERT INTO sleep_records (side, entered_bed_at, left_bed_at, sleep_duration_seconds, + times_exited_bed, present_intervals, not_present_intervals, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)""", + (side, entered, entered + CAP, CAP, exits, present, absent, entered + CAP)) + conn.commit() + return cur.lastrowid + + +def _vitals(conn, start, minutes, side="left"): + """One vitals row a minute for `minutes` minutes from `start`; returns the last ts.""" + rows = [(side, start + i * MIN, 60.0) for i in range(minutes + 1)] + conn.executemany("INSERT INTO vitals VALUES (?, ?, ?)", rows) + conn.commit() + return rows[-1][1] + + +def _one(conn): + items = repair.plan(conn) + assert len(items) == 1 + return items[0] + + +@pytest.fixture +def conn(tmp_path): + c = _db(str(tmp_path / "biometrics.db")) + yield c + c.close() + + +def test_constants_are_the_detectors(): + import main + assert (repair.MAX_SESSION_S, repair.ABSENCE_TIMEOUT_S, repair.MIN_SESSION_S) == ( + main.MAX_SESSION_S, main.ABSENCE_TIMEOUT_S, main.MIN_SESSION_S) + + +def test_repair_ends_at_the_last_long_vitals_run(conn): + _record(conn) + end = _vitals(conn, T0 + 10 * MIN, 7 * 60) + it = _one(conn) + assert it["action"] == "repair" + assert it["new_left"] == end + session_limits.ABSENCE_TIMEOUT_S + assert it["new_duration"] == it["new_left"] - T0 + assert json.loads(it["present"]) == [[T0, it["new_left"]]] + assert it["new_exits"] == 1 + assert it["dropped_runs"] == [] + + +def test_vitals_to_the_cap_are_kept(conn): + _record(conn) + _vitals(conn, T0, CAP // MIN) + assert _one(conn)["action"] == "keep" + + +def test_new_end_is_never_later_than_the_original(conn): + # The last run ends 60 s before the cap: +120 s would pass it. + _record(conn) + _vitals(conn, T0 + CAP - 61 * MIN, 60) + assert _one(conn)["action"] == "keep" + + +def test_no_long_run_is_no_evidence(conn): + _record(conn) + _vitals(conn, T0 + 60 * MIN, 20) + _vitals(conn, T0 + 120 * MIN, 29) + assert _one(conn)["action"] == "no-evidence" + + +def test_no_vitals_is_no_evidence(conn): + _record(conn) + assert _one(conn)["action"] == "no-evidence" + + +def test_gap_over_five_minutes_splits_a_run(conn): + _record(conn) + _vitals(conn, T0, 20) + _vitals(conn, T0 + 20 * MIN + repair.VITALS_GAP_S + 1, 20) + assert _one(conn)["action"] == "no-evidence" + + +def test_occupancy_under_min_session_is_too_short(conn, monkeypatch): + # Unreachable while MIN_RUN_S (30 min) exceeds MIN_SESSION_S; guards a + # shorter MIN_RUN_S. + monkeypatch.setattr(repair, "MIN_RUN_S", 60) + _record(conn) + _vitals(conn, T0, 1) + it = _one(conn) + assert it["action"] == "too-short" + assert it["new_left"] == T0 + MIN + session_limits.ABSENCE_TIMEOUT_S + + +@pytest.mark.parametrize("present, absent", [ + ("null", "[]"), + ("not json", "[]"), + ("[[1, 2, 3]]", "[]"), + ('[["a", 2]]', "[]"), + ("[]", "{}"), +]) +def test_malformed_intervals_are_listed_and_left_alone(conn, tmp_path, present, absent): + rid = _record(conn, present=present, absent=absent) + _vitals(conn, T0 + 10 * MIN, 7 * 60) + items = repair.plan(conn) + assert items[0]["action"] == "malformed" + repair.apply(str(tmp_path / "biometrics.db"), conn, items) + row = conn.execute("SELECT left_bed_at, present_intervals FROM sleep_records WHERE id = ?", + (rid,)).fetchone() + assert row == (T0 + CAP, present) + + +def test_exits_are_recounted_from_kept_intervals(conn): + # Two exits before the real end and one inside the padding: the later + # present interval is clipped away, so its exit goes with it. + present = [[T0, T0 + 3600], [T0 + 3700, T0 + 7200], [T0 + 9 * 3600, T0 + CAP]] + absent = [[T0 + 3600, T0 + 3700], [T0 + 7200, T0 + 9 * 3600]] + _record(conn, exits=2, present=json.dumps(present), absent=json.dumps(absent)) + _vitals(conn, T0, 115) + it = _one(conn) + assert it["action"] == "repair" + kept = json.loads(it["present"]) + assert kept == [[T0, T0 + 3600], [T0 + 3700, it["new_left"]]] + assert it["new_exits"] == 2 + assert json.loads(it["absent"]) == [[T0 + 3600, T0 + 3700]] + + +def test_record_without_intervals_counts_the_final_exit(conn): + _record(conn, exits=0, present="[]", absent="") + _vitals(conn, T0, 6 * 60) + it = _one(conn) + assert it["action"] == "repair" and it["new_exits"] == 1 + + +def test_short_final_run_is_repaired_and_listed(conn, capsys): + # 60 min, a 10-min gap, then 20 min: the tradeoff ends at the first run + # (the 20 min may be a genuine return to bed), so it's listed for review. + _record(conn) + end = _vitals(conn, T0, 60) + later = end + 10 * MIN + _vitals(conn, later, 20) + items = repair.plan(conn) + it = items[0] + assert it["action"] == "repair" + assert it["new_left"] == end + session_limits.ABSENCE_TIMEOUT_S + assert it["dropped_runs"] == [[later, later + 20 * MIN]] + repair.report(items, verbose=False) + out = capsys.readouterr().out + assert "short-final-run: 1" in out + assert "(20 min)" in out + + +def test_apply_writes_a_backup_first_and_is_idempotent(tmp_path, monkeypatch, capsys): + path = tmp_path / "biometrics.db" + c = _db(str(path)) + rid = _record(c) + _vitals(c, T0, 6 * 60) + c.close() + monkeypatch.setattr(repair.time, "time", lambda: 1_791_600_000) + + monkeypatch.setattr(sys, "argv", ["repair", "--db", str(path), "--apply"]) + repair.main() + assert "applied: 1 records updated" in capsys.readouterr().out + backup = tmp_path / "biometrics.db.bak.1791600000" + with sqlite3.connect(backup) as b: + assert b.execute("SELECT left_bed_at FROM sleep_records").fetchone() == (T0 + CAP,) + with sqlite3.connect(path) as c: + repaired = c.execute("SELECT * FROM sleep_records WHERE id = ?", (rid,)).fetchone() + assert repaired[3] < T0 + CAP + + repair.main() + assert "nothing to apply" in capsys.readouterr().out + with sqlite3.connect(path) as c: + assert c.execute("SELECT * FROM sleep_records WHERE id = ?", (rid,)).fetchone() == repaired + assert sorted(p.name for p in tmp_path.glob("*.bak.*")) == ["biometrics.db.bak.1791600000"] + + +def test_dry_run_writes_nothing(tmp_path, monkeypatch, capsys): + path = tmp_path / "biometrics.db" + c = _db(str(path)) + _record(c) + _vitals(c, T0, 6 * 60) + c.close() + monkeypatch.setattr(sys, "argv", ["repair", "--db", str(path)]) + repair.main() + assert "dry run" in capsys.readouterr().out + with sqlite3.connect(path) as c: + assert c.execute("SELECT left_bed_at FROM sleep_records").fetchone() == (T0 + CAP,) + assert list(tmp_path.glob("*.bak.*")) == [] + + +def test_backups_in_the_same_second_do_not_overwrite(tmp_path, monkeypatch): + monkeypatch.setattr(repair.time, "time", lambda: 1_791_600_000) + db = str(tmp_path / "biometrics.db") + first = repair.reserve_backup_path(db) + Path(first).write_text("first run's backup") + second = repair.reserve_backup_path(db) + assert first != second + assert Path(first).read_text() == "first run's backup" + assert second.endswith(".bak.1791600000.1") + + +def test_a_record_changed_underneath_rolls_back_every_update(conn, tmp_path): + a = _record(conn) + b = _record(conn, entered=T0 + CAP + 3600) + _vitals(conn, T0, 6 * 60) + _vitals(conn, T0 + CAP + 3600, 6 * 60) + items = repair.plan(conn) + assert [it["action"] for it in items] == ["repair", "repair"] + conn.execute("UPDATE sleep_records SET left_bed_at = left_bed_at - 1 WHERE id = ?", (b,)) + conn.commit() + with pytest.raises(RuntimeError, match=f"record {b} changed"): + repair.apply(str(tmp_path / "biometrics.db"), conn, items) + assert conn.execute("SELECT sleep_duration_seconds FROM sleep_records WHERE id = ?", + (a,)).fetchone() == (CAP,) diff --git a/next-env.d.ts b/next-env.d.ts index 9edff1c7c..ce4e94a6b 100644 --- a/next-env.d.ts +++ b/next-env.d.ts @@ -1,6 +1,7 @@ /// /// import "./.next/types/routes.d.ts"; +import "./.next/types/root-params.d.ts"; // NOTE: This file should not be edited // see https://nextjs.org/docs/app/api-reference/config/typescript for more information. diff --git a/next.config.mjs b/next.config.mjs index 9259cf4c6..0e209bdcd 100644 --- a/next.config.mjs +++ b/next.config.mjs @@ -21,6 +21,9 @@ const nextConfig = { // Turbopack bakes RELATIVE_ROOT_PATH at build time — without standalone, // the .next bundle only works on the machine that built it. output: 'standalone', + outputFileTracingIncludes: { + '/api/biometrics/watch-upload': ['./scripts/import-watch-health.py', './scripts/sleep-stage-dataset.py'], + }, // Keep native modules external — resolved from node_modules at runtime // so the correct platform binary (linux-arm64 on pod) is used. // hap-nodejs is included because its module-evaluation side effects diff --git a/package.json b/package.json index 5a8a28564..ee79c37d4 100644 --- a/package.json +++ b/package.json @@ -3,8 +3,8 @@ "type": "module", "private": true, "scripts": { - "prebuild": "node scripts/generate-git-info.mjs && lingui compile", - "build": "next build && node scripts/strip-turbopack-external-hashes.mjs && cp -r .next/static .next/standalone/.next/static && { [ ! -d public ] || cp -r public .next/standalone/public || true; }", + "prebuild": "node scripts/generate-git-info.mjs && lingui compile && node scripts/build/demo-biometric-worker.mjs", + "build": "next build && node scripts/strip-turbopack-external-hashes.mjs && cp -r .next/static .next/standalone/.next/static && { [ ! -d public ] || cp -r public .next/standalone/public || true; } && node scripts/build/biometrics-worker.mjs", "dev": "next dev", "db:generate": "drizzle-kit generate", "db:migrate": "drizzle-kit migrate", @@ -27,7 +27,10 @@ "test:mutation": "stryker run", "test:mutation:incremental": "stryker run --incremental", "tsc": "tsc --noEmit", - "prepare": "git rev-parse --git-dir >/dev/null 2>&1 && git config core.hooksPath .githooks || true" + "prepare": "git rev-parse --git-dir >/dev/null 2>&1 && git config core.hooksPath .githooks || true", + "build:biometrics-worker": "node scripts/build/biometrics-worker.mjs", + "biometrics:experiment": "tsx scripts/algorithm-experiments/cli.ts", + "test:browser": "node tests/browser/run.mjs" }, "lint-staged": { "*.{ts,tsx}": "eslint --fix", @@ -43,6 +46,7 @@ "@lingui/react": "^6.0.0", "@modelcontextprotocol/sdk": "^1.31.0", "@nats-io/transport-node": "^3.4.0", + "@noble/hashes": "2.4.0", "@tanstack/react-query": "^5.90.21", "@trpc/client": "^11.10.0", "@trpc/next": "^11.10.0", @@ -86,11 +90,12 @@ "@lingui/babel-plugin-lingui-macro": "^6.0.0", "@lingui/cli": "^6.0.0", "@lingui/loader": "^6.0.0", - "@lingui/swc-plugin": "^5.11.0", + "@lingui/swc-plugin": "^6.0.0", "@lingui/vite-plugin": "^6.0.0", + "@playwright/test": "1.58.2", "@rolldown/plugin-babel": "^0.2.4", "@semantic-release/commit-analyzer": "^13.0.1", - "@semantic-release/git": "^10.0.1", + "@semantic-release/git": "^11.0.0", "@stryker-mutator/core": "^9.6.1", "@stryker-mutator/typescript-checker": "^9.6.1", "@stryker-mutator/vitest-runner": "^9.6.1", @@ -108,12 +113,13 @@ "@types/qrcode": "^1.5.5", "@types/react": "^19.2.14", "@types/react-dom": "^19.2.3", - "@types/three": "0.170.0", + "@types/three": "0.186.0", "@types/ws": "^8.18.1", "@vitejs/plugin-react": "^6.0.0", "@vitest/coverage-v8": "4.1.11", "conventional-changelog-conventionalcommits": "^9.1.0", "drizzle-kit": "^0.31.9", + "esbuild": "0.28.2", "eslint": "^9.39.3", "eslint-config-next": "^16.1.6", "eslint-plugin-react": "^7.37.5", @@ -123,10 +129,10 @@ "lint-staged": "^16.4.0", "semantic-release": "^25.0.3", "tailwindcss": "4.3.3", - "three": "0.170.0", + "three": "0.186.1", "tsx": "^4.21.0", "typescript": "^6.0.0", - "typescript-eslint": "^8.56.1", + "typescript-eslint": "^8.71.0", "vite": "^8.3.1", "vite-tsconfig-paths": "^6.1.1", "vitest": "^4.0.18" @@ -145,7 +151,8 @@ "dbus-next@0.10.2": "patches/dbus-next@0.10.2.patch" }, "overrides": { - "dbus-next>usocket": "-" + "dbus-next>usocket": "-", + "@babel/runtime": "7.29.7" } } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6d3f050d0..f70c531ac 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -6,6 +6,7 @@ settings: overrides: dbus-next>usocket: '-' + '@babel/runtime': 7.29.7 patchedDependencies: dbus-next@0.10.2: @@ -18,28 +19,31 @@ importers: dependencies: '@ajayche/trpc-panel': specifier: ^2.0.4 - version: 2.1.0(@trpc/server@11.19.0(typescript@6.0.3))(@types/react@19.3.0)(next@16.3.8(@babel/core@7.29.7)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0))(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(zod@4.6.5) + version: 2.1.0(@trpc/server@11.19.0(typescript@6.0.3))(@types/react@19.3.0)(next@16.4.0(@babel/core@7.29.7)(@playwright/test@1.58.2)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0))(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(zod@4.6.5) '@base-ui/react': specifier: ^1.2.0 - version: 1.8.0(@types/react@19.3.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0) + version: 1.9.0(@types/react@19.3.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0) '@formatjs/intl-localematcher': specifier: ^0.9.0 version: 0.9.0 '@lingui/core': specifier: ^6.0.0 - version: 6.9.0(@babel/core@7.29.7) + version: 6.9.1(@babel/core@7.29.7) '@lingui/macro': specifier: ^5.9.2 - version: 5.9.5(@lingui/babel-plugin-lingui-macro@6.9.0(@babel/core@7.29.7))(react@19.3.0) + version: 5.9.5(@lingui/babel-plugin-lingui-macro@6.9.1(@babel/core@7.29.7))(react@19.3.0) '@lingui/react': specifier: ^6.0.0 - version: 6.9.0(@babel/core@7.29.7)(react@19.3.0) + version: 6.9.1(@babel/core@7.29.7)(react@19.3.0) '@modelcontextprotocol/sdk': specifier: ^1.31.0 - version: 1.32.0(zod@4.6.5) + version: 1.32.1(zod@4.6.5) '@nats-io/transport-node': specifier: ^3.4.0 version: 3.4.0 + '@noble/hashes': + specifier: 2.4.0 + version: 2.4.0 '@tanstack/react-query': specifier: ^5.90.21 version: 5.104.1(react@19.3.0) @@ -48,7 +52,7 @@ importers: version: 11.19.0(@trpc/server@11.19.0(typescript@6.0.3))(typescript@6.0.3) '@trpc/next': specifier: ^11.10.0 - version: 11.19.0(@tanstack/react-query@5.104.1(react@19.3.0))(@trpc/client@11.19.0(@trpc/server@11.19.0(typescript@6.0.3))(typescript@6.0.3))(@trpc/react-query@11.19.0(@tanstack/react-query@5.104.1(react@19.3.0))(@trpc/client@11.19.0(@trpc/server@11.19.0(typescript@6.0.3))(typescript@6.0.3))(@trpc/server@11.19.0(typescript@6.0.3))(react@19.3.0)(typescript@6.0.3))(@trpc/server@11.19.0(typescript@6.0.3))(next@16.3.8(@babel/core@7.29.7)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0))(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(typescript@6.0.3) + version: 11.19.0(@tanstack/react-query@5.104.1(react@19.3.0))(@trpc/client@11.19.0(@trpc/server@11.19.0(typescript@6.0.3))(typescript@6.0.3))(@trpc/react-query@11.19.0(@tanstack/react-query@5.104.1(react@19.3.0))(@trpc/client@11.19.0(@trpc/server@11.19.0(typescript@6.0.3))(typescript@6.0.3))(@trpc/server@11.19.0(typescript@6.0.3))(react@19.3.0)(typescript@6.0.3))(@trpc/server@11.19.0(typescript@6.0.3))(next@16.4.0(@babel/core@7.29.7)(@playwright/test@1.58.2)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0))(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(typescript@6.0.3) '@trpc/react-query': specifier: ^11.10.0 version: 11.19.0(@tanstack/react-query@5.104.1(react@19.3.0))(@trpc/client@11.19.0(@trpc/server@11.19.0(typescript@6.0.3))(typescript@6.0.3))(@trpc/server@11.19.0(typescript@6.0.3))(react@19.3.0)(typescript@6.0.3) @@ -57,7 +61,7 @@ importers: version: 11.19.0(typescript@6.0.3) '@vercel/analytics': specifier: ^2.0.1 - version: 2.0.1(next@16.3.8(@babel/core@7.29.7)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0))(react@19.3.0) + version: 2.0.1(next@16.4.0(@babel/core@7.29.7)(@playwright/test@1.58.2)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0))(react@19.3.0) better-sqlite3: specifier: ^12.6.2 version: 12.11.1 @@ -87,13 +91,13 @@ importers: version: 17.4.2 drizzle-orm: specifier: ^0.45.1 - version: 0.45.3(@types/better-sqlite3@7.6.13)(better-sqlite3@12.11.1) + version: 0.45.4(@types/better-sqlite3@7.6.13)(better-sqlite3@12.11.1) hap-nodejs: specifier: 1.2.0 version: 1.2.0 lucide-react: specifier: ^1.0.0 - version: 1.50.0(@types/react@19.3.0)(react@19.3.0) + version: 1.54.0(@types/react@19.3.0)(react@19.3.0) mqtt: specifier: ^5.15.1 version: 5.16.0 @@ -102,7 +106,7 @@ importers: version: 1.1.0 next: specifier: ^16.1.6 - version: 16.3.8(@babel/core@7.29.7)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0) + version: 16.4.0(@babel/core@7.29.7)(@playwright/test@1.58.2)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0) node-schedule: specifier: ^2.1.1 version: 2.1.1 @@ -120,7 +124,7 @@ importers: version: 3.10.1(@types/react@19.3.0)(react-dom@19.3.0(react@19.3.0))(react-is@18.3.1)(react@19.3.0)(redux@5.0.1) shadcn: specifier: ^4.0.0 - version: 4.21.1(typescript@6.0.3) + version: 4.21.4 superjson: specifier: ^2.2.6 version: 2.2.6 @@ -145,7 +149,7 @@ importers: devDependencies: '@codecov/nextjs-webpack-plugin': specifier: ^2.0.1 - version: 2.0.1(next@16.3.8(@babel/core@7.29.7)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0))(webpack@5.111.1(esbuild@0.28.2)) + version: 2.0.1(next@16.4.0(@babel/core@7.29.7)(@playwright/test@1.58.2)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0))(webpack@5.111.1(esbuild@0.28.2)) '@eslint/css': specifier: ^2.0.0 version: 2.0.0 @@ -154,31 +158,34 @@ importers: version: 9.39.5 '@eslint/json': specifier: ^2.0.0 - version: 2.1.0 + version: 2.1.1 '@lingui/babel-plugin-lingui-macro': specifier: ^6.0.0 - version: 6.9.0(@babel/core@7.29.7) + version: 6.9.1(@babel/core@7.29.7) '@lingui/cli': specifier: ^6.0.0 - version: 6.9.0(esbuild@0.28.2)(rolldown@1.2.12) + version: 6.9.1(esbuild@0.28.2)(rolldown@1.2.13) '@lingui/loader': specifier: ^6.0.0 - version: 6.9.0(esbuild@0.28.2)(rolldown@1.2.12)(webpack@5.111.1(esbuild@0.28.2)) + version: 6.9.1(esbuild@0.28.2)(rolldown@1.2.13)(webpack@5.111.1(esbuild@0.28.2)) '@lingui/swc-plugin': - specifier: ^5.11.0 - version: 5.11.0(@lingui/core@6.9.0(@babel/core@7.29.7))(next@16.3.8(@babel/core@7.29.7)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0)) + specifier: ^6.0.0 + version: 6.7.1(@lingui/core@6.9.1(@babel/core@7.29.7)) '@lingui/vite-plugin': specifier: ^6.0.0 - version: 6.9.0(@babel/core@7.29.7)(@lingui/babel-plugin-lingui-macro@6.9.0(@babel/core@7.29.7))(@rolldown/plugin-babel@0.2.4(@babel/core@7.29.7)(@babel/runtime@7.29.7)(rolldown@1.2.12)(vite@8.3.2(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)))(esbuild@0.28.2)(rolldown@1.2.12)(vite@8.3.2(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)) + version: 6.9.1(@babel/core@7.29.7)(@lingui/babel-plugin-lingui-macro@6.9.1(@babel/core@7.29.7))(@rolldown/plugin-babel@0.2.4(@babel/core@7.29.7)(@babel/runtime@7.29.7)(rolldown@1.2.13)(vite@8.3.4(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)))(esbuild@0.28.2)(rolldown@1.2.13)(vite@8.3.4(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)) + '@playwright/test': + specifier: 1.58.2 + version: 1.58.2 '@rolldown/plugin-babel': specifier: ^0.2.4 - version: 0.2.4(@babel/core@7.29.7)(@babel/runtime@7.29.7)(rolldown@1.2.12)(vite@8.3.2(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)) + version: 0.2.4(@babel/core@7.29.7)(@babel/runtime@7.29.7)(rolldown@1.2.13)(vite@8.3.4(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)) '@semantic-release/commit-analyzer': specifier: ^13.0.1 version: 13.0.1(semantic-release@25.0.9(typescript@6.0.3)) '@semantic-release/git': - specifier: ^10.0.1 - version: 10.0.1(semantic-release@25.0.9(typescript@6.0.3)) + specifier: ^11.0.0 + version: 11.0.1(semantic-release@25.0.9(typescript@6.0.3)) '@stryker-mutator/core': specifier: ^9.6.1 version: 9.6.1(@types/node@25.9.9) @@ -231,14 +238,14 @@ importers: specifier: ^19.2.3 version: 19.3.0(@types/react@19.3.0) '@types/three': - specifier: 0.170.0 - version: 0.170.0 + specifier: 0.186.0 + version: 0.186.0 '@types/ws': specifier: ^8.18.1 version: 8.18.2 '@vitejs/plugin-react': specifier: ^6.0.0 - version: 6.1.1(@rolldown/plugin-babel@0.2.4(@babel/core@7.29.7)(@babel/runtime@7.29.7)(rolldown@1.2.12)(vite@8.3.2(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)))(vite@8.3.2(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)) + version: 6.1.2(@rolldown/plugin-babel@0.2.4(@babel/core@7.29.7)(@babel/runtime@7.29.7)(rolldown@1.2.13)(vite@8.3.4(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)))(vite@8.3.4(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)) '@vitest/coverage-v8': specifier: 4.1.11 version: 4.1.11(vitest@4.1.11) @@ -248,12 +255,15 @@ importers: drizzle-kit: specifier: ^0.31.9 version: 0.31.11 + esbuild: + specifier: 0.28.2 + version: 0.28.2 eslint: specifier: ^9.39.3 version: 9.39.5(jiti@2.7.0) eslint-config-next: specifier: ^16.1.6 - version: 16.3.8(@typescript-eslint/parser@8.71.0(eslint@9.39.5(jiti@2.7.0))(typescript@6.0.3))(eslint@9.39.5(jiti@2.7.0))(typescript@6.0.3) + version: 16.4.0(@typescript-eslint/parser@8.71.0(eslint@9.39.5(jiti@2.7.0))(typescript@6.0.3))(eslint@9.39.5(jiti@2.7.0))(typescript@6.0.3) eslint-plugin-react: specifier: ^7.37.5 version: 7.37.5(eslint@9.39.5(jiti@2.7.0)) @@ -265,7 +275,7 @@ importers: version: 2.7.0 jsdom: specifier: ^29.0.0 - version: 29.1.1 + version: 29.1.1(@noble/hashes@2.4.0) lint-staged: specifier: ^16.4.0 version: 16.4.0 @@ -276,8 +286,8 @@ importers: specifier: 4.3.3 version: 4.3.3 three: - specifier: 0.170.0 - version: 0.170.0 + specifier: 0.186.1 + version: 0.186.1 tsx: specifier: ^4.21.0 version: 4.23.15 @@ -285,17 +295,17 @@ importers: specifier: ^6.0.0 version: 6.0.3 typescript-eslint: - specifier: ^8.56.1 + specifier: ^8.71.0 version: 8.71.0(eslint@9.39.5(jiti@2.7.0))(typescript@6.0.3) vite: specifier: ^8.3.1 - version: 8.3.2(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1) + version: 8.3.4(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1) vite-tsconfig-paths: specifier: ^6.1.1 - version: 6.1.1(typescript@6.0.3)(vite@8.3.2(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)) + version: 6.1.1(typescript@6.0.3)(vite@8.3.4(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)) vitest: specifier: ^4.0.18 - version: 4.1.11(@types/node@25.9.9)(@vitest/coverage-v8@4.1.11)(jsdom@29.1.1)(vite@8.3.2(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)) + version: 4.1.11(@types/node@25.9.9)(@vitest/coverage-v8@4.1.11)(jsdom@29.1.1(@noble/hashes@2.4.0))(vite@8.3.4(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)) packages: @@ -420,8 +430,8 @@ packages: resolution: {integrity: sha512-N9ZErrD+yW5geCDtBqnOoxmR8+tNKiGuxKlDpuJxfsqpa2dFcexaziGAE/qoHLiDDreVNMupxGmSoNlyvsA3gw==} engines: {node: '>=6.9.0'} - '@babel/helpers@7.29.7': - resolution: {integrity: sha512-1k2lAGRMfHTcwuNYcCNUmaUffmQv8KWMfh2iJUUeRlwlwH4FdNG7mfPI10NPfLHJFThE4Tyr4mv7kTNZOiPuBg==} + '@babel/helpers@7.29.10': + resolution: {integrity: sha512-EMCx3AGZ3WfAl6ht7AyQoPm2Lh+BGYEMdc4Q6Hf7uSCDOZFuLq44qNwL+iaim8VlyYhRAo4X43+weadK8ZB4Yg==} engines: {node: '>=6.9.0'} '@babel/parser@7.29.9': @@ -497,16 +507,16 @@ packages: resolution: {integrity: sha512-puq+Gf35oI24FeN11LkoUQFqv9uwNeWpxXZi/Ji3rRIoKAzKnxRaZ+Gkj0vKS9ZCiTESfng1N9LyOyXvo+m+Gg==} engines: {node: '>=6.9.0'} - '@babel/traverse@7.29.8': - resolution: {integrity: sha512-I5z7H3bf/41ktsNVLtpN0wAa336HkqIHQ5BuPLEhTkt1jVSyZpeNKIzTgEWmlxjdg81R0IgUCcaE+Ok3NvrfZg==} + '@babel/traverse@7.29.10': + resolution: {integrity: sha512-B3lYAs2nkKskbgQ1nfy8Ma9ao9Q1rntTAQHrzzyDuj+TSO+X3le6PfWwqDvodsb+OumE10eMFfuMCTQSQ5iuIQ==} engines: {node: '>=6.9.0'} '@babel/types@7.29.8': resolution: {integrity: sha512-Vj1jF3cPfxg7OAfoI7QnVKLoILlm2JF9pnVHrX8qx7AHMiYWT+NDAA7jChlNgRS4WTLc/fD1lXLmPixluj+3Gg==} engines: {node: '>=6.9.0'} - '@base-ui/react@1.8.0': - resolution: {integrity: sha512-P0/1sxo6SBVZOklKMIedvTWqw2s2IQzi9x5bIVsXu980cuSOD4NeuRSs+/L7LZQfDkZP/uRZyGPyfFl/B1oH+Q==} + '@base-ui/react@1.9.0': + resolution: {integrity: sha512-S8CSYosASBKtpL8SWUfS78rFElfLijiCgA7oHOK22jCJ95FULsYyKwBNFOgudt0RA+/7yNc7F81bOvQnRgbBiQ==} engines: {node: '>=14.0.0'} peerDependencies: '@date-fns/tz': ^1.2.0 @@ -522,8 +532,8 @@ packages: date-fns: optional: true - '@base-ui/utils@0.4.0': - resolution: {integrity: sha512-bO9fz25kKtPf+aZVyfQrC0PDmJdmVni31W2hCS5/Owb+inwdIL3XU26pCPRPlt4LSxZrBgLwubXQXQlKaFEZzw==} + '@base-ui/utils@0.5.0': + resolution: {integrity: sha512-dYGGcEttjkYiMoa5nHm8vLaJ2UcuTWxo3BFAwOYXeRg/q7UVVvA6cacEIXEs3FgBfK1CGtW1ptElqs7wsx+t9A==} peerDependencies: '@types/react': ^17 || ^18 || ^19 react: ^17 || ^18 || ^19 @@ -627,6 +637,9 @@ packages: resolution: {integrity: sha512-OoKoR0f76dCY666JlcbhmVTs2drYj1GUXZTYTcbUgJjh9Nv41aFfZ21bPQTERm5+L5cBDo466NltB2lplS5GBw==} engines: {node: '>=20.19.0'} + '@dimforge/rapier3d-compat@0.12.0': + resolution: {integrity: sha512-uekIGetywIgopfD97oDL5PfeezkFpNhwlzlaEYNOA0N6ghdsOvh/HYjSMek5Q2O1PYvRSDFcqFVJl4r4ZBwOow==} + '@dotenvx/dotenvx@1.75.1': resolution: {integrity: sha512-/BITOC9dmS/edY2zQwZNicQ059O6RKabtQfyEafV0nGtfYRNHYy1DIPiYVcov40+tob9hfmBnbR963dS+EQ1DQ==} hasBin: true @@ -1149,8 +1162,8 @@ packages: resolution: {integrity: sha512-QywQuszQh77pIXCsq998c8hbhSTI/azTty1Z6N53dmAudKHhy573j3yvRLsX2BSp8YpLtoCEG8E9DJe+8zUh4A==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} - '@eslint/json@2.1.0': - resolution: {integrity: sha512-Zcg+SXHmdAlo+0ZFwxOvDQye2zxhaFCu/CPdJTsMDSEYZXkX67AWvdwzhI/YdPYNW68Q1Tr1Rv2SsuUZydnjBg==} + '@eslint/json@2.1.1': + resolution: {integrity: sha512-ffVzPZALaJzxuo46ggBtJ9c8YBWKLMh/R35aFBHk/dRobmFNortJlHJfBWbHGuydoFeVZbvPy0PLX2wvCI9T9w==} engines: {node: ^20.19.0 || ^22.13.0 || >=24} '@eslint/object-schema@2.1.7': @@ -1206,11 +1219,11 @@ packages: resolution: {integrity: sha512-PFOeWyXfH2eoM4o6Zmj9PzCUDlrrbtrbAJVzHYXpXK5rh/dz84Ygu79WHKK3DWW2/oXieYPV6up+i1izSPbc3g==} hasBin: true - '@hono/node-server@2.1.3': - resolution: {integrity: sha512-TA//nWMqPhbfdfneACk6t5a9eqbS9lABEPyKn0/xZTah3H3U2XaVg85rJFl0/Fyit0I552YDHgXGVSf3GwqbUw==} + '@hono/node-server@2.1.4': + resolution: {integrity: sha512-2g1qeHl4BduwDn6L8nkAvDrPUWPMlkQ//a5NhP/zIyaXKMjZNRV+cG9Csu+BiJFf6+68pqzjf4Hi5sJGKHuS1g==} engines: {node: '>=20'} peerDependencies: - hono: ^4 + hono: ^4 || ^5.0.0-0 '@humanfs/core@0.19.2': resolution: {integrity: sha512-UhXNm+CFMWcbChXywFwkmhqjs3PRCmcSa/hfBgLIb7oQ5HNb1wS0icWsGtSAUNgefHeI+eBrA8I1fxmbHsGdvA==} @@ -1562,12 +1575,12 @@ packages: '@leichtgewicht/ip-codec@2.0.5': resolution: {integrity: sha512-Vo+PSpZG2/fmgmiNzYK9qWRh8h/CHrwD0mo1h1DzL4yzHNSfWYujGTYsWGreD000gcgmZ7K4Ys6Tx9TxtsKdDw==} - '@lingui/babel-plugin-extract-messages@6.9.0': - resolution: {integrity: sha512-udKlLtGtJ9TjVjqtogIage/Z3merz5Ko5u72uIaHMZ9aPFjSNDnrsglWEfMLWTeSxmkdOrASDpn8YuBMLbhvPA==} + '@lingui/babel-plugin-extract-messages@6.9.1': + resolution: {integrity: sha512-oRN0J93Oq0+1FXV7nOGSlKTuqJz3TfvadebqG+N4YuN9ZxNgkInW8XH6+2sjDBDucJbjUivgw7J5xvOWsgDm0A==} engines: {node: '>=22.19.0'} - '@lingui/babel-plugin-lingui-macro@6.9.0': - resolution: {integrity: sha512-AwbD2JKdy3L0Vppv5X8xPZJzgY4W9Y+DcH5Tzb7oZA6o96N+VVeJU9xsLFbh5UGIQAwLeY7BZKDH88qmWj1gUA==} + '@lingui/babel-plugin-lingui-macro@6.9.1': + resolution: {integrity: sha512-KhOXhdBoWbyCAb76r49+9QojBkWyMfFqD5PF9Irl0av4x0rGdxknm/bj3PzB8nx1tq2+b3mAgyQCJAMwdvY2Jg==} engines: {node: '>=22.19.0'} peerDependencies: '@babel/core': ^7.20.12 || ^8.0.0 @@ -1575,8 +1588,8 @@ packages: '@babel/core': optional: true - '@lingui/cli@6.9.0': - resolution: {integrity: sha512-/fO982w4GsMsxs5kTf/MnvkpD5/hm3c5uFaPmTWilvf5lnaJbjA3aBaBYliRocYDhvD71xMvLqcsfITB4wC4NA==} + '@lingui/cli@6.9.1': + resolution: {integrity: sha512-X00XHnA980zzUwOsyKlbRkC4QhcgGXQG5LD6IHDYeI27iqtFNxBwi73jzJ9hO62o+qanNVusI+1gIqd4ZKagag==} engines: {node: '>=22.19.0'} hasBin: true peerDependencies: @@ -1592,6 +1605,10 @@ packages: resolution: {integrity: sha512-1fwYejoGLeoy5P1/mop0WP35fDRXkB7fUFxyPV5E+vBbgOXtw+QDRt1y9WtyqjI+dz9zQtNYmND3+tBSeAADfw==} engines: {node: '>=22.19.0'} + '@lingui/conf@6.9.1': + resolution: {integrity: sha512-9kDH5v9AToFOemUzNrhDO9NjDngu0mvTUjdO4r9gSOUO7xydpKh0oL7itHGT2PKx1HIbpDXUJxkIzGZ7FcW2GQ==} + engines: {node: '>=22.19.0'} + '@lingui/core@5.9.5': resolution: {integrity: sha512-Y+iZq9NqnqZOqHNgPomUFP21KH/zs4oTTizWoz0AKAkBbq9T9yb1DSz/ugtBRjF1YLtKMF9tq28v3thMHANSiQ==} engines: {node: '>=20.0.0'} @@ -1604,8 +1621,8 @@ packages: babel-plugin-macros: optional: true - '@lingui/core@6.9.0': - resolution: {integrity: sha512-woQCOsmbG3LxhqTyECcpxn6t3V5tQOgYOG7LL424a/dmpKttyqtJ1/qq3/U7H+ZG+ewFHh56UvQrXSJVnjCf2g==} + '@lingui/core@6.9.1': + resolution: {integrity: sha512-/WsHqJMxx1ONTI9i8iWMJmWodZhV6GjS93JziTmot2x18fF2WlmVQBjDAzGLK3ZcX/2D/6SF7vv72MIJip3Jkg==} engines: {node: '>=22.19.0'} peerDependencies: babel-plugin-macros: 2 || 3 @@ -1613,12 +1630,12 @@ packages: babel-plugin-macros: optional: true - '@lingui/format-po@6.9.0': - resolution: {integrity: sha512-nfeAxAj3An1zq0F8k01fvw+cMgqU4KAIDemylwNnTI+Mamo1x6WUo+vtpaJQjtDDTPBMVKkMoGx9G9fpy5xrtQ==} + '@lingui/format-po@6.9.1': + resolution: {integrity: sha512-dSq4XIiD35jYkrcCBOWm7Sw/hH2QNNDZRExinVabjLLLk8+XCVRinCDu5ZaREv1H3CeJD0ipjc9ro9Z68cqJPg==} engines: {node: '>=22.19.0'} - '@lingui/loader@6.9.0': - resolution: {integrity: sha512-mDW2qVTXq5GBa/2lWSQn7xRnznRiUrOk28SVu7oXaHEplDbl/scWPlEx4NAk9PD5UMYXL4ArR6qXZIcSPfnRpg==} + '@lingui/loader@6.9.1': + resolution: {integrity: sha512-ehDilevDiPAuf13TagSCs4adsDcvpsTgkxCiTwMOP8vrDEL8JbrISoTRcjsZe4H8jW66la0F8OZhLFTS+5HzWg==} engines: {node: '>=22.19.0'} peerDependencies: webpack: ^5.0.0 @@ -1642,70 +1659,70 @@ packages: resolution: {integrity: sha512-t3dNbjb1dWkvcpXGMXIEyBDO3l4B8J2ColZXi0NTG1ioAj+sDfFxFB8fepVgd3JAk+AwARlOLvF14oS0mAdgpw==} engines: {node: '>=20.0.0'} - '@lingui/message-utils@6.9.0': - resolution: {integrity: sha512-LaFvx9Vc/lx4EPbbDaiDtyumDM4aAglRA3yvFQsJkhs1UWrCgDaaUBWhkXKYcmVe9gb7qeGYCqmGWOL84N9xDw==} + '@lingui/message-utils@6.9.1': + resolution: {integrity: sha512-pEfdWUhZoNrE8+06KOsv1r8VTKkdRvaT0gcXVO/FEVfphGiszSMQtoNhv3ftFbet7HKen2xx0LrzJ93mx3zVAw==} engines: {node: '>=22.19.0'} - '@lingui/native-tools-android-arm64@0.1.5': - resolution: {integrity: sha512-2wmD2Fyr/wekxlHZ3+34l/dzyT3WzWtfEZK0CgPyuZpPmlb/MKrNkMHQ2QU1wBPmUcp3qiMtxv/qLLDcXj78Fw==} + '@lingui/native-tools-android-arm64@0.1.6': + resolution: {integrity: sha512-kAwVqJlKv868dGJOhqXXqHwhI3bOlvdHSZz8rozlEEidvBuw6Srr0Es5hwisLlwybzcTYstjOeVGxmvu3QAV4g==} engines: {node: '>=22.19.0'} cpu: [arm64] os: [android] - '@lingui/native-tools-darwin-arm64@0.1.5': - resolution: {integrity: sha512-i/Owg8RUWJ0u27T2268nAfSyL5tExXQgKfon5LxI21rUFQMaZ9cREu39KCLbE0dx40/09mEU3+qWgVgqaMhtFQ==} + '@lingui/native-tools-darwin-arm64@0.1.6': + resolution: {integrity: sha512-0KEoSob+pYS1jyRB+2bJp78L2L2i/VODFIk5Gs8LmKV58xZ+uN8MOhf1dYBBsi6+0tiqPartSCk3WA4i+lKPUQ==} engines: {node: '>=22.19.0'} cpu: [arm64] os: [darwin] - '@lingui/native-tools-darwin-x64@0.1.5': - resolution: {integrity: sha512-EuRwq9KVEHB2jy8RhUIUEd/0ZP4KghVikYVU21cgVxVD1hmO0/Ii/uDUW7atgYF3UGuPwQzOqvhjz3gctG2OKg==} + '@lingui/native-tools-darwin-x64@0.1.6': + resolution: {integrity: sha512-e7liIjkzbJ8w3F9PHzZPmpYpeiv7XO4Np9J1tEpUamKYEdYBNt4AKr/UmWAQ7AjxbZSV9N8lj1ulUl3pdFUuaQ==} engines: {node: '>=22.19.0'} cpu: [x64] os: [darwin] - '@lingui/native-tools-linux-arm64-gnu@0.1.5': - resolution: {integrity: sha512-n5lrZMOnNQnYyg4sGyQ23BiRvJXAIKI8O+jutpiBVNmFflWIgNll/mmvnEoo3699NknEM5lrD8c07CvhnlDqYQ==} + '@lingui/native-tools-linux-arm64-gnu@0.1.6': + resolution: {integrity: sha512-JKu/8CvLsYUiRbRSnuzmrQk24tfuizgnDYlMxE5/awUUrRE+GlpaMP76PhJpkl+tV0v7t6zVe3+I5lbq7LaqVw==} engines: {node: '>=22.19.0'} cpu: [arm64] os: [linux] libc: [glibc] - '@lingui/native-tools-linux-arm64-musl@0.1.5': - resolution: {integrity: sha512-J6d0FX/paKFOuvvE+JJKyX34EYqlsqhbxKxzBMwwG49upB4LI2AkdsnSOXJJssS9umzhzGaLSZdz2cZ3nFACYA==} + '@lingui/native-tools-linux-arm64-musl@0.1.6': + resolution: {integrity: sha512-Aeexf/Ta0nZDZz3cDMAZ95GmFZ2lOxMwf1FK/ShK+U4WcNVSn3aI2KeuOeaLUfot9ZT8AiIIELPV4gO6b/GJgg==} engines: {node: '>=22.19.0'} cpu: [arm64] os: [linux] libc: [musl] - '@lingui/native-tools-linux-x64-gnu@0.1.5': - resolution: {integrity: sha512-r6UV7YxDjl+r4Q6kViOtgqFG7i1uToPsmHokpdV079CroGm7m6SCUNMzbh88N7n0MAeb+VHA207YyMd8X0Jy8Q==} + '@lingui/native-tools-linux-x64-gnu@0.1.6': + resolution: {integrity: sha512-SwX1dLcQE9pVcXwIxSGTE2+zBmaO6NbjD9XPqCiyoO8RFTkUTLjdnRDGHxFlFpIGMlVArQz5mMQ6coyUFl+AiA==} engines: {node: '>=22.19.0'} cpu: [x64] os: [linux] libc: [glibc] - '@lingui/native-tools-linux-x64-musl@0.1.5': - resolution: {integrity: sha512-aADK9gUPz6BC6c0M/z6g48nViHb+BZuoPVLlTdc1ADuexYmcM4kEY7xyRJ/otKpNDtkBtFjYzJ+0Di62aLok9A==} + '@lingui/native-tools-linux-x64-musl@0.1.6': + resolution: {integrity: sha512-aWI8e10I6i7K9BG77LJDHREozx/VT8j8onf0wxd43ACor2S1VwS0ZNlIfpNWQTWovmU/CUliGO/VWQq2jGa+Sg==} engines: {node: '>=22.19.0'} cpu: [x64] os: [linux] libc: [musl] - '@lingui/native-tools-win32-arm64-msvc@0.1.5': - resolution: {integrity: sha512-838QHobbJEwehJQC3JMhSV8I7sypyNdxUOMZaK/p5f/+d1Q4YjvHaBMN0tmU+6pKKUXb3B6yK+PqNischjORgw==} + '@lingui/native-tools-win32-arm64-msvc@0.1.6': + resolution: {integrity: sha512-M2TZP4gKgLaLbiboXIrY5+8GsaQLHvZ8E6cUFgAuSaBa8AMa73+pZO1eay/St9t8yDh6EL6hOlqN0HedUHMwyQ==} engines: {node: '>=22.19.0'} cpu: [arm64] os: [win32] - '@lingui/native-tools-win32-x64-msvc@0.1.5': - resolution: {integrity: sha512-GkOI5ubf+NTvghE9sqWgdWs5ywNTDtXHTumDWHtI+FWAhdsLraEcIqy0ozUY24UansemSWBz81I4EphytN0i0A==} + '@lingui/native-tools-win32-x64-msvc@0.1.6': + resolution: {integrity: sha512-5KvuSTFtDB8UGHS3Rxd+juYZgc51isQSx0hPukDObre6UxZGuUVCBmJ9YwoGVzXedViiNGmsQvrnCTBbVqI75g==} engines: {node: '>=22.19.0'} cpu: [x64] os: [win32] - '@lingui/native-tools@0.1.5': - resolution: {integrity: sha512-K3CBSGN2vjFCqum6LmI/OxoMXDpfVhcUP3U0TNk3G75M4VTY9hSfi8YyknS8ifJ6D1+7R+a7TeoSGM3uA32X3w==} + '@lingui/native-tools@0.1.6': + resolution: {integrity: sha512-xsn2R9YvWL9WuemM0ivSlXy14ORlfvPYLxFP3kbcFU3Pg5T9aRnmYOTfwszwGnjt4QsORO1mHHZT6k/Wr7V9pA==} engines: {node: '>=22.19.0'} peerDependencies: '@lingui/conf': ^6.0.1 @@ -1723,8 +1740,8 @@ packages: babel-plugin-macros: optional: true - '@lingui/react@6.9.0': - resolution: {integrity: sha512-0oSy/GpQUzMrylviy/IiDg6b0DH4HwGusSD8ht2dsDCdunH+wX8bfJ/njo7qtjyaawTjEbdyLCUzlZc2Gd2kVQ==} + '@lingui/react@6.9.1': + resolution: {integrity: sha512-mc0yX+unl9hLzpoYQZ4IsuhCdU0zTlUpZXqM9ElGS/SZTPA2BVV7CmcVpQ8hXm/GomGn2ewQRg2jNDkB+P/R6Q==} engines: {node: '>=22.19.0'} peerDependencies: babel-plugin-macros: 2 || 3 @@ -1733,20 +1750,13 @@ packages: babel-plugin-macros: optional: true - '@lingui/swc-plugin@5.11.0': - resolution: {integrity: sha512-0PBYyozQOiA4WtxKPsnCyrbA9da9zOYehyZjvrv56Kzt9w81xbgnNjuAQZyjIX7UYuqFFE48GvzBqnB5SLLOFA==} + '@lingui/swc-plugin@6.7.1': + resolution: {integrity: sha512-AgfQCypr6VhWCoJTO033j+8LqcPHAYuV5YYoqLQgRZDVB5kjpBqun8WdrRLMkprag0g2lnCUd51KWU7yZywC8w==} peerDependencies: - '@lingui/core': '5' - '@swc/core': '*' - next: '*' - peerDependenciesMeta: - '@swc/core': - optional: true - next: - optional: true + '@lingui/core': 5 || 6 - '@lingui/vite-plugin@6.9.0': - resolution: {integrity: sha512-CrOjXhhxV+qpbSt1EVqO8+LD37K4M/87w5Xajic0m4K8opcS+B2h+4H4a/ZThu3LnqPkL50ZlOPDYLQpuRwY+Q==} + '@lingui/vite-plugin@6.9.1': + resolution: {integrity: sha512-95qeAWrRr8kZv3aZnVesW4azu9KdohV1Kjwx2J+EI/X8KRZ5lM7DpJwAa2oMKyoLJQzY8FGoShYy9kZBR6aNdQ==} engines: {node: '>=22.19.0'} peerDependencies: '@babel/core': ^7.29.0 || ^8.0.0 @@ -1767,8 +1777,8 @@ packages: '@messageformat/date-skeleton@1.1.0': resolution: {integrity: sha512-rmGAfB1tIPER+gh3p/RgA+PVeRE/gxuQ2w4snFWPF5xtb5mbWR7Cbw7wCOftcUypbD6HVoxrVdyyghPm3WzP5A==} - '@messageformat/parser@5.1.1': - resolution: {integrity: sha512-3p0YRGCcTUCYvBKLIxtDDyrJ0YijGIwrTRu1DT8gIviIDZru8H23+FkY6MJBzM1n9n20CiM4VeDYuBsrrwnLjg==} + '@messageformat/parser@5.2.0': + resolution: {integrity: sha512-sc3hNZ0JtC32GhL5nUgPbh2ilcwji4G+Vmj/vhMd8hZaH//NDOvkfwKK1mFnRWG5h2DpksN8xr/HB5/+vMg3iw==} '@microlink/react-json-view@1.31.38': resolution: {integrity: sha512-utEYo7dRhlFpgPovgYv3LkIuiL7H0z72xClfpN5/MvxMJP2WkbdivTGTEf43fy7pxn7D0ED+KFlN89Tbw6QlpA==} @@ -1777,8 +1787,8 @@ packages: react: '>= 15' react-dom: '>= 15' - '@modelcontextprotocol/sdk@1.32.0': - resolution: {integrity: sha512-8BviX/hK4Gd2eL1KdTwm0gfl4d3wdoErW0Jd00h6nomUAtk6IJk5vwQJc6kfWtJTzN5OEV/lgIiJKI8fiDGAlA==} + '@modelcontextprotocol/sdk@1.32.1': + resolution: {integrity: sha512-2DdE+SJDtzLEEWzY1ZjY7Q+VcPhcV1KisD3zI4u0XZyktsjHum1mwbMI+JaulUBi2OZk+KJAi2uPXzxichPkdw==} engines: {node: '>=18'} peerDependencies: '@cfworker/json-schema': ^4.1.1 @@ -1809,64 +1819,68 @@ packages: resolution: {integrity: sha512-hH7u7ejIBTFEJIZ8rIcMrHJI6wl+HhpO5sVFs1+ppmXa8RuB2+Lh1+UwTzZ5xTNNm1TKcRkYy+2qCV56qp8RxA==} engines: {node: '>= 18.0.0'} - '@next/env@16.3.8': - resolution: {integrity: sha512-Al9zqHVV7TJv0eFuOU4U7Lvv74PTih4Ch63sk2xCIpSTkE3udFnaOcnzP2lQVymiL7yS9Cj2iClUXlR3EQ5sEw==} + '@next/env@16.4.0': + resolution: {integrity: sha512-pyLlT5oafGlX8X7xjgGNXz5dNIDSxREm+qD6i9VLSZiZFopA/XWl8NM/KPmLx/8EG/++c49iid3gNWSQW1+2dQ==} - '@next/eslint-plugin-next@16.3.8': - resolution: {integrity: sha512-eCR9RcLTZrVS+K3+VOhi0xMdUUGNuriCHHhU1ZoABC/QQ/Gp0OWbLKQxV2/9DkmihYdYRcQWFmZXW8M1AMUXYw==} + '@next/eslint-plugin-next@16.4.0': + resolution: {integrity: sha512-ffeflB/dL8RrvWejQximw01pOpKHN692Y3FjDB82DCd0YVvXvBD6vY7mcZp27yvSVFVM6wA73xVodEKF4spizA==} - '@next/swc-darwin-arm64@16.3.8': - resolution: {integrity: sha512-2JPRMh2nmQG5CiL7cXGL9AGwnPWJQ//cTtAUCT+w511QHk79SYz3LGv/pc5X643B/WEO0rvu3Yww0hqwt3kgeA==} + '@next/swc-darwin-arm64@16.4.0': + resolution: {integrity: sha512-yyZ+9MgViAKLLb5l1gPBGjRwnq3teNLb5Zx6JEw8LzNSTjlsnp+baUHFxTgl7PpjCmKf48VQfyipG1eBflb48Q==} engines: {node: '>= 10'} cpu: [arm64] os: [darwin] - '@next/swc-darwin-x64@16.3.8': - resolution: {integrity: sha512-GZtCCOBKJ4leVIT/Th0llWKhD1ca92lzbQiS5R5ON9QkoiFnilFsebDae1JU2a3HWoKMEmEZWGs1AGLavVM72Q==} + '@next/swc-darwin-x64@16.4.0': + resolution: {integrity: sha512-RePEEejEj1Auf5s9qEcG0JbcDP3ceNUS/ofNX8AhxzgIjTVlZIHMatlp16YplcSMpsVo/ZMQHJTjbMYJznjGUg==} engines: {node: '>= 10'} cpu: [x64] os: [darwin] - '@next/swc-linux-arm64-gnu@16.3.8': - resolution: {integrity: sha512-O659ygeQYqneJ1fBKMpFxIFqYkYswu8IAS1OCKK/4f3ZgJJm1dRz4fVJZRi/kLLWjnBKnebOePA4WNv+sV1pVA==} + '@next/swc-linux-arm64-gnu@16.4.0': + resolution: {integrity: sha512-HhsyCqd+NpmvDDVY/LqkAN24f+QVvXh3ie+PfcwKinWL2Mu826JymQxJXGjAIfmJ4VlHjp4Pe/nZtXtUoEMAqw==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] libc: [glibc] - '@next/swc-linux-arm64-musl@16.3.8': - resolution: {integrity: sha512-dSjKSyWpzxoO1d3DIZZcP4XJcNaKeLmxQMFOiYl5vuBRMmweIqnAhty8tAmRsvTss779cK1FtYnDMj40e4TQlg==} + '@next/swc-linux-arm64-musl@16.4.0': + resolution: {integrity: sha512-sN9Pf+Zh+RpZAOmbR7d8KAA8T61CVhawtbEJ2Bk6QrRzx6Z/6haaF4UiwBzytM96GRRRWZh9/WALIEO3npMamA==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] libc: [musl] - '@next/swc-linux-x64-gnu@16.3.8': - resolution: {integrity: sha512-lbqOuz3RPRcv+o9msNsJw5x4+Y1ZwPTs6vmL6DCf7i0fZfvng/F59wyeDwqHIvV0mK//RBy/jJkZ+nCKsSMXjQ==} + '@next/swc-linux-x64-gnu@16.4.0': + resolution: {integrity: sha512-D5M5xLWQVd1CvR+L7BickM1weRfmM41x3L1NWcnKY/oyJJqr1qwaCgmu3dMAhLBOMdZwSuDCRV6DJRRpiUkbDQ==} engines: {node: '>= 10'} cpu: [x64] os: [linux] libc: [glibc] - '@next/swc-linux-x64-musl@16.3.8': - resolution: {integrity: sha512-+316WswI8ScVgZeUd+1KGaXkHhaYQzCjvH/05TZSpJ8zBizb1a4G7DtO7F12jcBIqMOtsz9ji1t48fmKtzqsGA==} + '@next/swc-linux-x64-musl@16.4.0': + resolution: {integrity: sha512-8d5WLS6a9YalqpOgXdlbvonFu+sIgjA+CO6tGzetrpPVcUL56foa7B7664h41rAlbLi5fJwK4vWji3zf4QDy+w==} engines: {node: '>= 10'} cpu: [x64] os: [linux] libc: [musl] - '@next/swc-win32-arm64-msvc@16.3.8': - resolution: {integrity: sha512-ji0gd4kMYUxO+1fJBIbiBVRCjzG/lloiyCccnlebvb1ZJ5qXCPZqYg4Jl1DrrixnWNMKylzgpmMWx0yNDYXlzw==} + '@next/swc-win32-arm64-msvc@16.4.0': + resolution: {integrity: sha512-sQuOIiW3snb48UwrBcDMvFA/u03qtw2LeuSIxFIQ9AJuW9UG/4Ddsfo+eV73PnZwBhyncMtsKVh7PrzMol2/fA==} engines: {node: '>= 10'} cpu: [arm64] os: [win32] - '@next/swc-win32-x64-msvc@16.3.8': - resolution: {integrity: sha512-WcTlaKt/TWkh5kUjdJcUmB1XgZ+1c6fz4Y9fDHL73YNSdGaUWjceeWrrlwF0nv19iABYWC4iAq1oX1w4Bn0vfg==} + '@next/swc-win32-x64-msvc@16.4.0': + resolution: {integrity: sha512-6/33lCuxpdHUy27RegyGM/5uTpF7YZd3UR4D0gpkOA/tWY5YR7msbQjqPgCZmSmxrrTLGCXYA4A8vT2H5DPGTQ==} engines: {node: '>= 10'} cpu: [x64] os: [win32] + '@noble/hashes@2.4.0': + resolution: {integrity: sha512-X5XaVWZIBCT7HHZGm5I7ZQXDwLG+bGXuSrMQAW+7Zvl87h1kmc1ZB1VSRJcpUfoUrGQp4Fkoxm5kZ+Ms+aW+eA==} + engines: {node: '>= 20.19.0'} + '@nodelib/fs.scandir@2.1.5': resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==} engines: {node: '>= 8'} @@ -1959,8 +1973,13 @@ packages: '@octokit/types@18.0.0': resolution: {integrity: sha512-l6bAF43PNxkJp6g+W4PjoUSSkxHomXw2nOum5CTftJz1NlV3vu93NImgOYtLf6CbBUb5j+fiuzW0PPQ5JTSvZA==} - '@oxc-project/types@0.152.0': - resolution: {integrity: sha512-oM/5rLBm2tPkg0iBgkH/FOeR3PCDpY19GTgAZjMFM8h9WI9VW7cLgzp6nwtarYKmovavIQZ+Fe/RKX/8C8O/Rw==} + '@oxc-project/types@0.153.0': + resolution: {integrity: sha512-jSNrRIyvPVLL5E1z/tS5zLAprosGz/FuI1PWwEOz59j3E4TTTzSpqKb/RDGqJhhW0QHYISGXxq582wGx6T11Tw==} + + '@playwright/test@1.58.2': + resolution: {integrity: sha512-akea+6bHYBBfA9uQqSYmlJXn61cTa+jbO87xVLCWbTqbWadRVmhxlXATaOjOgcBaWU4ePo0wB41KMFv3o35IXA==} + engines: {node: '>=18'} + hasBin: true '@pnpm/config.env-replace@1.1.0': resolution: {integrity: sha512-htyl8TWnKL7K/ESFa1oW2UB5lVDxuF5DpM7tBi6Hu2LNL3mWkIzNLG6N4zoCUP1lCKNxWy/3iu8mS8MvToGd6w==} @@ -1985,98 +2004,98 @@ packages: react-redux: optional: true - '@rolldown/binding-android-arm-eabi@1.2.12': - resolution: {integrity: sha512-dB/a1214qKfHMXCpgqR4OZT+jS4kTyEXbQGJPqzobt5EwH5rX080pxE37alt3RzvR1bf1Yz/yGqRfrYAxuPw0A==} + '@rolldown/binding-android-arm-eabi@1.2.13': + resolution: {integrity: sha512-Vu/NICrgdRvUOeaJZIyB2FP63s6fO/4ud5l8dXmxDx9yDyn+mN0y+gsvsVviSli+T361Aqo0h46EuMoobrwawA==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm] os: [android] - '@rolldown/binding-android-arm64@1.2.12': - resolution: {integrity: sha512-7KHFgQ5VJxIHcLlrwrc3Xbds7oTNQT7Pgi9gQCJKrd2VGab/UksIOYp6VD8MzCstGxOKMgNamPwUCfxPdP1OHg==} + '@rolldown/binding-android-arm64@1.2.13': + resolution: {integrity: sha512-6LEoIrGy5vvCd2boNnv2soPSUZj/PRvhp8R/t9KlX+mIZxThIE/ZqxrGXxmuNX0FuvQFvLdZr7S4HrYpYBVhEQ==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [android] - '@rolldown/binding-darwin-arm64@1.2.12': - resolution: {integrity: sha512-3YIhqHD96nA5SaYNRBR16HnGv4oavZvXfD/ayHM+oYZ0WD/8lBAtf6zQua4kEyAvpqrluKXl0lnOBoiNby7x9w==} + '@rolldown/binding-darwin-arm64@1.2.13': + resolution: {integrity: sha512-CaJSRzlQ6Q+zkDMVA9OkVxBjVoPRCDEE3v0FSJ9kIQBgRjFApA3Si4ijhbNVUji7ZJt+PfO0e46qy4nPInHUPA==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [darwin] - '@rolldown/binding-darwin-x64@1.2.12': - resolution: {integrity: sha512-UuuJ35MFw4gmFOrE9pEqIV+K3syIKveph+Qc1/ljHZVdoDW4pz/JHR/eMVom+TZGl/5OOvGJOWaOCVt3ZfqhxA==} + '@rolldown/binding-darwin-x64@1.2.13': + resolution: {integrity: sha512-e4r340I5hxSzJkD3YJBxgzq/rJ3wwxwdLLb2FPt1Zpyem20nrQPT11DDEEkazwxaHumH8wpglsp/Ozgj/wT6mw==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [darwin] - '@rolldown/binding-freebsd-x64@1.2.12': - resolution: {integrity: sha512-uMvssit0a4W+/7D8CbHUvG719mH3R2jwXAlh/XcPvuHTE0g++LymF88DCGNX0HM2rBOn0xrzgXktIB6fLSJBTQ==} + '@rolldown/binding-freebsd-x64@1.2.13': + resolution: {integrity: sha512-hoBWDZ4/K/mJKVMbsgUd1jfpA2DiOrJ54mpegObU9TBHBBrtX4FLIXJJQRDPZ/2Dn91GvPob+Z2hwKJzf4E5SQ==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [freebsd] - '@rolldown/binding-linux-arm-gnueabihf@1.2.12': - resolution: {integrity: sha512-XcFu0R0xWnwzSf4IQgFH1rJIckPN1pLy2R+4r9IDB7Yfu/ys9cVqfa4pBrMHj7a3gl8mIR4nRNPg0e5IvEVs6g==} + '@rolldown/binding-linux-arm-gnueabihf@1.2.13': + resolution: {integrity: sha512-oF9fZu3vTWnTiVAkDFxU9PJLt1nh20K9RRpuHsR7ZnE2L7XookBbDBVLojCVSLGbvtSlxeUNnjAj6DbNsKyzbg==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm] os: [linux] - '@rolldown/binding-linux-arm64-gnu@1.2.12': - resolution: {integrity: sha512-260UrKgn8tz39ak+SMDOirKzr7V04M9dWPw5llW00SwBivCZoWcRBKV1d8cXnRkUmSZA3BdiUmBHWk7734Ulpw==} + '@rolldown/binding-linux-arm64-gnu@1.2.13': + resolution: {integrity: sha512-DTpb/+jMiqWMPgphn4ksW1zW+XLJ6dJJ9FMvnIk9I291LhCctMPhMYJrP5t9p6C9qa3KXadDdUEZHBaPh3kNxQ==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [linux] libc: [glibc] - '@rolldown/binding-linux-arm64-musl@1.2.12': - resolution: {integrity: sha512-5YK1I9SqDkbPgc1IA8BgDl34suqUS2q0KWnBrirm0E51YjOs6eo6dV6jbQfNE/argHRSvd0QUGgtpIoYx+WWpw==} + '@rolldown/binding-linux-arm64-musl@1.2.13': + resolution: {integrity: sha512-xU+Kvv9N2A3SY35zFh1FWWNjxbadE/BKeqnkXe7vScU4GZsw757BlVmDLITZEq8DPpk6m7uoDYkK7P+x/uU5Ow==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [linux] libc: [musl] - '@rolldown/binding-linux-ppc64-gnu@1.2.12': - resolution: {integrity: sha512-Rkcrmp7eFRg74yL5fXEU91JEWbdEPLevWwGtXpmhbjlD1StScbWTmO94Bhly+Mo+ketKYkdmM1vNUKeWSlx8cQ==} + '@rolldown/binding-linux-ppc64-gnu@1.2.13': + resolution: {integrity: sha512-XMk/hnhh25oQ756Lclb1Q11h4ll0xC0Esa5w4Vzp+mAE2S3AGJgeM0SonrIgY8fHhjarb6hcoTuwoFG0VVP2ig==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [ppc64] os: [linux] libc: [glibc] - '@rolldown/binding-linux-s390x-gnu@1.2.12': - resolution: {integrity: sha512-qvK4DuAsQc2BSjlx+Xr+IzOIvvxbGZqxFwdWfG6F518Erj0GGISyQbJ6pIappnOxlNPzNHvo/L0BwB30GZ+zVw==} + '@rolldown/binding-linux-s390x-gnu@1.2.13': + resolution: {integrity: sha512-U1BoTHYmlywE2n3HUWjzZ+8EQsKr+OwpxYuDWaubx42tUL8QSL7nOqRz/NVHanibbFP3M1Rxuprz/wNcbMKjqw==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [s390x] os: [linux] libc: [glibc] - '@rolldown/binding-linux-x64-gnu@1.2.12': - resolution: {integrity: sha512-Q9uLBO53Xd4QIq1WOycVQyPP1O4HhraEV2qqb3uTrnVw6QZih9duY4vNXOivL1xoUS1/z+W8eF4NMfl2a8Sdjw==} + '@rolldown/binding-linux-x64-gnu@1.2.13': + resolution: {integrity: sha512-J4cm80Pc6wEwV5AZ42o62sIQgDk+Y7R2s646jIwuvO+4V46QB58N4vVuMPWCBaNgursXLpmUWJV//WQAkPvHBQ==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [linux] libc: [glibc] - '@rolldown/binding-linux-x64-musl@1.2.12': - resolution: {integrity: sha512-3IBxWFMjbOZskDPKv8Lf9BCnahlKuHthWkYnyIxOH/QcJrFcS4EmcenthApkwr/5+nEqZlLzeYbxeMaX7A5u4g==} + '@rolldown/binding-linux-x64-musl@1.2.13': + resolution: {integrity: sha512-W0euZS40U4V6JkmEnVCIX9mcsL6cbXQ4X8c2w+MMOIjFg68TXUW+u3RT+2e1t1pv9mg/EP3pDtK3POXdQ7Rxyw==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [linux] libc: [musl] - '@rolldown/binding-openharmony-arm64@1.2.12': - resolution: {integrity: sha512-xtX61xg4LKPkPWilZU1ynKClz5Gj4bf74LML4r3eVLWumKnGjoEr1OSHQhMdbBDoYTi+yjrujvpZe2pUnqCrrA==} + '@rolldown/binding-openharmony-arm64@1.2.13': + resolution: {integrity: sha512-do/AF/4n9MRshia3a5aXoy8xM2c6mgfT6D7H/dY6my5z+hfCIR4ab7mk4IRj++nsEQpKy39kd0DQnDq+r692tw==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [openharmony] - '@rolldown/binding-win32-arm64-msvc@1.2.12': - resolution: {integrity: sha512-At7fPB6PCaIjzgIhEZFxuT+BBFqiQibJDT4d3PhiR3f4E7bbMZF4aKblbFfEM3sETRDd1YiQx/+U/g/B/ou5Ew==} + '@rolldown/binding-win32-arm64-msvc@1.2.13': + resolution: {integrity: sha512-6FtUf2cJZobutJdxnGYX8z0vtbgqU0b2v8vfBhkXb9wp2MppMFtkunE+IaOntyi01rkIQoPW2GHABtfzsC0BRQ==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [win32] - '@rolldown/binding-win32-x64-msvc@1.2.12': - resolution: {integrity: sha512-WIw2haVKwjuYdXkHaoC0mF8Le71TuCBxjrdKqLbJGctbBABj+ClfmNvtbOnzpq3RokNo5+V1qhtSzJyXorsklQ==} + '@rolldown/binding-win32-x64-msvc@1.2.13': + resolution: {integrity: sha512-UKhFaZTeTcbKrb3iU82m17jVcPEAoqDkCN5MBqgPyZSwp/cQmib21crTS0DdH7T1B7wZRWEI89YQAOJMet5yhg==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [win32] @@ -2087,7 +2106,7 @@ packages: peerDependencies: '@babel/core': ^7.29.0 || ^8.0.0-rc.1 '@babel/plugin-transform-runtime': ^7.29.0 || ^8.0.0-rc.1 - '@babel/runtime': ^7.27.0 || ^8.0.0-rc.1 + '@babel/runtime': 7.29.7 rolldown: ^1.0.0-rc.5 vite: ^8.0.0 peerDependenciesMeta: @@ -2119,19 +2138,15 @@ packages: peerDependencies: semantic-release: '>=20.1.0' - '@semantic-release/error@3.0.0': - resolution: {integrity: sha512-5hiM4Un+tpl4cKw3lV4UgzJj+SmfNIDCLLw0TepzQxz9ZGV5ixnqkzIVF+3tp0ZHgcMKE+VNGHJjEeyFG2dcSw==} - engines: {node: '>=14.17'} - '@semantic-release/error@4.0.0': resolution: {integrity: sha512-mgdxrHTLOjOddRVYIYDo0fR3/v61GNN1YGkfbrjuIKg/uMgCd+Qzo3UAXJ+woLQQpos4pl5Esuw5A7AoNlzjUQ==} engines: {node: '>=18'} - '@semantic-release/git@10.0.1': - resolution: {integrity: sha512-eWrx5KguUcU2wUPaO6sfvZI0wPafUKAMNC18aXY4EnNcrZL86dEmpNVnC9uMpGZkmZJ9EfCVJBQx4pV4EMGT1w==} - engines: {node: '>=14.17'} + '@semantic-release/git@11.0.1': + resolution: {integrity: sha512-Zr8BUYCTZMc8V6wDKN2dpR7nJgewd9I6THL3ydLTnp3OEdTo1/4RBLNYaeRucYMsjMv+BXoCNfXA0NADj1kwhw==} + engines: {node: ^22.22.2 || >=24.15} peerDependencies: - semantic-release: '>=18.0.0' + semantic-release: '>=20.1.0' '@semantic-release/github@12.0.10': resolution: {integrity: sha512-QskygvpX8hFx9UEbqJ+C46OIhaJjsysNQ8PYJPuRDc/ek5+dh8WYGD1ofUahCcGLAOmnnwZTcmCJCEXM8XxhRw==} @@ -2151,8 +2166,8 @@ packages: peerDependencies: semantic-release: '>=20.1.0' - '@shadcn/registry@0.1.0': - resolution: {integrity: sha512-eMF6tPNi9SWZgPvqBvPQFhJN7gV9PcjRdF3kPiiWo5yZJocWqVgxFuspCHrOij3bIJ797NIGZ+j/v7AiXfA6qg==} + '@shadcn/registry@0.1.3': + resolution: {integrity: sha512-82/OmKAVM9pChDHFVAsDP9PcimPGSKZGdbcdwdGGGcSI69ImpN4ZnqsfZ4hkOFEMal92t3y9djYJHMjO4PcVZA==} engines: {node: '>=20.18.1'} '@simple-libs/stream-utils@1.2.0': @@ -2502,8 +2517,8 @@ packages: '@types/stats.js@0.17.4': resolution: {integrity: sha512-jIBvWWShCvlBqBNIZt0KAshWpvSjhkwkEu4ZUcASoAvhmrgAUI2t1dXrjSL4xXVLB4FznPrIsX3nKXFl/Dt4vA==} - '@types/three@0.170.0': - resolution: {integrity: sha512-CUm2uckq+zkCY7ZbFpviRttY+6f9fvwm6YqSqPfA5K22s9w7R4VnA3rzJse8kHVvuzLcTx+CjNCs2NYe0QFAyg==} + '@types/three@0.186.0': + resolution: {integrity: sha512-mxYSBpDC+D0pLfSP6sW4WZTcT+nrtmZcimMqnVmy36Hte3XpeYSrvgg4TRdaM1GemGog1AWzI5qL2VoIfMXbJQ==} '@types/tough-cookie@4.0.5': resolution: {integrity: sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==} @@ -2563,6 +2578,12 @@ packages: peerDependencies: typescript: '>=4.8.4 <6.1.0' + '@typescript-eslint/tsconfig-utils@8.71.1': + resolution: {integrity: sha512-oTvkml5SxXhgg+WWKhod5qWQICDXtOB+/Hi9VdPKFAANJ2b7YOgMffgvzXuqog7R6A/JtrzRf128dsxummyWhg==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + typescript: '>=4.8.4 <6.1.0' + '@typescript-eslint/type-utils@8.71.0': resolution: {integrity: sha512-i8uO1qbdxeKgRnS5sCRt6On3/nfo2d2DwQe3Yvjx543zLy7r8ySqRuPPiIIXAhS03U0v5NfAFx+rUgxFzKKwNw==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} @@ -2574,6 +2595,10 @@ packages: resolution: {integrity: sha512-cJ4OoxPGWvFnBTnSZyaU+qJzGTqPTGJY+gDchj6cRyLRdmIdt4rcsE4twj+zPfrNiWuVi38wijHzShL++Z9atQ==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + '@typescript-eslint/types@8.71.1': + resolution: {integrity: sha512-ybYnPTUwg3VQQOFEiBPLs8Jxl1ry2vXdn1kVKguomBbvxpetdu3MKTNDqFNUrlG8ylP9MpZosmXMFYlGQHk1oQ==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + '@typescript-eslint/typescript-estree@8.71.0': resolution: {integrity: sha512-PEEF4G5sLLWAS5BpPrUvms4ySZkiBQQZM4z+3ReI46axK5Vqr/vXBQatJQIZZOYdGyPUAKTtsrWzpqKuU+3DEw==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} @@ -2743,13 +2768,13 @@ packages: vue-router: optional: true - '@vitejs/plugin-react@6.1.1': - resolution: {integrity: sha512-yxLaQV9gkhS8ezJqCM6+ndU7mDY6gqAg75NQ+0IjwEI8IYOmQCgkRwHKVSfWXW076DsqMo0Dk+0FK1U+M5RgFw==} + '@vitejs/plugin-react@6.1.2': + resolution: {integrity: sha512-fhJNGUdscQa4HkI0xFxwonWyhnVReCPDbXI7FH7jmzpnVCrFQI7PRrhWecO263610vOrs9tr2Y4xcf8uAdFxbw==} engines: {node: ^20.19.0 || >=22.12.0} peerDependencies: '@rolldown/plugin-babel': ^0.1.7 || ^0.2.0 babel-plugin-react-compiler: ^1.0.0 - oxc-transform-react: ^0.145.0 + oxc-transform-react: ^0.152.0 vite: ^8.0.0 peerDependenciesMeta: '@rolldown/plugin-babel': @@ -2842,9 +2867,6 @@ packages: '@webassemblyjs/wast-printer@1.14.1': resolution: {integrity: sha512-kPSSXE6De1XOR820C90RIo2ogvZG+c3KiHzqUoO/F34Y2shGzesfqv7o57xrxovZJH/MetF5UjroJ/R/3isoiw==} - '@webgpu/types@0.1.74': - resolution: {integrity: sha512-lgiI4hbuLcI9unnm2cL/tvCaQU45dp0xcLWJh5uB/9MBGvIA4F8XIk7nSiBeg+K4xWGYwgZKn80LmERI5CxoTA==} - '@xtuc/ieee754@1.2.0': resolution: {integrity: sha512-DX8nKgqcGwsc0eJSqYt5lwP4DH5FlHnmuWWBRy7X0NcaGR0ZtuyeESgMwTYVEtxmsNGY+qit4QYT/MIYTOTPeA==} @@ -2878,10 +2900,6 @@ packages: resolution: {integrity: sha512-TQf59BsZnytt8GdJKLPfUZ54g/iaUL2OWDSFCCvMOhsHduDQxO8xC4PNeyIkVcA5KwL2phPSv0douC0fgWzmnA==} engines: {node: '>= 20'} - aggregate-error@3.1.0: - resolution: {integrity: sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==} - engines: {node: '>=8'} - aggregate-error@5.0.0: resolution: {integrity: sha512-gOsf2YwSlleG6IjRYG2A7k0HmBMEo6qVNk9Bp/EaLgAJT5ngH6PXbqa4ItvnEwCm/velL5jAnQgsHsWnjhGmvw==} engines: {node: '>=18'} @@ -3050,8 +3068,8 @@ packages: base64-js@1.5.1: resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} - baseline-browser-mapping@2.11.27: - resolution: {integrity: sha512-ElY12DaROGuan+lMmZ8Cvo/ZUbXPe7Enc/9VU/b1T3Kp4dwytRcNdR8DoSJN5SNJT/CuvcCA0DHDVmMOCePdRQ==} + baseline-browser-mapping@2.11.28: + resolution: {integrity: sha512-br0LOmjJqr6Rt4AEkadbcNf67Zpo7EPwKsWbjnRrJIa1QYFHQBqM4C7F2WTbV8ahTEcg1Eo+Y0aWjjORInYFyA==} engines: {node: '>=6.0.0'} hasBin: true @@ -3150,8 +3168,8 @@ packages: resolution: {integrity: sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==} engines: {node: '>=10'} - caniuse-lite@1.0.30001814: - resolution: {integrity: sha512-/Uaf1lAzr59XcMpW0o96WoEfr+VXK2OX4U9AgFoiSHsVJ4HppnIFUjtYzsyDH2+tgANaQb2/oxYGwCPapN1FpA==} + caniuse-lite@1.0.30001815: + resolution: {integrity: sha512-uDybUEyfgbyJJ93J1vikTylBX31AkdVYeVPeffMQpf8XN7hSfr0oZOn4VFOIMNvtU5+rHvC5Udg2TgtJKHm23Q==} cbor-extract@2.2.2: resolution: {integrity: sha512-hlSxxI9XO2yQfe9g6msd3g4xCfDqK5T5P0fRMLuaLHhxn4ViPrm+a+MUfhrvH2W962RGxcBwEGzLQyjbDG1gng==} @@ -3212,10 +3230,6 @@ packages: class-variance-authority@0.7.1: resolution: {integrity: sha512-Ka+9Trutv7G8M6WT6SeiRWz792K5qEqIGEGzXKhAE6xOWAY6pPH8U+9IY3oCMv6kqTmLsv7Xh/2w2RigkePMsg==} - clean-stack@2.2.0: - resolution: {integrity: sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==} - engines: {node: '>=6'} - clean-stack@5.3.0: resolution: {integrity: sha512-9ngPTOhYGQqNVSfeJkYXHmF7AGWp4/nN5D/QqNQs3Dvxd1Kk/WpjHfNujKHYUQ/5CoGyOyFNoWSPk5afzP0QVg==} engines: {node: '>=14.16'} @@ -3339,8 +3353,8 @@ packages: config-chain@1.1.13: resolution: {integrity: sha512-qj+f8APARXHrM0hraqXYb2/bOVSV4PvJQlNZ/DVj0QrmNM2q2euizkeuVckQ57J+W0mRH6Hvi+k50M4Jul2VRQ==} - content-disposition@1.1.0: - resolution: {integrity: sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==} + content-disposition@2.0.1: + resolution: {integrity: sha512-e+H0ZXHSWYrENhQzw1LPuP4oF5MzVKmDU6d3hxlvaPEYLLg62MxtQNPRx4SYSuYJSBUgnQIG4HIN2tEtNv7Dog==} engines: {node: '>=18'} content-type@1.0.5: @@ -3655,8 +3669,8 @@ packages: resolution: {integrity: sha512-YCYqxTLIB2OCqf6w9/Vef13baBVbwQIPcbT4Y56AfO6B9ajZhDhD8Jkveg/hJvT/iuMloKD/IYYkyMMNhr7Kqg==} hasBin: true - drizzle-orm@0.45.3: - resolution: {integrity: sha512-CAloER21cDdcgZ1OmjrZX82EeRNsdP+y0onX/eyoYDaYzki/adrAa91lzM5jWh7zZPK4qUbw/6LGv9J+A+uizA==} + drizzle-orm@0.45.4: + resolution: {integrity: sha512-NiM1YZpJ2UxyeE67DpDDZ+Tsw4YLX3trSlkNT7Tm9eKoUNb1LvaAaQzXgYQsWs+AQ9TUmmwMmBOCXuqqxUWKKQ==} peerDependencies: '@aws-sdk/client-rds-data': '>=3' '@cloudflare/workers-types': '>=4' @@ -3763,8 +3777,8 @@ packages: ee-first@1.1.1: resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} - electron-to-chromium@1.5.444: - resolution: {integrity: sha512-5ss/uJfoDYDHT0lfJzT6FbcskIzROIOPf0BbbFkGcvDzoJU7i//9GDrwwIHQVmIsrAGiF3ihpADBRIsrEFt1rQ==} + electron-to-chromium@1.5.452: + resolution: {integrity: sha512-cVvk1YBSj6571VmV6hdJTqj94Y79xSu8cj6Y9t6ae4Y4431z8/7EJFvu3rVp90xhfuv9BpPKUmaLfTo4+FOGBg==} emoji-regex@10.6.0: resolution: {integrity: sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==} @@ -3888,8 +3902,8 @@ packages: resolution: {integrity: sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==} engines: {node: '>=12'} - eslint-config-next@16.3.8: - resolution: {integrity: sha512-81vovwMe6NGnoFsl0KUJWzlS+y239i3fdsxs49gSEX3pDQW2cuKy+fxUPXNkQqqx6OCPheYPtBlSufc9aZ0k+w==} + eslint-config-next@16.4.0: + resolution: {integrity: sha512-2RHLYswzc+hNRktWAjIL6XAvwb1cKcjLawJ5KWywAxn/JivNHtG42ZigL1yobVXDqsimuuL2F9jcEcnHFzbppQ==} peerDependencies: eslint: '>=9.0.0' typescript: '>=3.3.1' @@ -4049,6 +4063,10 @@ packages: resolution: {integrity: sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA==} engines: {node: '>=18.0.0'} + execa@10.1.0: + resolution: {integrity: sha512-5HT/600HqXEtmkb9mK/PkTWw+bQSCzPnWiZ4/87wD3Hi0wFzCapOuNT9p4M08PDd+KLHdHvFhKjOWmNx/npRNQ==} + engines: {node: '>=22'} + execa@5.1.1: resolution: {integrity: sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==} engines: {node: '>=10'} @@ -4069,14 +4087,14 @@ packages: resolution: {integrity: sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==} engines: {node: '>=12.0.0'} - express-rate-limit@8.7.0: - resolution: {integrity: sha512-hOwV7WOxXfjRpAM1DSJWZDXx3GhplwD8IfwuwvogD8i1Qnkgosw/H45s4ZnFAUHDAhPjlY9hLBvJhKmGMyY26g==} + express-rate-limit@8.7.1: + resolution: {integrity: sha512-2FXbqkygil3mZiXzEtM0IjE/UBmBNot0J1GEVrJT4JbrvwOSoEtqtO+wHbiMQELWe1eNvHoYRg2dkjOid+6JkA==} engines: {node: '>= 16'} peerDependencies: express: '>= 4.11' - express@5.2.1: - resolution: {integrity: sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==} + express@5.3.0: + resolution: {integrity: sha512-qCqy1xSoaugxOD4IVHb1ZWdkvCF2W3aE2okKouWj8VwJ+ztskSiXoBnCSJqfKT+cVPJhuJqe9YLbY425+Zh00g==} engines: {node: '>= 18'} extend@3.0.2: @@ -4210,6 +4228,11 @@ packages: resolution: {integrity: sha512-KYAb4c9BJQI6QqGKthV68OHe0badztdXJWKo0WtBA9IuCFPTKvE5ZdUBglP833aMjhaSPNO4A5j/EkzZtGlKjA==} engines: {node: '>=14.14'} + fsevents@2.3.2: + resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + fsevents@2.3.3: resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} @@ -4486,10 +4509,6 @@ packages: resolution: {integrity: sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==} engines: {node: '>=0.8.19'} - indent-string@4.0.0: - resolution: {integrity: sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==} - engines: {node: '>=8'} - indent-string@5.0.0: resolution: {integrity: sha512-m6FAo/spmsW2Ab2fU35JTYwtOKa2yAwXSwgjSv1TJzh4Mh7mC3lzAOVLBprb72XsTrgkEIsl7YrFNAiDiRhIGg==} engines: {node: '>=12'} @@ -4522,8 +4541,8 @@ packages: resolution: {integrity: sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg==} engines: {node: '>=12'} - ip-address@10.7.3: - resolution: {integrity: sha512-A1kdq/tSb5QjvKvAMgIoEvDBIgL7qaqVP/jkvSwYYRZ9iEzvPpopxp2wQfu3SuZRHtpHNxMn8Fs0bS+gf5Xmwg==} + ip-address@10.7.4: + resolution: {integrity: sha512-DWBMKyeM25D70nrIiTkgQth59/G+naKOTep6huF9j+GmeK3DMD60Zvxg5d5qQnz+38AHgou6ean2LECX+lrKDg==} engines: {node: '>= 12'} ipaddr.js@1.9.1: @@ -5108,9 +5127,6 @@ packages: lodash.uniqby@4.7.0: resolution: {integrity: sha512-e/zcLx6CSbmaEgFHCA7BnoQKyCtKMxnuWrJygbwPs/AIn+IMKl66L8/s+wBUn5LRw2pZx3bUHibiV1b6aTWIww==} - lodash@4.18.1: - resolution: {integrity: sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==} - log-symbols@6.0.0: resolution: {integrity: sha512-i24m8rpwhmPIS4zscNzK6MSEhk0DUWa/8iYQWxhffV8jkI4Phvs3F+quL5xvS0gdQR0FyTCMMH33Y78dDTzzIw==} engines: {node: '>=18'} @@ -5149,8 +5165,8 @@ packages: lru-cache@5.1.1: resolution: {integrity: sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==} - lucide-react@1.50.0: - resolution: {integrity: sha512-RqHPQtKX6S9IE1xOL4oTHHsVQbcwJWTnmlxI6AdI9IpOwKiy7q+COPCQyMepI/Qu1x3h7sunpyEMvqtgFbgNiQ==} + lucide-react@1.54.0: + resolution: {integrity: sha512-kVmnVCBoLztTGe3S76xu8utwxijRLOinknNq9e8btZ0R2fm6WCgxY+weaLiA50odo/yi+apm/4B2A/LsFZhTrw==} peerDependencies: '@types/react': '*' react: ^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0 @@ -5254,8 +5270,8 @@ packages: resolution: {integrity: sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==} engines: {node: '>= 8'} - meshoptimizer@0.18.1: - resolution: {integrity: sha512-ZhoIoL7TNV4s5B6+rx5mC//fw8/POGyNxS/DZyCJeiZ12ScLfVwRE/GfsxwiTkMYYD5DmK2/JXnEVXqL4rF+Sw==} + meshoptimizer@1.1.1: + resolution: {integrity: sha512-oRFNWJRDA/WTrVj7NWvqa5HqE1t9MYDj2VaWirQCzCCrAd2GHrqR/sQezCxiWATPNlKTcRaPRHPJwIRoPBAp5g==} micromark-core-commonmark@2.0.4: resolution: {integrity: sha512-wxEeE8v8XVvOrxn1TZj74qYhAtTQsSqufHVS3uNVyT1MupXxhyaHk8/9xDNPh9BmL77QNDgWTkZ3RsYHB2ry7w==} @@ -5515,8 +5531,8 @@ packages: nerf-dart@1.0.0: resolution: {integrity: sha512-EZSPZB70jiVsivaBLYDCyntd5eH8NTSMOn3rB+HxwdmKThGELLdYv8qVIMWvZEFy9w8ZZpW9h9OB32l1rGtj7g==} - next@16.3.8: - resolution: {integrity: sha512-U7QEZaTini6wKrb8A8hqLLqYQyCetegKjCpJOyxk642vWoMoU1x5PyZCJFvgYgiptA8xc5j/9xYlZFO7w9Sjmw==} + next@16.4.0: + resolution: {integrity: sha512-N2C+9q89Z/oLrsepSRS4jRMtwmUZAoZuAHU0e9T6+sAB5DdhADmrnOW+zjYijlrYFmVfmQhGVGy9eMFoB7UgKg==} engines: {node: '>=20.9.0'} hasBin: true peerDependencies: @@ -5558,8 +5574,8 @@ packages: node-persist@0.0.12: resolution: {integrity: sha512-Fbia3FYnURzaql53wLu0t19dmAwQg/tXT6O7YPmdwNwysNKEyFmgoT2BQlPD3XXQnYeiQVNvR5lfvufGwKuxhg==} - node-releases@2.0.57: - resolution: {integrity: sha512-kQK9LGGFiHtrWiNhZtA7Qbw17AQz+dmsEKODRIVTXA9+e5MS/2gZEBhYJt13GrAz5/IOZKddH/0Z3TP/Zgo+yw==} + node-releases@2.0.58: + resolution: {integrity: sha512-BdiKAgbfRlvfMq/XzO/AF9NPmaA8a+YpLAyAFTVFBj3gARrAIfWvNq7Q7NE4se3KgFu0KkrvQMxTS0XVnsGzHg==} engines: {node: '>=18'} node-schedule@2.1.1: @@ -5816,10 +5832,6 @@ packages: resolution: {integrity: sha512-MitaVsCuCFIvOLLPIU7NnfrZvS9H9h7kwMUkDo+T2pEISaJD48IV9S8iIdXB7PsvvdxyYcsSTTrr90XKsbulNw==} engines: {node: '>=18'} - p-reduce@2.1.0: - resolution: {integrity: sha512-2USApvnsutq8uoxZBGbbWM0JIYLiEMJ9RlaN7fAzVNb9OZN0SHjjTTfIcb667XynS5Y1VhwDJVDa72TnPzAYWw==} - engines: {node: '>=8'} - p-reduce@3.0.0: resolution: {integrity: sha512-xsrIUgI0Kn6iyDYm9StOpOeK29XM1aboGji26+QEortiFST1hGZaUQOLhtEbqHErPpGW/aSz6allwK2qcptp0Q==} engines: {node: '>=12'} @@ -5944,6 +5956,16 @@ packages: resolution: {integrity: sha512-nDywThFk1i4BQK4twPQ6TA4RT8bDY96yeuCVBWL3ePARCiEKDRSrNGbFIgUJpLp+XeIR65v8ra7WuJOFUBtkMA==} engines: {node: '>=8'} + playwright-core@1.58.2: + resolution: {integrity: sha512-yZkEtftgwS8CsfYo7nm0KE8jsvm6i/PTgVtB8DL726wNf6H2IMsDuxCpJj59KDaxCtSnrWan2AeDqM7JBaultg==} + engines: {node: '>=18'} + hasBin: true + + playwright@1.58.2: + resolution: {integrity: sha512-vA30H8Nvkq/cPBnNw4Q8TWz1EJyqgpuinBcHET0YVJVFldr8JDNiU9LaWAE1KqSkRYazuaBhTpB5ZzShOezQ6A==} + engines: {node: '>=18'} + hasBin: true + pngjs@5.0.0: resolution: {integrity: sha512-40QW5YalBNfQo5yRYmiw7Yz6TKKVr3h6970B2YE+3fQpsWcrbj1PzJgxeJ19DRQjhMbKPIuMY8rFaXc8moolVw==} engines: {node: '>=10.13.0'} @@ -6262,8 +6284,8 @@ packages: rfdc@1.4.1: resolution: {integrity: sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA==} - rolldown@1.2.12: - resolution: {integrity: sha512-8wafseiaG80xmXSfqidUNqZcylTlhmPZZt+za2m+js2sFZ8dTNlhIOV2WcbIPx2hgwPBJpEUGFAMZ9bgBBLTSQ==} + rolldown@1.2.13: + resolution: {integrity: sha512-/5apOkwWxtACcQtZ5WlrAsKS4JwXVThQPJZ3RqIruRY9UG9M8e0VlXN068xum/JRKA4aIUoJ13ghe/kGCvidiQ==} engines: {node: ^20.19.0 || >=22.12.0} hasBin: true @@ -6366,8 +6388,8 @@ packages: setprototypeof@1.2.0: resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==} - shadcn@4.21.1: - resolution: {integrity: sha512-keQmMdKI1eLKTtYpwFwrFfFegsskOpUgt1l3aF359pbCFt0taq0TMuZ75yK+q/StcV2YvjmBMenrs0diWoezXQ==} + shadcn@4.21.4: + resolution: {integrity: sha512-YYfn/hsHbRH2mGwnKBx9tk+DjTQH+RUMS5iCxjKzwzuJ5uoHPqLOFUCJSeX0sXfrcBSfgQaXiOLGKNLjlIQxxw==} engines: {node: '>=20.18.1'} hasBin: true @@ -6725,8 +6747,8 @@ packages: thenify@3.3.1: resolution: {integrity: sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw==} - three@0.170.0: - resolution: {integrity: sha512-FQK+LEpYc0fBD+J8g6oSEyyNzjp+Q7Ks1C568WWaoMRLW+TkNNWmenWeGgJjV105Gd+p/2ql1ZcjYvNiPZBhuQ==} + three@0.186.1: + resolution: {integrity: sha512-blFeqb49wRCSGUGj7gtpfnSGHy2lwDk94RhUmS1c/hTby70kvChbWpkJ4Pm1390LqzzvTmzgXKHPEafJwCb8jA==} through2@2.0.5: resolution: {integrity: sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==} @@ -7010,8 +7032,8 @@ packages: unrs-resolver@1.12.2: resolution: {integrity: sha512-dmlRxBJJayXjqTwC+JtF1HhJmgf3ftQ3YejFcZrf4+KKtJv0qDsK1pjqaaVjG7wJ5NJ6UVP1OqRMQ71Z4C3rxQ==} - update-browserslist-db@1.3.3: - resolution: {integrity: sha512-pJ2sYawQS0R/WI928Gj5GlPhTGzbMelq0+4INtSYNDV9ErKJcX6xjGWkoG/VnB3dpUm00zALaqkrUD77pO5TDQ==} + update-browserslist-db@1.3.4: + resolution: {integrity: sha512-t8wonIOdQyOjtPd2VktlBu6QjTVTks6v8bSi7kd0r2Vll8TK1JYDZtRRdbmCHOAMphkZgrzK+L+vXK4pyZ8t4A==} hasBin: true peerDependencies: browserslist: '>= 4.21.0' @@ -7090,8 +7112,8 @@ packages: peerDependencies: vite: '*' - vite@8.3.2: - resolution: {integrity: sha512-SQr1x6W5vVSbROg7vsyXIaxK9b0G7zsT68acdWWRmnBUsgDieLCRG+Rep9WdZgcposvv/GSnr4GUUBqB3vXq6w==} + vite@8.3.4: + resolution: {integrity: sha512-IftLE8phJhdNRIGNEV4JCCvfNScoZQVfcXaN8TQropWGJtOHcu+iAB9pHcuGvmEb/kddw0LBe/jS8ztdC6xdiQ==} engines: {node: ^20.19.0 || >=22.12.0} hasBin: true peerDependencies: @@ -7229,6 +7251,11 @@ packages: resolution: {integrity: sha512-K4jVyjnBdgvc86Y6BkaLZEN933SwYOuBFkdmBu9ZfkcAbdVbpITnDmjvZ/aQjRXQrv5EPkTnD1s39GiiqbngCw==} engines: {node: '>= 0.4'} + which-command@0.1.0: + resolution: {integrity: sha512-XZyoF5/5hZtXitIwzrU4NKK+Wtbb9aB9CezUEw2Q0wlYK8NUYQxC1rRXgNueYLtBAJwXIb+/tFVk4dozciNJMA==} + engines: {node: '>=22'} + hasBin: true + which-module@2.0.1: resolution: {integrity: sha512-iBdZ57RDvnOR9AGBhML2vFZf7h8vmBjhoaZqODJBFWHVtKkDmKuHai3cx5PgVMrX5YDNp27AofYbAwctSS+vhQ==} @@ -7434,13 +7461,13 @@ snapshots: '@actions/io@3.0.2': {} - '@ajayche/trpc-panel@2.1.0(@trpc/server@11.19.0(typescript@6.0.3))(@types/react@19.3.0)(next@16.3.8(@babel/core@7.29.7)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0))(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(zod@4.6.5)': + '@ajayche/trpc-panel@2.1.0(@trpc/server@11.19.0(typescript@6.0.3))(@types/react@19.3.0)(next@16.4.0(@babel/core@7.29.7)(@playwright/test@1.58.2)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0))(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(zod@4.6.5)': dependencies: '@microlink/react-json-view': 1.31.38(@types/react@19.3.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0) '@trpc/server': 11.19.0(typescript@6.0.3) clsx: 2.1.1 fuzzysort: 2.0.4 - nuqs: 2.10.1(next@16.3.8(@babel/core@7.29.7)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0))(react@19.3.0) + nuqs: 2.10.1(next@16.4.0(@babel/core@7.29.7)(@playwright/test@1.58.2)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0))(react@19.3.0) path: 0.12.7 pretty-bytes: 6.1.1 pretty-ms: 8.0.0 @@ -7497,10 +7524,10 @@ snapshots: '@babel/generator': 7.29.8 '@babel/helper-compilation-targets': 7.29.7 '@babel/helper-module-transforms': 7.29.7(@babel/core@7.29.7) - '@babel/helpers': 7.29.7 + '@babel/helpers': 7.29.10 '@babel/parser': 7.29.9 '@babel/template': 7.29.7 - '@babel/traverse': 7.29.8 + '@babel/traverse': 7.29.10 '@babel/types': 7.29.8 '@jridgewell/remapping': 2.3.5 convert-source-map: 2.0.0 @@ -7539,7 +7566,7 @@ snapshots: '@babel/helper-optimise-call-expression': 7.29.7 '@babel/helper-replace-supers': 7.29.7(@babel/core@7.29.7) '@babel/helper-skip-transparent-expression-wrappers': 7.29.7 - '@babel/traverse': 7.29.8 + '@babel/traverse': 7.29.10 semver: 6.3.1 transitivePeerDependencies: - supports-color @@ -7548,14 +7575,14 @@ snapshots: '@babel/helper-member-expression-to-functions@7.29.7': dependencies: - '@babel/traverse': 7.29.8 + '@babel/traverse': 7.29.10 '@babel/types': 7.29.8 transitivePeerDependencies: - supports-color '@babel/helper-module-imports@7.29.7': dependencies: - '@babel/traverse': 7.29.8 + '@babel/traverse': 7.29.10 '@babel/types': 7.29.8 transitivePeerDependencies: - supports-color @@ -7565,7 +7592,7 @@ snapshots: '@babel/core': 7.29.7 '@babel/helper-module-imports': 7.29.7 '@babel/helper-validator-identifier': 7.29.7 - '@babel/traverse': 7.29.8 + '@babel/traverse': 7.29.10 transitivePeerDependencies: - supports-color @@ -7580,13 +7607,13 @@ snapshots: '@babel/core': 7.29.7 '@babel/helper-member-expression-to-functions': 7.29.7 '@babel/helper-optimise-call-expression': 7.29.7 - '@babel/traverse': 7.29.8 + '@babel/traverse': 7.29.10 transitivePeerDependencies: - supports-color '@babel/helper-skip-transparent-expression-wrappers@7.29.7': dependencies: - '@babel/traverse': 7.29.8 + '@babel/traverse': 7.29.10 '@babel/types': 7.29.8 transitivePeerDependencies: - supports-color @@ -7597,7 +7624,7 @@ snapshots: '@babel/helper-validator-option@7.29.7': {} - '@babel/helpers@7.29.7': + '@babel/helpers@7.29.10': dependencies: '@babel/template': 7.29.7 '@babel/types': 7.29.8 @@ -7634,7 +7661,7 @@ snapshots: dependencies: '@babel/core': 7.29.7 '@babel/helper-plugin-utils': 7.29.7 - '@babel/traverse': 7.29.8 + '@babel/traverse': 7.29.10 transitivePeerDependencies: - supports-color @@ -7695,7 +7722,7 @@ snapshots: '@babel/parser': 7.29.9 '@babel/types': 7.29.8 - '@babel/traverse@7.29.8': + '@babel/traverse@7.29.10': dependencies: '@babel/code-frame': 7.29.7 '@babel/generator': 7.29.8 @@ -7712,10 +7739,10 @@ snapshots: '@babel/helper-string-parser': 7.29.7 '@babel/helper-validator-identifier': 7.29.7 - '@base-ui/react@1.8.0(@types/react@19.3.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0)': + '@base-ui/react@1.9.0(@types/react@19.3.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0)': dependencies: '@babel/runtime': 7.29.7 - '@base-ui/utils': 0.4.0(@types/react@19.3.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0) + '@base-ui/utils': 0.5.0(@types/react@19.3.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0) '@floating-ui/react-dom': 2.1.9(react-dom@19.3.0(react@19.3.0))(react@19.3.0) '@floating-ui/utils': 0.2.12 react: 19.3.0 @@ -7724,7 +7751,7 @@ snapshots: optionalDependencies: '@types/react': 19.3.0 - '@base-ui/utils@0.4.0(@types/react@19.3.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0)': + '@base-ui/utils@0.5.0(@types/react@19.3.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0)': dependencies: '@babel/runtime': 7.29.7 '@floating-ui/utils': 0.2.12 @@ -7768,11 +7795,11 @@ snapshots: unplugin: 1.16.1 zod: 3.25.76 - '@codecov/nextjs-webpack-plugin@2.0.1(next@16.3.8(@babel/core@7.29.7)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0))(webpack@5.111.1(esbuild@0.28.2))': + '@codecov/nextjs-webpack-plugin@2.0.1(next@16.4.0(@babel/core@7.29.7)(@playwright/test@1.58.2)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0))(webpack@5.111.1(esbuild@0.28.2))': dependencies: '@codecov/bundler-plugin-core': 2.0.1 '@codecov/webpack-plugin': 2.0.1(webpack@5.111.1(esbuild@0.28.2)) - next: 16.3.8(@babel/core@7.29.7)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0) + next: 16.4.0(@babel/core@7.29.7)(@playwright/test@1.58.2)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0) unplugin: 1.16.1 webpack: 5.111.1(esbuild@0.28.2) @@ -7809,6 +7836,8 @@ snapshots: '@csstools/css-tokenizer@4.0.2': {} + '@dimforge/rapier3d-compat@0.12.0': {} + '@dotenvx/dotenvx@1.75.1': dependencies: '@dotenvx/primitives': 0.8.0 @@ -8144,7 +8173,7 @@ snapshots: '@eslint/js@9.39.5': {} - '@eslint/json@2.1.0': + '@eslint/json@2.1.1': dependencies: '@eslint/core': 1.2.1 '@eslint/plugin-kit': 0.7.3 @@ -8163,7 +8192,9 @@ snapshots: '@eslint/core': 1.2.1 levn: 0.4.1 - '@exodus/bytes@1.16.0': {} + '@exodus/bytes@1.16.0(@noble/hashes@2.4.0)': + optionalDependencies: + '@noble/hashes': 2.4.0 '@floating-ui/core@1.8.0': dependencies: @@ -8208,7 +8239,7 @@ snapshots: safe-buffer: 5.2.1 xml2js: 0.6.2 - '@hono/node-server@2.1.3(hono@4.13.13)': + '@hono/node-server@2.1.4(hono@4.13.13)': dependencies: hono: 4.13.13 @@ -8495,31 +8526,31 @@ snapshots: '@leichtgewicht/ip-codec@2.0.5': {} - '@lingui/babel-plugin-extract-messages@6.9.0': + '@lingui/babel-plugin-extract-messages@6.9.1': dependencies: - '@lingui/conf': 6.9.0 + '@lingui/conf': 6.9.1 - '@lingui/babel-plugin-lingui-macro@6.9.0(@babel/core@7.29.7)': + '@lingui/babel-plugin-lingui-macro@6.9.1(@babel/core@7.29.7)': dependencies: '@babel/types': 7.29.8 - '@lingui/conf': 6.9.0 - '@lingui/message-utils': 6.9.0 + '@lingui/conf': 6.9.1 + '@lingui/message-utils': 6.9.1 optionalDependencies: '@babel/core': 7.29.7 - '@lingui/cli@6.9.0(esbuild@0.28.2)(rolldown@1.2.12)': + '@lingui/cli@6.9.1(esbuild@0.28.2)(rolldown@1.2.13)': dependencies: '@babel/core': 7.29.7 '@babel/generator': 7.29.8 '@babel/parser': 7.29.9 '@babel/types': 7.29.8 - '@lingui/babel-plugin-extract-messages': 6.9.0 - '@lingui/babel-plugin-lingui-macro': 6.9.0(@babel/core@7.29.7) - '@lingui/conf': 6.9.0 - '@lingui/core': 6.9.0(@babel/core@7.29.7) - '@lingui/format-po': 6.9.0 - '@lingui/message-utils': 6.9.0 - '@lingui/native-tools': 0.1.5(@lingui/conf@6.9.0) + '@lingui/babel-plugin-extract-messages': 6.9.1 + '@lingui/babel-plugin-lingui-macro': 6.9.1(@babel/core@7.29.7) + '@lingui/conf': 6.9.1 + '@lingui/core': 6.9.1(@babel/core@7.29.7) + '@lingui/format-po': 6.9.1 + '@lingui/message-utils': 6.9.1 + '@lingui/native-tools': 0.1.6(@lingui/conf@6.9.1) chokidar: 5.0.0 cli-table3: 0.6.5 commander: 14.0.3 @@ -8533,7 +8564,7 @@ snapshots: tinypool: 2.2.0 optionalDependencies: esbuild: 0.28.2 - rolldown: 1.2.12 + rolldown: 1.2.13 transitivePeerDependencies: - babel-plugin-macros - supports-color @@ -8545,30 +8576,37 @@ snapshots: lilconfig: 3.1.3 normalize-path: 3.0.0 - '@lingui/core@5.9.5(@lingui/babel-plugin-lingui-macro@6.9.0(@babel/core@7.29.7))': + '@lingui/conf@6.9.1': + dependencies: + jest-validate: 29.7.0 + jiti: 2.7.0 + lilconfig: 3.1.3 + normalize-path: 3.0.0 + + '@lingui/core@5.9.5(@lingui/babel-plugin-lingui-macro@6.9.1(@babel/core@7.29.7))': dependencies: '@babel/runtime': 7.29.7 '@lingui/message-utils': 5.9.5 optionalDependencies: - '@lingui/babel-plugin-lingui-macro': 6.9.0(@babel/core@7.29.7) + '@lingui/babel-plugin-lingui-macro': 6.9.1(@babel/core@7.29.7) - '@lingui/core@6.9.0(@babel/core@7.29.7)': + '@lingui/core@6.9.1(@babel/core@7.29.7)': dependencies: - '@lingui/babel-plugin-lingui-macro': 6.9.0(@babel/core@7.29.7) - '@lingui/message-utils': 6.9.0 + '@lingui/babel-plugin-lingui-macro': 6.9.1(@babel/core@7.29.7) + '@lingui/message-utils': 6.9.1 transitivePeerDependencies: - '@babel/core' - '@lingui/format-po@6.9.0': + '@lingui/format-po@6.9.1': dependencies: - '@lingui/conf': 6.9.0 - '@lingui/message-utils': 6.9.0 + '@lingui/conf': 6.9.1 + '@lingui/message-utils': 6.9.1 pofile-ts: 4.0.3 - '@lingui/loader@6.9.0(esbuild@0.28.2)(rolldown@1.2.12)(webpack@5.111.1(esbuild@0.28.2))': + '@lingui/loader@6.9.1(esbuild@0.28.2)(rolldown@1.2.13)(webpack@5.111.1(esbuild@0.28.2))': dependencies: - '@lingui/cli': 6.9.0(esbuild@0.28.2)(rolldown@1.2.12) - '@lingui/conf': 6.9.0 + '@lingui/cli': 6.9.1(esbuild@0.28.2)(rolldown@1.2.13) + '@lingui/conf': 6.9.1 optionalDependencies: webpack: 5.111.1(esbuild@0.28.2) transitivePeerDependencies: @@ -8577,102 +8615,101 @@ snapshots: - rolldown - supports-color - '@lingui/macro@5.9.5(@lingui/babel-plugin-lingui-macro@6.9.0(@babel/core@7.29.7))(react@19.3.0)': + '@lingui/macro@5.9.5(@lingui/babel-plugin-lingui-macro@6.9.1(@babel/core@7.29.7))(react@19.3.0)': dependencies: - '@lingui/core': 5.9.5(@lingui/babel-plugin-lingui-macro@6.9.0(@babel/core@7.29.7)) - '@lingui/react': 5.9.5(@lingui/babel-plugin-lingui-macro@6.9.0(@babel/core@7.29.7))(react@19.3.0) + '@lingui/core': 5.9.5(@lingui/babel-plugin-lingui-macro@6.9.1(@babel/core@7.29.7)) + '@lingui/react': 5.9.5(@lingui/babel-plugin-lingui-macro@6.9.1(@babel/core@7.29.7))(react@19.3.0) optionalDependencies: - '@lingui/babel-plugin-lingui-macro': 6.9.0(@babel/core@7.29.7) + '@lingui/babel-plugin-lingui-macro': 6.9.1(@babel/core@7.29.7) transitivePeerDependencies: - react '@lingui/message-utils@5.9.5': dependencies: - '@messageformat/parser': 5.1.1 + '@messageformat/parser': 5.2.0 js-sha256: 0.10.1 - '@lingui/message-utils@6.9.0': + '@lingui/message-utils@6.9.1': dependencies: '@messageformat/date-skeleton': 1.1.0 - '@messageformat/parser': 5.1.1 + '@messageformat/parser': 5.2.0 js-sha256: 0.10.1 - '@lingui/native-tools-android-arm64@0.1.5': + '@lingui/native-tools-android-arm64@0.1.6': optional: true - '@lingui/native-tools-darwin-arm64@0.1.5': + '@lingui/native-tools-darwin-arm64@0.1.6': optional: true - '@lingui/native-tools-darwin-x64@0.1.5': + '@lingui/native-tools-darwin-x64@0.1.6': optional: true - '@lingui/native-tools-linux-arm64-gnu@0.1.5': + '@lingui/native-tools-linux-arm64-gnu@0.1.6': optional: true - '@lingui/native-tools-linux-arm64-musl@0.1.5': + '@lingui/native-tools-linux-arm64-musl@0.1.6': optional: true - '@lingui/native-tools-linux-x64-gnu@0.1.5': + '@lingui/native-tools-linux-x64-gnu@0.1.6': optional: true - '@lingui/native-tools-linux-x64-musl@0.1.5': + '@lingui/native-tools-linux-x64-musl@0.1.6': optional: true - '@lingui/native-tools-win32-arm64-msvc@0.1.5': + '@lingui/native-tools-win32-arm64-msvc@0.1.6': optional: true - '@lingui/native-tools-win32-x64-msvc@0.1.5': + '@lingui/native-tools-win32-x64-msvc@0.1.6': optional: true - '@lingui/native-tools@0.1.5(@lingui/conf@6.9.0)': + '@lingui/native-tools@0.1.6(@lingui/conf@6.9.1)': dependencies: - '@lingui/conf': 6.9.0 + '@lingui/conf': 6.9.1 '@swc/types': 0.1.28 optionalDependencies: - '@lingui/native-tools-android-arm64': 0.1.5 - '@lingui/native-tools-darwin-arm64': 0.1.5 - '@lingui/native-tools-darwin-x64': 0.1.5 - '@lingui/native-tools-linux-arm64-gnu': 0.1.5 - '@lingui/native-tools-linux-arm64-musl': 0.1.5 - '@lingui/native-tools-linux-x64-gnu': 0.1.5 - '@lingui/native-tools-linux-x64-musl': 0.1.5 - '@lingui/native-tools-win32-arm64-msvc': 0.1.5 - '@lingui/native-tools-win32-x64-msvc': 0.1.5 - - '@lingui/react@5.9.5(@lingui/babel-plugin-lingui-macro@6.9.0(@babel/core@7.29.7))(react@19.3.0)': + '@lingui/native-tools-android-arm64': 0.1.6 + '@lingui/native-tools-darwin-arm64': 0.1.6 + '@lingui/native-tools-darwin-x64': 0.1.6 + '@lingui/native-tools-linux-arm64-gnu': 0.1.6 + '@lingui/native-tools-linux-arm64-musl': 0.1.6 + '@lingui/native-tools-linux-x64-gnu': 0.1.6 + '@lingui/native-tools-linux-x64-musl': 0.1.6 + '@lingui/native-tools-win32-arm64-msvc': 0.1.6 + '@lingui/native-tools-win32-x64-msvc': 0.1.6 + + '@lingui/react@5.9.5(@lingui/babel-plugin-lingui-macro@6.9.1(@babel/core@7.29.7))(react@19.3.0)': dependencies: '@babel/runtime': 7.29.7 - '@lingui/core': 5.9.5(@lingui/babel-plugin-lingui-macro@6.9.0(@babel/core@7.29.7)) + '@lingui/core': 5.9.5(@lingui/babel-plugin-lingui-macro@6.9.1(@babel/core@7.29.7)) react: 19.3.0 optionalDependencies: - '@lingui/babel-plugin-lingui-macro': 6.9.0(@babel/core@7.29.7) + '@lingui/babel-plugin-lingui-macro': 6.9.1(@babel/core@7.29.7) - '@lingui/react@6.9.0(@babel/core@7.29.7)(react@19.3.0)': + '@lingui/react@6.9.1(@babel/core@7.29.7)(react@19.3.0)': dependencies: - '@lingui/babel-plugin-lingui-macro': 6.9.0(@babel/core@7.29.7) - '@lingui/core': 6.9.0(@babel/core@7.29.7) + '@lingui/babel-plugin-lingui-macro': 6.9.1(@babel/core@7.29.7) + '@lingui/core': 6.9.1(@babel/core@7.29.7) react: 19.3.0 use-sync-external-store: 1.7.0(react@19.3.0) transitivePeerDependencies: - '@babel/core' - '@lingui/swc-plugin@5.11.0(@lingui/core@6.9.0(@babel/core@7.29.7))(next@16.3.8(@babel/core@7.29.7)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0))': + '@lingui/swc-plugin@6.7.1(@lingui/core@6.9.1(@babel/core@7.29.7))': dependencies: - '@lingui/core': 6.9.0(@babel/core@7.29.7) - optionalDependencies: - next: 16.3.8(@babel/core@7.29.7)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0) + '@lingui/conf': 6.9.0 + '@lingui/core': 6.9.1(@babel/core@7.29.7) - '@lingui/vite-plugin@6.9.0(@babel/core@7.29.7)(@lingui/babel-plugin-lingui-macro@6.9.0(@babel/core@7.29.7))(@rolldown/plugin-babel@0.2.4(@babel/core@7.29.7)(@babel/runtime@7.29.7)(rolldown@1.2.12)(vite@8.3.2(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)))(esbuild@0.28.2)(rolldown@1.2.12)(vite@8.3.2(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1))': + '@lingui/vite-plugin@6.9.1(@babel/core@7.29.7)(@lingui/babel-plugin-lingui-macro@6.9.1(@babel/core@7.29.7))(@rolldown/plugin-babel@0.2.4(@babel/core@7.29.7)(@babel/runtime@7.29.7)(rolldown@1.2.13)(vite@8.3.4(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)))(esbuild@0.28.2)(rolldown@1.2.13)(vite@8.3.4(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1))': dependencies: - '@lingui/cli': 6.9.0(esbuild@0.28.2)(rolldown@1.2.12) - '@lingui/conf': 6.9.0 - '@lingui/native-tools': 0.1.5(@lingui/conf@6.9.0) - vite: 8.3.2(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1) + '@lingui/cli': 6.9.1(esbuild@0.28.2)(rolldown@1.2.13) + '@lingui/conf': 6.9.1 + '@lingui/native-tools': 0.1.6(@lingui/conf@6.9.1) + vite: 8.3.4(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1) optionalDependencies: '@babel/core': 7.29.7 - '@lingui/babel-plugin-lingui-macro': 6.9.0(@babel/core@7.29.7) - '@rolldown/plugin-babel': 0.2.4(@babel/core@7.29.7)(@babel/runtime@7.29.7)(rolldown@1.2.12)(vite@8.3.2(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)) - rolldown: 1.2.12 + '@lingui/babel-plugin-lingui-macro': 6.9.1(@babel/core@7.29.7) + '@rolldown/plugin-babel': 0.2.4(@babel/core@7.29.7)(@babel/runtime@7.29.7)(rolldown@1.2.13)(vite@8.3.4(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)) + rolldown: 1.2.13 transitivePeerDependencies: - babel-plugin-macros - esbuild @@ -8680,7 +8717,7 @@ snapshots: '@messageformat/date-skeleton@1.1.0': {} - '@messageformat/parser@5.1.1': + '@messageformat/parser@5.2.0': dependencies: moo: 0.5.3 @@ -8694,9 +8731,9 @@ snapshots: transitivePeerDependencies: - '@types/react' - '@modelcontextprotocol/sdk@1.32.0(zod@3.25.76)': + '@modelcontextprotocol/sdk@1.32.1(zod@3.25.76)': dependencies: - '@hono/node-server': 2.1.3(hono@4.13.13) + '@hono/node-server': 2.1.4(hono@4.13.13) ajv: 8.20.0 ajv-formats: 3.0.1(ajv@8.20.0) content-type: 1.0.5 @@ -8704,8 +8741,8 @@ snapshots: cross-spawn: 7.0.6 eventsource: 3.0.7 eventsource-parser: 3.1.1 - express: 5.2.1 - express-rate-limit: 8.7.0(express@5.2.1) + express: 5.3.0 + express-rate-limit: 8.7.1(express@5.3.0) hono: 4.13.13 jose: 6.2.12 json-schema-typed: 8.0.2 @@ -8716,9 +8753,9 @@ snapshots: transitivePeerDependencies: - supports-color - '@modelcontextprotocol/sdk@1.32.0(zod@4.6.5)': + '@modelcontextprotocol/sdk@1.32.1(zod@4.6.5)': dependencies: - '@hono/node-server': 2.1.3(hono@4.13.13) + '@hono/node-server': 2.1.4(hono@4.13.13) ajv: 8.20.0 ajv-formats: 3.0.1(ajv@8.20.0) content-type: 1.0.5 @@ -8726,8 +8763,8 @@ snapshots: cross-spawn: 7.0.6 eventsource: 3.0.7 eventsource-parser: 3.1.1 - express: 5.2.1 - express-rate-limit: 8.7.0(express@5.2.1) + express: 5.3.0 + express-rate-limit: 8.7.1(express@5.3.0) hono: 4.13.13 jose: 6.2.12 json-schema-typed: 8.0.2 @@ -8762,39 +8799,41 @@ snapshots: '@nats-io/nkeys': 2.0.3 '@nats-io/nuid': 3.0.0 - '@next/env@16.3.8': {} + '@next/env@16.4.0': {} - '@next/eslint-plugin-next@16.3.8(eslint@9.39.5(jiti@2.7.0))': + '@next/eslint-plugin-next@16.4.0(eslint@9.39.5(jiti@2.7.0))': dependencies: '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.5(jiti@2.7.0)) fast-glob: 3.3.1 transitivePeerDependencies: - eslint - '@next/swc-darwin-arm64@16.3.8': + '@next/swc-darwin-arm64@16.4.0': optional: true - '@next/swc-darwin-x64@16.3.8': + '@next/swc-darwin-x64@16.4.0': optional: true - '@next/swc-linux-arm64-gnu@16.3.8': + '@next/swc-linux-arm64-gnu@16.4.0': optional: true - '@next/swc-linux-arm64-musl@16.3.8': + '@next/swc-linux-arm64-musl@16.4.0': optional: true - '@next/swc-linux-x64-gnu@16.3.8': + '@next/swc-linux-x64-gnu@16.4.0': optional: true - '@next/swc-linux-x64-musl@16.3.8': + '@next/swc-linux-x64-musl@16.4.0': optional: true - '@next/swc-win32-arm64-msvc@16.3.8': + '@next/swc-win32-arm64-msvc@16.4.0': optional: true - '@next/swc-win32-x64-msvc@16.3.8': + '@next/swc-win32-x64-msvc@16.4.0': optional: true + '@noble/hashes@2.4.0': {} + '@nodelib/fs.scandir@2.1.5': dependencies: '@nodelib/fs.stat': 2.0.5 @@ -8893,7 +8932,11 @@ snapshots: dependencies: '@octokit/openapi-types': 29.0.1 - '@oxc-project/types@0.152.0': {} + '@oxc-project/types@0.153.0': {} + + '@playwright/test@1.58.2': + dependencies: + playwright: 1.58.2 '@pnpm/config.env-replace@1.1.0': {} @@ -8919,59 +8962,59 @@ snapshots: react: 19.3.0 react-redux: 9.3.0(@types/react@19.3.0)(react@19.3.0)(redux@5.0.1) - '@rolldown/binding-android-arm-eabi@1.2.12': + '@rolldown/binding-android-arm-eabi@1.2.13': optional: true - '@rolldown/binding-android-arm64@1.2.12': + '@rolldown/binding-android-arm64@1.2.13': optional: true - '@rolldown/binding-darwin-arm64@1.2.12': + '@rolldown/binding-darwin-arm64@1.2.13': optional: true - '@rolldown/binding-darwin-x64@1.2.12': + '@rolldown/binding-darwin-x64@1.2.13': optional: true - '@rolldown/binding-freebsd-x64@1.2.12': + '@rolldown/binding-freebsd-x64@1.2.13': optional: true - '@rolldown/binding-linux-arm-gnueabihf@1.2.12': + '@rolldown/binding-linux-arm-gnueabihf@1.2.13': optional: true - '@rolldown/binding-linux-arm64-gnu@1.2.12': + '@rolldown/binding-linux-arm64-gnu@1.2.13': optional: true - '@rolldown/binding-linux-arm64-musl@1.2.12': + '@rolldown/binding-linux-arm64-musl@1.2.13': optional: true - '@rolldown/binding-linux-ppc64-gnu@1.2.12': + '@rolldown/binding-linux-ppc64-gnu@1.2.13': optional: true - '@rolldown/binding-linux-s390x-gnu@1.2.12': + '@rolldown/binding-linux-s390x-gnu@1.2.13': optional: true - '@rolldown/binding-linux-x64-gnu@1.2.12': + '@rolldown/binding-linux-x64-gnu@1.2.13': optional: true - '@rolldown/binding-linux-x64-musl@1.2.12': + '@rolldown/binding-linux-x64-musl@1.2.13': optional: true - '@rolldown/binding-openharmony-arm64@1.2.12': + '@rolldown/binding-openharmony-arm64@1.2.13': optional: true - '@rolldown/binding-win32-arm64-msvc@1.2.12': + '@rolldown/binding-win32-arm64-msvc@1.2.13': optional: true - '@rolldown/binding-win32-x64-msvc@1.2.12': + '@rolldown/binding-win32-x64-msvc@1.2.13': optional: true - '@rolldown/plugin-babel@0.2.4(@babel/core@7.29.7)(@babel/runtime@7.29.7)(rolldown@1.2.12)(vite@8.3.2(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1))': + '@rolldown/plugin-babel@0.2.4(@babel/core@7.29.7)(@babel/runtime@7.29.7)(rolldown@1.2.13)(vite@8.3.4(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1))': dependencies: '@babel/core': 7.29.7 picomatch: 4.0.7 - rolldown: 1.2.12 + rolldown: 1.2.13 optionalDependencies: '@babel/runtime': 7.29.7 - vite: 8.3.2(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1) + vite: 8.3.4(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1) '@rolldown/pluginutils@1.0.1': {} @@ -8996,20 +9039,18 @@ snapshots: transitivePeerDependencies: - supports-color - '@semantic-release/error@3.0.0': {} - '@semantic-release/error@4.0.0': {} - '@semantic-release/git@10.0.1(semantic-release@25.0.9(typescript@6.0.3))': + '@semantic-release/git@11.0.1(semantic-release@25.0.9(typescript@6.0.3))': dependencies: - '@semantic-release/error': 3.0.0 - aggregate-error: 3.1.0 + '@semantic-release/error': 4.0.0 + aggregate-error: 5.0.0 debug: 4.4.3 dir-glob: 3.0.1 - execa: 5.1.1 - lodash: 4.18.1 + execa: 10.1.0 + lodash-es: 4.18.1 micromatch: 4.0.8 - p-reduce: 2.1.0 + p-reduce: 3.0.0 semantic-release: 25.0.9(typescript@6.0.3) transitivePeerDependencies: - supports-color @@ -9071,7 +9112,7 @@ snapshots: transitivePeerDependencies: - supports-color - '@shadcn/registry@0.1.0(typescript@6.0.3)': + '@shadcn/registry@0.1.3': dependencies: '@babel/core': 7.29.7 '@babel/parser': 7.29.9 @@ -9079,7 +9120,7 @@ snapshots: '@babel/preset-typescript': 7.29.7(@babel/core@7.29.7) '@dotenvx/dotenvx': 1.75.1 cn: 0.2.6 - cosmiconfig: 9.0.2(typescript@6.0.3) + code-block-writer: 13.0.3 deepmerge: 4.3.1 execa: 9.6.1 fast-glob: 3.3.3 @@ -9092,14 +9133,12 @@ snapshots: recast: 0.23.21 socks: 2.8.10 stringify-object: 5.0.0 - ts-morph: 26.0.0 tsconfig-paths: 4.2.0 type-fest: 4.41.0 undici: 7.30.0 zod: 3.25.76 transitivePeerDependencies: - supports-color - - typescript '@simple-libs/stream-utils@1.2.0': {} @@ -9186,7 +9225,7 @@ snapshots: '@stryker-mutator/util': 9.6.1 semver: 7.8.5 tslib: 2.8.1 - vitest: 4.1.11(@types/node@25.9.9)(@vitest/coverage-v8@4.1.11)(jsdom@29.1.1)(vite@8.3.2(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)) + vitest: 4.1.11(@types/node@25.9.9)(@vitest/coverage-v8@4.1.11)(jsdom@29.1.1(@noble/hashes@2.4.0))(vite@8.3.4(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)) '@stylistic/eslint-plugin@5.10.0(eslint@9.39.5(jiti@2.7.0))': dependencies: @@ -9310,11 +9349,11 @@ snapshots: '@trpc/server': 11.19.0(typescript@6.0.3) typescript: 6.0.3 - '@trpc/next@11.19.0(@tanstack/react-query@5.104.1(react@19.3.0))(@trpc/client@11.19.0(@trpc/server@11.19.0(typescript@6.0.3))(typescript@6.0.3))(@trpc/react-query@11.19.0(@tanstack/react-query@5.104.1(react@19.3.0))(@trpc/client@11.19.0(@trpc/server@11.19.0(typescript@6.0.3))(typescript@6.0.3))(@trpc/server@11.19.0(typescript@6.0.3))(react@19.3.0)(typescript@6.0.3))(@trpc/server@11.19.0(typescript@6.0.3))(next@16.3.8(@babel/core@7.29.7)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0))(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(typescript@6.0.3)': + '@trpc/next@11.19.0(@tanstack/react-query@5.104.1(react@19.3.0))(@trpc/client@11.19.0(@trpc/server@11.19.0(typescript@6.0.3))(typescript@6.0.3))(@trpc/react-query@11.19.0(@tanstack/react-query@5.104.1(react@19.3.0))(@trpc/client@11.19.0(@trpc/server@11.19.0(typescript@6.0.3))(typescript@6.0.3))(@trpc/server@11.19.0(typescript@6.0.3))(react@19.3.0)(typescript@6.0.3))(@trpc/server@11.19.0(typescript@6.0.3))(next@16.4.0(@babel/core@7.29.7)(@playwright/test@1.58.2)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0))(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(typescript@6.0.3)': dependencies: '@trpc/client': 11.19.0(@trpc/server@11.19.0(typescript@6.0.3))(typescript@6.0.3) '@trpc/server': 11.19.0(typescript@6.0.3) - next: 16.3.8(@babel/core@7.29.7)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0) + next: 16.4.0(@babel/core@7.29.7)(@playwright/test@1.58.2)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0) react: 19.3.0 react-dom: 19.3.0(react@19.3.0) typescript: 6.0.3 @@ -9463,14 +9502,14 @@ snapshots: '@types/stats.js@0.17.4': {} - '@types/three@0.170.0': + '@types/three@0.186.0': dependencies: + '@dimforge/rapier3d-compat': 0.12.0 '@tweenjs/tween.js': 23.1.3 '@types/stats.js': 0.17.4 '@types/webxr': 0.5.24 - '@webgpu/types': 0.1.74 fflate: 0.8.3 - meshoptimizer: 0.18.1 + meshoptimizer: 1.1.1 '@types/tough-cookie@4.0.5': {} @@ -9524,8 +9563,8 @@ snapshots: '@typescript-eslint/project-service@8.71.0(typescript@6.0.3)': dependencies: - '@typescript-eslint/tsconfig-utils': 8.71.0(typescript@6.0.3) - '@typescript-eslint/types': 8.71.0 + '@typescript-eslint/tsconfig-utils': 8.71.1(typescript@6.0.3) + '@typescript-eslint/types': 8.71.1 debug: 4.4.3 typescript: 6.0.3 transitivePeerDependencies: @@ -9540,6 +9579,10 @@ snapshots: dependencies: typescript: 6.0.3 + '@typescript-eslint/tsconfig-utils@8.71.1(typescript@6.0.3)': + dependencies: + typescript: 6.0.3 + '@typescript-eslint/type-utils@8.71.0(eslint@9.39.5(jiti@2.7.0))(typescript@6.0.3)': dependencies: '@typescript-eslint/types': 8.71.0 @@ -9554,6 +9597,8 @@ snapshots: '@typescript-eslint/types@8.71.0': {} + '@typescript-eslint/types@8.71.1': {} + '@typescript-eslint/typescript-estree@8.71.0(typescript@6.0.3)': dependencies: '@typescript-eslint/project-service': 8.71.0(typescript@6.0.3) @@ -9657,17 +9702,17 @@ snapshots: '@unrs/resolver-binding-win32-x64-msvc@1.12.2': optional: true - '@vercel/analytics@2.0.1(next@16.3.8(@babel/core@7.29.7)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0))(react@19.3.0)': + '@vercel/analytics@2.0.1(next@16.4.0(@babel/core@7.29.7)(@playwright/test@1.58.2)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0))(react@19.3.0)': optionalDependencies: - next: 16.3.8(@babel/core@7.29.7)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0) + next: 16.4.0(@babel/core@7.29.7)(@playwright/test@1.58.2)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0) react: 19.3.0 - '@vitejs/plugin-react@6.1.1(@rolldown/plugin-babel@0.2.4(@babel/core@7.29.7)(@babel/runtime@7.29.7)(rolldown@1.2.12)(vite@8.3.2(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)))(vite@8.3.2(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1))': + '@vitejs/plugin-react@6.1.2(@rolldown/plugin-babel@0.2.4(@babel/core@7.29.7)(@babel/runtime@7.29.7)(rolldown@1.2.13)(vite@8.3.4(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)))(vite@8.3.4(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1))': dependencies: '@rolldown/pluginutils': 1.0.1 - vite: 8.3.2(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1) + vite: 8.3.4(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1) optionalDependencies: - '@rolldown/plugin-babel': 0.2.4(@babel/core@7.29.7)(@babel/runtime@7.29.7)(rolldown@1.2.12)(vite@8.3.2(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)) + '@rolldown/plugin-babel': 0.2.4(@babel/core@7.29.7)(@babel/runtime@7.29.7)(rolldown@1.2.13)(vite@8.3.4(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)) '@vitest/coverage-v8@4.1.11(vitest@4.1.11)': dependencies: @@ -9681,7 +9726,7 @@ snapshots: obug: 2.2.1 std-env: 4.3.0 tinyrainbow: 3.1.1 - vitest: 4.1.11(@types/node@25.9.9)(@vitest/coverage-v8@4.1.11)(jsdom@29.1.1)(vite@8.3.2(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)) + vitest: 4.1.11(@types/node@25.9.9)(@vitest/coverage-v8@4.1.11)(jsdom@29.1.1(@noble/hashes@2.4.0))(vite@8.3.4(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)) '@vitest/expect@4.1.11': dependencies: @@ -9692,13 +9737,13 @@ snapshots: chai: 6.2.2 tinyrainbow: 3.1.1 - '@vitest/mocker@4.1.11(vite@8.3.2(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1))': + '@vitest/mocker@4.1.11(vite@8.3.4(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1))': dependencies: '@vitest/spy': 4.1.11 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 8.3.2(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1) + vite: 8.3.4(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1) '@vitest/pretty-format@4.1.11': dependencies: @@ -9800,8 +9845,6 @@ snapshots: '@webassemblyjs/ast': 1.14.1 '@xtuc/long': 4.2.2 - '@webgpu/types@0.1.74': {} - '@xtuc/ieee754@1.2.0': {} '@xtuc/long@4.2.2': {} @@ -9825,11 +9868,6 @@ snapshots: agent-base@9.0.0: {} - aggregate-error@3.1.0: - dependencies: - clean-stack: 2.2.0 - indent-string: 4.0.0 - aggregate-error@5.0.0: dependencies: clean-stack: 5.3.0 @@ -10008,7 +10046,7 @@ snapshots: base64-js@1.5.1: {} - baseline-browser-mapping@2.11.27: {} + baseline-browser-mapping@2.11.28: {} before-after-hook@4.0.0: {} @@ -10091,11 +10129,11 @@ snapshots: browserslist@4.29.3: dependencies: - baseline-browser-mapping: 2.11.27 - caniuse-lite: 1.0.30001814 - electron-to-chromium: 1.5.444 - node-releases: 2.0.57 - update-browserslist-db: 1.3.3(browserslist@4.29.3) + baseline-browser-mapping: 2.11.28 + caniuse-lite: 1.0.30001815 + electron-to-chromium: 1.5.452 + node-releases: 2.0.58 + update-browserslist-db: 1.3.4(browserslist@4.29.3) buffer-from@1.1.2: {} @@ -10138,7 +10176,7 @@ snapshots: camelcase@6.3.0: {} - caniuse-lite@1.0.30001814: {} + caniuse-lite@1.0.30001815: {} cbor-extract@2.2.2: dependencies: @@ -10197,8 +10235,6 @@ snapshots: dependencies: clsx: 2.1.1 - clean-stack@2.2.0: {} - clean-stack@5.3.0: dependencies: escape-string-regexp: 5.0.0 @@ -10335,7 +10371,7 @@ snapshots: ini: 1.3.8 proto-list: 1.2.4 - content-disposition@1.1.0: {} + content-disposition@2.0.1: {} content-type@1.0.5: {} @@ -10461,10 +10497,10 @@ snapshots: damerau-levenshtein@1.0.8: {} - data-urls@7.0.0: + data-urls@7.0.0(@noble/hashes@2.4.0): dependencies: whatwg-mimetype: 5.0.0 - whatwg-url: 16.0.1 + whatwg-url: 16.0.1(@noble/hashes@2.4.0) transitivePeerDependencies: - '@noble/hashes' @@ -10609,7 +10645,7 @@ snapshots: esbuild: 0.25.12 tsx: 4.23.15 - drizzle-orm@0.45.3(@types/better-sqlite3@7.6.13)(better-sqlite3@12.11.1): + drizzle-orm@0.45.4(@types/better-sqlite3@7.6.13)(better-sqlite3@12.11.1): optionalDependencies: '@types/better-sqlite3': 7.6.13 better-sqlite3: 12.11.1 @@ -10628,7 +10664,7 @@ snapshots: ee-first@1.1.1: {} - electron-to-chromium@1.5.444: {} + electron-to-chromium@1.5.452: {} emoji-regex@10.6.0: {} @@ -10877,9 +10913,9 @@ snapshots: escape-string-regexp@5.0.0: {} - eslint-config-next@16.3.8(@typescript-eslint/parser@8.71.0(eslint@9.39.5(jiti@2.7.0))(typescript@6.0.3))(eslint@9.39.5(jiti@2.7.0))(typescript@6.0.3): + eslint-config-next@16.4.0(@typescript-eslint/parser@8.71.0(eslint@9.39.5(jiti@2.7.0))(typescript@6.0.3))(eslint@9.39.5(jiti@2.7.0))(typescript@6.0.3): dependencies: - '@next/eslint-plugin-next': 16.3.8(eslint@9.39.5(jiti@2.7.0)) + '@next/eslint-plugin-next': 16.4.0(eslint@9.39.5(jiti@2.7.0)) eslint: 9.39.5(jiti@2.7.0) eslint-import-resolver-node: 0.3.10 eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0)(eslint@9.39.5(jiti@2.7.0)) @@ -11124,6 +11160,21 @@ snapshots: dependencies: eventsource-parser: 3.1.1 + execa@10.1.0: + dependencies: + '@sindresorhus/merge-streams': 4.0.0 + figures: 6.1.0 + get-stream: 9.0.1 + human-signals: 8.0.1 + is-plain-obj: 4.1.0 + is-stream: 4.0.1 + npm-run-path: 6.0.0 + pretty-ms: 9.3.1 + signal-exit: 4.1.0 + strip-final-newline: 4.0.0 + which-command: 0.1.0 + yoctocolors: 2.2.0 + execa@5.1.1: dependencies: cross-spawn: 7.0.6 @@ -11167,20 +11218,20 @@ snapshots: expect-type@1.4.0: {} - express-rate-limit@8.7.0(express@5.2.1): + express-rate-limit@8.7.1(express@5.3.0): dependencies: debug: 4.4.3 - express: 5.2.1 - ip-address: 10.7.3 + express: 5.3.0 + ip-address: 10.7.4 transitivePeerDependencies: - supports-color - express@5.2.1: + express@5.3.0: dependencies: accepts: 2.0.0 body-parser: 2.3.0 - content-disposition: 1.1.0 - content-type: 1.0.5 + content-disposition: 2.0.1 + content-type: 2.1.0 cookie: 0.7.2 cookie-signature: 1.2.2 debug: 4.4.3 @@ -11340,6 +11391,9 @@ snapshots: jsonfile: 6.2.1 universalify: 2.0.1 + fsevents@2.3.2: + optional: true + fsevents@2.3.3: optional: true @@ -11562,9 +11616,9 @@ snapshots: dependencies: lru-cache: 11.5.3 - html-encoding-sniffer@6.0.0: + html-encoding-sniffer@6.0.0(@noble/hashes@2.4.0): dependencies: - '@exodus/bytes': 1.16.0 + '@exodus/bytes': 1.16.0(@noble/hashes@2.4.0) transitivePeerDependencies: - '@noble/hashes' @@ -11636,8 +11690,6 @@ snapshots: imurmurhash@0.1.4: {} - indent-string@4.0.0: {} - indent-string@5.0.0: {} index-to-position@1.2.0: {} @@ -11660,7 +11712,7 @@ snapshots: internmap@2.0.3: {} - ip-address@10.7.3: {} + ip-address@10.7.4: {} ipaddr.js@1.9.1: {} @@ -11920,17 +11972,17 @@ snapshots: jsbi@2.0.5: {} - jsdom@29.1.1: + jsdom@29.1.1(@noble/hashes@2.4.0): dependencies: '@asamuzakjp/css-color': 5.1.11 '@asamuzakjp/dom-selector': 7.1.1 '@bramus/specificity': 2.4.2 '@csstools/css-syntax-patches-for-csstree': 1.1.14(css-tree@3.2.1) - '@exodus/bytes': 1.16.0 + '@exodus/bytes': 1.16.0(@noble/hashes@2.4.0) css-tree: 3.2.1 - data-urls: 7.0.0 + data-urls: 7.0.0(@noble/hashes@2.4.0) decimal.js: 10.6.0 - html-encoding-sniffer: 6.0.0 + html-encoding-sniffer: 6.0.0(@noble/hashes@2.4.0) is-potential-custom-element-name: 1.0.1 lru-cache: 11.5.3 parse5: 8.0.1 @@ -11941,7 +11993,7 @@ snapshots: w3c-xmlserializer: 5.0.0 webidl-conversions: 8.0.1 whatwg-mimetype: 5.0.0 - whatwg-url: 16.0.1 + whatwg-url: 16.0.1(@noble/hashes@2.4.0) xml-name-validator: 5.0.0 transitivePeerDependencies: - '@noble/hashes' @@ -12169,8 +12221,6 @@ snapshots: lodash.uniqby@4.7.0: {} - lodash@4.18.1: {} - log-symbols@6.0.0: dependencies: chalk: 5.6.2 @@ -12209,7 +12259,7 @@ snapshots: dependencies: yallist: 3.1.1 - lucide-react@1.50.0(@types/react@19.3.0)(react@19.3.0): + lucide-react@1.54.0(@types/react@19.3.0)(react@19.3.0): dependencies: react: 19.3.0 optionalDependencies: @@ -12365,7 +12415,7 @@ snapshots: merge2@1.4.1: {} - meshoptimizer@0.18.1: {} + meshoptimizer@1.1.1: {} micromark-core-commonmark@2.0.4: dependencies: @@ -12642,25 +12692,26 @@ snapshots: nerf-dart@1.0.0: {} - next@16.3.8(@babel/core@7.29.7)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0): + next@16.4.0(@babel/core@7.29.7)(@playwright/test@1.58.2)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0): dependencies: - '@next/env': 16.3.8 + '@next/env': 16.4.0 '@swc/helpers': 0.5.23 - baseline-browser-mapping: 2.11.27 - caniuse-lite: 1.0.30001814 + baseline-browser-mapping: 2.11.28 + caniuse-lite: 1.0.30001815 postcss: 8.5.23 react: 19.3.0 react-dom: 19.3.0(react@19.3.0) styled-jsx: 5.1.6(@babel/core@7.29.7)(react@19.3.0) optionalDependencies: - '@next/swc-darwin-arm64': 16.3.8 - '@next/swc-darwin-x64': 16.3.8 - '@next/swc-linux-arm64-gnu': 16.3.8 - '@next/swc-linux-arm64-musl': 16.3.8 - '@next/swc-linux-x64-gnu': 16.3.8 - '@next/swc-linux-x64-musl': 16.3.8 - '@next/swc-win32-arm64-msvc': 16.3.8 - '@next/swc-win32-x64-msvc': 16.3.8 + '@next/swc-darwin-arm64': 16.4.0 + '@next/swc-darwin-x64': 16.4.0 + '@next/swc-linux-arm64-gnu': 16.4.0 + '@next/swc-linux-arm64-musl': 16.4.0 + '@next/swc-linux-x64-gnu': 16.4.0 + '@next/swc-linux-x64-musl': 16.4.0 + '@next/swc-win32-arm64-msvc': 16.4.0 + '@next/swc-win32-x64-msvc': 16.4.0 + '@playwright/test': 1.58.2 sharp: 0.35.5(@types/node@25.9.9) transitivePeerDependencies: - '@babel/core' @@ -12697,7 +12748,7 @@ snapshots: mkdirp: 0.5.6 q: 1.1.2 - node-releases@2.0.57: {} + node-releases@2.0.58: {} node-schedule@2.1.1: dependencies: @@ -12736,12 +12787,12 @@ snapshots: npm@11.20.0: {} - nuqs@2.10.1(next@16.3.8(@babel/core@7.29.7)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0))(react@19.3.0): + nuqs@2.10.1(next@16.4.0(@babel/core@7.29.7)(@playwright/test@1.58.2)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0))(react@19.3.0): dependencies: '@standard-schema/spec': 1.1.0 react: 19.3.0 optionalDependencies: - next: 16.3.8(@babel/core@7.29.7)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0) + next: 16.4.0(@babel/core@7.29.7)(@playwright/test@1.58.2)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0) object-assign@4.1.1: {} @@ -12907,8 +12958,6 @@ snapshots: p-map@7.0.8: {} - p-reduce@2.1.0: {} - p-reduce@3.0.0: {} p-timeout@6.1.4: {} @@ -13013,6 +13062,14 @@ snapshots: dependencies: find-up: 3.0.0 + playwright-core@1.58.2: {} + + playwright@1.58.2: + dependencies: + playwright-core: 1.58.2 + optionalDependencies: + fsevents: 2.3.2 + pngjs@5.0.0: {} pofile-ts@4.0.3: {} @@ -13389,26 +13446,26 @@ snapshots: rfdc@1.4.1: {} - rolldown@1.2.12: + rolldown@1.2.13: dependencies: - '@oxc-project/types': 0.152.0 + '@oxc-project/types': 0.153.0 '@rolldown/pluginutils': 1.0.1 optionalDependencies: - '@rolldown/binding-android-arm-eabi': 1.2.12 - '@rolldown/binding-android-arm64': 1.2.12 - '@rolldown/binding-darwin-arm64': 1.2.12 - '@rolldown/binding-darwin-x64': 1.2.12 - '@rolldown/binding-freebsd-x64': 1.2.12 - '@rolldown/binding-linux-arm-gnueabihf': 1.2.12 - '@rolldown/binding-linux-arm64-gnu': 1.2.12 - '@rolldown/binding-linux-arm64-musl': 1.2.12 - '@rolldown/binding-linux-ppc64-gnu': 1.2.12 - '@rolldown/binding-linux-s390x-gnu': 1.2.12 - '@rolldown/binding-linux-x64-gnu': 1.2.12 - '@rolldown/binding-linux-x64-musl': 1.2.12 - '@rolldown/binding-openharmony-arm64': 1.2.12 - '@rolldown/binding-win32-arm64-msvc': 1.2.12 - '@rolldown/binding-win32-x64-msvc': 1.2.12 + '@rolldown/binding-android-arm-eabi': 1.2.13 + '@rolldown/binding-android-arm64': 1.2.13 + '@rolldown/binding-darwin-arm64': 1.2.13 + '@rolldown/binding-darwin-x64': 1.2.13 + '@rolldown/binding-freebsd-x64': 1.2.13 + '@rolldown/binding-linux-arm-gnueabihf': 1.2.13 + '@rolldown/binding-linux-arm64-gnu': 1.2.13 + '@rolldown/binding-linux-arm64-musl': 1.2.13 + '@rolldown/binding-linux-ppc64-gnu': 1.2.13 + '@rolldown/binding-linux-s390x-gnu': 1.2.13 + '@rolldown/binding-linux-x64-gnu': 1.2.13 + '@rolldown/binding-linux-x64-musl': 1.2.13 + '@rolldown/binding-openharmony-arm64': 1.2.13 + '@rolldown/binding-win32-arm64-msvc': 1.2.13 + '@rolldown/binding-win32-x64-msvc': 1.2.13 router@2.2.0: dependencies: @@ -13564,20 +13621,19 @@ snapshots: setprototypeof@1.2.0: {} - shadcn@4.21.1(typescript@6.0.3): + shadcn@4.21.4: dependencies: '@babel/core': 7.29.7 '@babel/parser': 7.29.9 '@babel/plugin-transform-typescript': 7.29.9(@babel/core@7.29.7) '@babel/preset-typescript': 7.29.7(@babel/core@7.29.7) '@dotenvx/dotenvx': 1.75.1 - '@modelcontextprotocol/sdk': 1.32.0(zod@3.25.76) - '@shadcn/registry': 0.1.0(typescript@6.0.3) + '@modelcontextprotocol/sdk': 1.32.1(zod@3.25.76) + '@shadcn/registry': 0.1.3 '@types/validate-npm-package-name': 4.0.2 browserslist: 4.29.3 cn: 0.2.6 commander: 14.0.3 - cosmiconfig: 9.0.2(typescript@6.0.3) dedent: 1.7.2 deepmerge: 4.3.1 diff: 8.0.4 @@ -13604,7 +13660,6 @@ snapshots: - '@cfworker/json-schema' - babel-plugin-macros - supports-color - - typescript sharp@0.35.5(@types/node@25.9.9): dependencies: @@ -13718,7 +13773,7 @@ snapshots: socks@2.8.10: dependencies: - ip-address: 10.7.3 + ip-address: 10.7.4 smart-buffer: 4.2.0 sorted-array-functions@1.3.0: {} @@ -14007,7 +14062,7 @@ snapshots: dependencies: any-promise: 1.3.0 - three@0.170.0: {} + three@0.186.1: {} through2@2.0.5: dependencies: @@ -14309,7 +14364,7 @@ snapshots: '@unrs/resolver-binding-win32-ia32-msvc': 1.12.2 '@unrs/resolver-binding-win32-x64-msvc': 1.12.2 - update-browserslist-db@1.3.3(browserslist@4.29.3): + update-browserslist-db@1.3.4(browserslist@4.29.3): dependencies: browserslist: 4.29.3 escalade: 3.2.0 @@ -14391,22 +14446,22 @@ snapshots: d3-time: 3.1.0 d3-timer: 3.0.1 - vite-tsconfig-paths@6.1.1(typescript@6.0.3)(vite@8.3.2(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)): + vite-tsconfig-paths@6.1.1(typescript@6.0.3)(vite@8.3.4(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)): dependencies: debug: 4.4.3 globrex: 0.1.2 tsconfck: 3.1.6(typescript@6.0.3) - vite: 8.3.2(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1) + vite: 8.3.4(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1) transitivePeerDependencies: - supports-color - typescript - vite@8.3.2(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1): + vite@8.3.4(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1): dependencies: lightningcss: 1.33.0 picomatch: 4.0.7 postcss: 8.5.29 - rolldown: 1.2.12 + rolldown: 1.2.13 tinyglobby: 0.2.17 optionalDependencies: '@types/node': 25.9.9 @@ -14417,10 +14472,10 @@ snapshots: tsx: 4.23.15 yaml: 2.9.1 - vitest@4.1.11(@types/node@25.9.9)(@vitest/coverage-v8@4.1.11)(jsdom@29.1.1)(vite@8.3.2(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)): + vitest@4.1.11(@types/node@25.9.9)(@vitest/coverage-v8@4.1.11)(jsdom@29.1.1(@noble/hashes@2.4.0))(vite@8.3.4(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)): dependencies: '@vitest/expect': 4.1.11 - '@vitest/mocker': 4.1.11(vite@8.3.2(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)) + '@vitest/mocker': 4.1.11(vite@8.3.4(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)) '@vitest/pretty-format': 4.1.11 '@vitest/runner': 4.1.11 '@vitest/snapshot': 4.1.11 @@ -14437,12 +14492,12 @@ snapshots: tinyexec: 1.3.1 tinyglobby: 0.2.17 tinyrainbow: 3.1.1 - vite: 8.3.2(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1) + vite: 8.3.4(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1) why-is-node-running: 2.3.0 optionalDependencies: '@types/node': 25.9.9 '@vitest/coverage-v8': 4.1.11(vitest@4.1.11) - jsdom: 29.1.1 + jsdom: 29.1.1(@noble/hashes@2.4.0) transitivePeerDependencies: - msw @@ -14503,9 +14558,9 @@ snapshots: whatwg-mimetype@5.0.0: {} - whatwg-url@16.0.1: + whatwg-url@16.0.1(@noble/hashes@2.4.0): dependencies: - '@exodus/bytes': 1.16.0 + '@exodus/bytes': 1.16.0(@noble/hashes@2.4.0) tr46: 6.0.0 webidl-conversions: 8.0.1 transitivePeerDependencies: @@ -14542,6 +14597,8 @@ snapshots: is-weakmap: 2.0.2 is-weakset: 2.0.4 + which-command@0.1.0: {} + which-module@2.0.1: {} which-typed-array@1.1.24: diff --git a/proxy.ts b/proxy.ts index e4ba2bfe2..0ea24cfe2 100644 --- a/proxy.ts +++ b/proxy.ts @@ -1,9 +1,16 @@ import linguiConfig from 'lingui.config' import Negotiator from 'negotiator' import { type NextRequest, NextResponse } from 'next/server' +import { ALLOWED_HOSTS_ENV, isHostAllowed, parseAllowedHosts } from '@/src/lib/hostAllowlist' const { locales } = linguiConfig +// Paths served without a locale prefix: +// api — tRPC + REST endpoints, no locale needed +// panel — tRPC panel dev tool, served as a plain route handler +// vendor — static vendored scripts in public/vendor (three.js fallback) +const NO_LOCALE_PATH = /^\/(?:api|panel(?:\/|$)|vendor\/)/ + const getRequestLocale = (requestHeaders: Headers): string => { const langHeader = requestHeaders.get('accept-language') || undefined const languages = new Negotiator({ @@ -16,9 +23,34 @@ const getRequestLocale = (requestHeaders: Headers): string => { return activeLocale } +// DNS-rebinding guard (see src/lib/hostAllowlist.ts). A request without a +// Host header passes: Node rejects HTTP/1.1 requests that lack one, and the +// browsers that carry a rebinding attack always send it, so only HTTP/1.0 +// tools get here without one. The Vercel demo is served from a public domain +// with no pod behind it, so it skips the check. +const rejectUnknownHost = (request: NextRequest): NextResponse | undefined => { + if (process.env.NEXT_PUBLIC_DEMO === '1') return + const host = request.headers.get('host') + if (host === null) return + if (isHostAllowed(host, parseAllowedHosts(process.env[ALLOWED_HOSTS_ENV]))) return + return new NextResponse( + `Host not allowed. Use the pod's IP address, or add this host to ${ALLOWED_HOSTS_ENV}.\n`, + { status: 403, headers: { 'content-type': 'text/plain; charset=utf-8' } }, + ) +} + export const proxy = (request: NextRequest) => { + const rejected = rejectUnknownHost(request) + if (rejected) { + return rejected + } + const { pathname } = request.nextUrl + if (NO_LOCALE_PATH.test(pathname)) { + return + } + console.log('Current pathname:', pathname) const pathnameHasLocale = locales.some( (locale: string) => pathname.startsWith(`/${locale}/`) || pathname === `/${locale}`, @@ -38,12 +70,11 @@ export const proxy = (request: NextRequest) => { return NextResponse.redirect(request.nextUrl) } -// Routes excluded from locale redirect: -// api — tRPC + REST endpoints, no locale needed -// panel — tRPC panel dev tool, served as a plain route handler -// vendor — static vendored scripts in public/vendor (three.js fallback) +// Every API route and page runs the Host check. Build output and images are +// left out: they carry no pod data. export const config = { matcher: [ - '/((?!api|panel(?:/|$)|vendor/|_vercel|_next/static|_next/image|favicon.ico|.*\\.(?:svg|png|jpg|jpeg|gif|webp)$).*)', + '/api/:path*', + '/((?!api|_vercel|_next/static|_next/image|favicon.ico|.*\\.(?:svg|png|jpg|jpeg|gif|webp)$).*)', ], } diff --git a/scripts/algorithm-experiments/README.md b/scripts/algorithm-experiments/README.md new file mode 100644 index 000000000..e9d95a1d5 --- /dev/null +++ b/scripts/algorithm-experiments/README.md @@ -0,0 +1,578 @@ +# Off-pod algorithm experiments + +`executePackageExperiment` runs an inert, hash-verified package in a reviewed +local Docker image selected by an operator policy. It is development tooling, +not a pod API, installation service, or qualification decision. Known pod DAC +socket paths are rejected. Never bundle this runner into the pod web server. + +The image must be operator-reviewed and selected by immutable local `sha256:` +ID. The package's dependency-lock artifact uses the +`biometric-python-environment-v1` JSON schema: exact image ID, Python version, +implementation, OS, architecture and the complete sorted installed distribution +inventory. A restricted probe container with no package files mounted collects +that inventory before candidate execution. Any mismatch rejects the experiment. +The result carries the exact lock-byte hash and observed environment hash. +Do not pass a candidate-selected image or policy. + +Use `inspectPythonEnvironment(imageId)` to capture a lock from a reviewed image, +then include those JSON bytes as a hashed package artifact. This is an +**environment lock**, not a pip or uv dependency-resolution file; existing uv +locks may also be retained as package artifacts for review. Preparing an image +from the source dependency lock remains an operator's reviewed build step. The +runner never installs dependencies or executes build hooks. The immutable image +ID binds the configuration and layers; inventory verification additionally +makes interpreter/platform/package-version mismatches explicit. See +[Docker image identities](https://docs.docker.com/reference/cli/docker/image/pull/) +and [Python distribution metadata](https://docs.python.org/3.12/library/importlib.metadata.html). +A matching lock does not establish that dependencies or the image are safe. + +The process runs as UID 65534 with no network, capabilities, privilege elevation, +or writable root filesystem. Only a fresh verified copy of the package is +mounted, read-only. No pod files, host credentials, Docker socket, or database +are mounted. Docker's default seccomp profile remains enabled. Memory, swap, +PID count, stdout/stderr bytes, input bytes, and a deadline are bounded. The +runner checks that the daemon reports the required limit and seccomp support. +These controls use Docker's [documented container options](https://docs.docker.com/engine/containers/run/). + +CPU is restricted to one core and the deadline is the smaller of requested CPU +and wall time, conservatively charging idle time against CPU budget. Scheduler +and kill latency can overshoot the deadline; execution metadata is not a CPU or +peak-memory qualification receipt. Experiments still require measured resource +qualification before any promotion. Use a dedicated external Docker host or VM +with no pod credentials; containers share its kernel and are not a substitute +for maintaining that host's isolation and security updates. + +The package entrypoint is a Python executable implementing JSON Lines: + +1. Read one `initialize` object with `request` and `runHash`; reset all state and + initialize deterministically from the explicit seed/configuration. +2. Read each `frame` object with `frame` and `frameHash`. +3. Write exactly one `biometric-metric-frame-v1` response line per input frame. +4. Exit successfully at stdin EOF. Logs may go to stderr within the same total + output budget. Additional stdout is an error. + +Only one frame is in flight. Host validation runs before sending a frame and +before retaining its response. The caller supplies a trusted synchronous frame iterator. Before starting the +candidate, the runner writes canonical frames to a private bounded file and +checks their domain-separated SHA-256 hash against the request's `inputHash`. +The file is not mounted into the container; only one frame at a time is sent +from that frozen copy. The hash encoding is UTF-8 +`biometric-package-input-v1\n` followed by each canonical frame plus LF. +Input hash validation does not establish a clinical reference or holdout split; +those associations still need the qualification dataset pipeline. +The host staging parent stays mode 0700, with frozen input mode 0600. Only its +`package/` subdirectory (0755, files 0444) is bind-mounted read-only. The container +UID can traverse that mount without granting other host users access through the +private parent. An opt-in real-Docker test verifies these modes after container +creation, successful execution as UID 65534, and removal after completion. +Package files and outputs are never imported as host code. Cleanup removes the +owned container and staging directory on normal completion and caught errors; +hard host termination is handled by the explicit container recovery sweep +described below, followed by ownership-checked private staging recovery. No arbitrary dependency installation occurs during execution. + +Run the opt-in Docker regression tests with a reviewed Python image: + +```sh +BIOMETRIC_EXPERIMENT_TEST_IMAGE=sha256: pnpm exec vitest run scripts/algorithm-experiments/container.test.ts +``` + +The tests cover a valid exchange, filesystem/network/environment/PID probes, +output floods, memory exhaustion, deadlines, and invalid results. They do not +qualify any biometric algorithm or prove resistance to kernel vulnerabilities. + +## Determinism evidence + +`checkPackageDeterminism` runs the same manifest, byte snapshots, request and +operator policy two to five times, with a fresh container each time. Its trusted +frame factory must reproduce the retained input: every run independently +verifies that hash. It hashes every response, including serialized state, in +order using the `biometric-package-output-v1` domain. Execution timings and +transport measurements are excluded from the output identity. + +The result records run/input/output identities and execution metadata. A +changing state fails even when numeric readings are identical. A deterministic +result is only one qualification dimension: it does not approve dependency +compatibility, resource fitness, artifact/noise behavior, held-out agreement, +shadow use, installation, or live activation. Those checks and reviewed release +promotion remain required. + +## File-based command line + +Run these commands on the external experiment host, from the repository: + +```sh +pnpm biometrics:experiment lock sha256: environment.lock.json +pnpm biometrics:experiment inspect proposal.json intake.json +pnpm biometrics:experiment input-hash frames.jsonl 33554432 input-identity.json +pnpm biometrics:experiment run proposal.json request.json frames.jsonl operator-policy.json experiment.json +pnpm biometrics:experiment determinism proposal.json request.json frames.jsonl operator-policy.json 2 determinism.json +``` + +The proposal is a single inert JSON envelope with `schema` equal to +`biometric-package-proposal-v1`, a complete `manifest`, and `files` containing +`{ "path": "engine.py", "base64": "..." }` entries for every declared artifact. +Include the captured environment lock as the manifest's `dependencyLock` file; +its exact bytes, like all package files, must match the artifact size and hash. +`inspect` verifies the proposal without launching Docker or executing its code. + +`frames.jsonl` contains one `biometric-signal-frame-v1` object per line. +`request.json` follows `packageRunRequestSchema` in `protocol.ts`: set its +`packageHash` from the intake report and `inputHash` from the input-identity +report, plus explicit hardware, sample rate, side, configuration, and seed. + +`operator-policy.json` is separate from candidate artifacts and contains the +reviewed image ID and positive integer ceilings: `maximumWallMs` (at most one +hour), `maximumMemoryBytes`, `maximumProcesses`, `maximumOutputBytes`, and +`maximumInputBytes`. Select these for the external host; they do not establish +that an algorithm fits the pod. Candidate resource requests must fit the policy. + +All file inputs must be bounded regular files, not symlinks, devices or FIFOs. +JSONL has a separate per-line cap. Reports are mode 0600, written completely +before atomic publication, and never overwrite an existing path. Errors return +nonzero and do not publish a success report. The report includes raw candidate +outputs and should be treated as private biometric data. No command sends it to +a reviewer or installs the package. A successful `run` is explicitly marked +`experiment_only`; use the remaining qualification checks and normal reviewed +release process before promotion. + +## Host resource observations + +Experiment reports now include `execution.measurements`, collected from the +Docker Engine stats API on the host, never from candidate output. The collector +uses the actual created container ID, validates each response's identity, and +records observed cumulative CPU milliseconds, memory bytes including cache, +and process counts. It attempts sampling every 250 ms without overlapping +requests, and requests a final sample while the candidate is still waiting for +stdin EOF. Reads have time and byte limits. + +The receipt records attempts, usable samples, and maximum gaps including +startup and shutdown. Missing, stale, or regressing samples leave it incomplete. Memory is an observed peak, +not an exact lifetime high-water mark; CPU is the largest observed cumulative +counter, not a guaranteed final lifetime total. A process may consume resources +between samples. The collector's overhead counts toward the experiment's wall +deadline. Qualification must inspect coverage and gaps and retain these limits. +No fit-for-pod decision is inferred from measurements on an external host. + +Currently measurement supports local Unix Docker endpoints and API 1.41 or +newer. Unsupported transports or failed observations leave nullable counters +and incomplete evidence; they do not fabricate zero usage or waive the existing +container resource limits. See the [Docker stats API](https://docs.docker.com/reference/api/engine/version/v1.51/). + +## Artifact/noise fixture suites + +```sh +pnpm biometrics:experiment fixtures proposal.json suite.json operator-policy.json fixture-report.json +``` + +A reviewed suite uses `biometric-package-fixtures-v1` and a `fixtures` array. +Each fixture has a unique `id`, `category`, full run `request`, inline signal +`frames`, and `expectations`. Its request must bind the proposal and actual +frame hashes. The suite file is bounded to 32 MiB; inline fixtures are intended +for focused signal tests, not whole-night held-out comparisons. + +Expectation windows use inclusive `startMs`, exclusive `endMs`, and a declared +`metric`. A `numeric` expectation specifies `minimum`, `maximum`, +`minimumReadings`, and `minimumQuality`. Missing/null readings cannot satisfy +numeric coverage. An `abstain` expectation permits absent or null output and +rejects numeric values. Duplicate readings, unexpected metrics/times, +out-of-range values, low quality, insufficient coverage and overlapping +expectations fail. Windows must fall inside the supplied input interval. +Clean-signal fixtures must require numeric output; empty-bed fixtures must +require abstention. + +Each fixture executes in a fresh isolated container. Reports retain the suite, +fixture, input and output identities, the complete frozen suite with frames and +expectations, execution output, independent resource observations and failures. +A crashed or invalid-output case remains a failed case in the report. + +Required category coverage is `clean_signal`, `pump`, `empty_bed`, `motion`, +`cross_side`, `harmonics`, `gaps`, `warmup`, and `boundary_reset`. Omitted categories +make an otherwise passing report `incomplete`. All covered checks passing yields +`fixture_checks_passed`, never automatic qualification or installation. Category +labels do not prove that a signal realistically represents that condition: +review the exact hashed suite and its expected behavior. The test suite includes +synthetic clean/empty-bed examples to verify this workflow; representative +shipped-algorithm fixtures, held-out agreement, compatibility and pod resource +qualification remain separate requirements. + +Each fixture suite also requires an explicit reviewed `resourcePolicy` with +schema `biometric-package-resource-policy-v1`, `maximumWallMs`, `maximumCpuMs`, +`maximumMemoryBytes`, `maximumProcesses`, `minimumSamples`, +`maximumObservationGapMs`, and `evidenceMode` equal to +`sampled_counters_and_enforced_limits`. There are no production default +thresholds. The policy is part of the frozen suite hash and every per-case +resource assessment records its own policy hash. + +An assessment can be `within_observed_limits`, `insufficient_evidence`, or +`failed`. It requires consistent host receipts, adequate sample count and gaps, +and enforced limits no wider than the reviewed budget. Known wall, observed +CPU/memory/process, or enforced-limit overruns fail even when sampling is also +incomplete. Missing measurements never mean zero cost. A passing signal check +with incomplete resource evidence makes the overall fixture report incomplete; +a resource failure makes it fail. These sampled observations plus hard limits +still do not establish exact lifetime costs or target-pod fitness. + +## Reference comparison + +```sh +pnpm biometrics:experiment compare comparison-input.json comparison-report.json +``` + +This command reuses the application's reference normalization, point/interval +alignment, and common-support numeric comparison. The input uses schema +`biometric-package-comparison-v1`, a `metricId`, whole-night `startMs`/`endMs` +(up to 24 hours), explicit `minimumQuality`, raw `referenceImport`, frozen +`alignment`, and exactly three `outputs` labelled `default`, `current`, and +`candidate`. Each output supplies its `manifest`, run `request`, and complete +executor `result`. Inputs are bounded to 32 MiB. Run IDs include configuration; +reported baseline/candidate hashes in the shared comparison are those run IDs. + +Each selected manifest metric must declare a `reference` contract with a +supported metric definition and nullable `windowMs`. HRV definitions require +an explicit window, and units must be canonical (`bpm`, `breaths/min`, or `ms`). +A free-text description or a coincidentally matching output name is insufficient. +All packages must agree on that contract, input identity, hardware, sample rate, +side and input-frame sequence. Different HRV definitions or windows cannot be +compared as the same metric. + +Reference rows are normalized using the shared adapter. Out-of-night rows and +compatibility exclusions are reported explicitly. Comparisons use identical, +unambiguous observation timestamps shared by all three runs before reference +pairing; rejected/low-quality readings remain missing support, never filled +values. MAE, bias and coverage differences are descriptive reference agreement. +The command does not trust file-provided results as execution attestations, +record holdout access, estimate whole-night uncertainty, designate an objective, +or authorize promotion. Those require the authoritative experiment/qualification +workflow, not a report assembled from arbitrary files. + +## Durable study ledger (orchestration component) + +`PackageExperimentLedger` maintains frozen study identities, exact trial +requests, idempotency keys, expiring worker leases and report hashes in a +**dedicated off-pod SQLite database**. Its generated schema and migrations are +under `src/db/package-experiment-*`; generate changes with +`pnpm exec drizzle-kit generate --config=drizzle.package-experiments.config.ts`. +Do not use either pod database for this ledger. Initialization refuses nonempty +foreign databases, uses an application identifier, and rejects future format +versions rather than migrating them backward. + +Study nights bind subject/side, start/end, input/reference hashes, split and +protocol hash. Splits must be chronological and nonoverlapping. Claims record +exposure transactionally before selected input is opened. Every attempt counts, +including failures, expired leases and retries; changed study IDs or corrected +input/reference hashes cannot erase overlap with an earlier attempt on the +same subject and side. A stale attempt cannot renew a replacement's lease or +publish over its report. Completed trials return their retained report identity +rather than silently running again. + +Claim-time `unexposedHoldoutAtClaim` is historical information. Re-read +`currentEvidence` for current eligibility: another overlapping trial can make a +previously unique holdout attempt non-unique. Keeping this ledger across studies +is required; a new/deleted ledger, relabelled subjects, or data inspected outside +its workflow cannot prove independent holdout use. Final qualification must +bind subjects and windows to retained datasets and recheck exposure while +publishing its decision. + +The exploratory `run`, `compare`, `determinism`, and `fixtures` commands do not +claim studies through this ledger and cannot issue independent-holdout promotion +evidence. `runManagedTrial` and the `managed` command integrate it as described +below. `assessManagedQualification` combines verified external evidence as +described below; installation remains a separate reviewed code/release step. + +## Managed whole-night runner + +`runManagedTrial` in `managed.ts` accepts a reserved ledger trial, its frozen +`biometric-managed-protocol-v1`, exactly three package variants, a private report +directory, and a trusted retained-night loader. The protocol freezes hardware, +sample rate, metric, quality threshold, reference alignment, whole-night +bootstrap settings and resource policy. Each trial variant binds the package +hash, runtime image, and `configHash({ config, seed })`. + +It verifies package/configuration/protocol identities before claiming a lease. +The loader runs only after all selected night exposures are committed. Loaded +references must match the study's subject, side and shared import content hash. +Frames must lie within their frozen night and match its actual input hash. All +three variants run in independent containers; results use the shared typed +comparison and paired whole-night uncertainty functions. Resource assessments +remain attached to every variant. + +Private per-night and trial reports are published under their content hashes, +then the ledger's live attempt fence selects the winning report. Heartbeats and +checks between expensive steps renew leases; lost leases abort active execution +or reject publication. Failures retain partial-night evidence and exposure. +An old worker may leave an unreferenced artifact but cannot overwrite the winning +ledger pointer. Restarted completed trials verify the complete retained report graph before +reusing it and never silently reread holdout input. Missing or corrupted +per-night artifacts fail rather than trigger a new evaluation. + +A successful run means the evaluation completed, not that its candidate passed +all promotion criteria. Check current ledger exposure, coverage, uncertainty, +resource assessments, deterministic/noise/compatibility evidence and reviewed +release admission. The runner does not install packages or claim pod fitness. +Hard process termination can still leave orphan containers/artifacts; use the +ownership-checked recovery procedures below. Combined external qualification +rechecks the retained evidence and current exposure without authorizing installation. + +## Managed descriptor command + +```sh +pnpm biometrics:experiment managed ./experiment-workspace descriptor.json receipt.json +``` + +The command creates a private workspace (0700) containing a dedicated +`experiments.sqlite` (0600) and private `reports/`. Reuse the same workspace +across studies; deleting or replacing it destroys the exposure history needed +for future decisions. Existing public directories, symlinked workspace/report +paths, symlinked database/sidecar files, and foreign databases are rejected. +No pod database is opened or migrated. + +Managed execution acquires an OS-released SQLite write lock in `.writer.sqlite` +for the entire workspace operation. Competing controllers fail before changing +the ledger or starting containers. Never unlink this lock file to resolve a busy +workspace: doing so would create a second lock inode. A killed controller releases +the database lock automatically; container and private-staging recovery are still +separate responsibilities. Low-level trial APIs are trusted building blocks; +`managed` is the bounded persistent-workspace entry point. + +The default workstation storage policy allows 2,048 report files, 1 GiB of total +report bytes, 32 MiB per report and a 64 MiB ledger, while requiring 1 GiB of free +space plus room for ledger growth/rollback and the next report. These are off-pod +storage limits, not production pod resource budgets. Supply a strict JSON policy +as the optional fourth managed argument to override them: + +```sh +pnpm biometrics:experiment managed WORKSPACE DESCRIPTOR RECEIPT STORAGE_POLICY +``` + +Policy fields are `maximumReportFiles`, `maximumReportBytes`, +`maximumSingleReportBytes`, `maximumDatabaseBytes`, and `minimumFreeBytes`. +Checks run before work, before each retained-night load, and before each report +publication. They bound managed report files, including abandoned temporary files, +and SQLite growth via `max_page_count` with DELETE-mode rollback journals. The +report directory must contain only private, owned, single-link regular files. +Symlinks, subdirectories and unsafe database/lock files are rejected. No existing +reports or exposure history are deleted to satisfy a quota. When publication is +refused, the failure-finalization behavior below still applies. + +Free-space checks are admission snapshots; unrelated applications can consume +space afterward. The ledger cap plus its rollback journal is accounted separately +from report bytes. Operator source files, the output receipt path, Docker storage, +and container input staging outside this workspace are not covered by these +workspace limits. Their existing per-input/output limits still apply; aggregate +staging quotas and crash recovery require separate controls. + +A descriptor uses `biometric-managed-descriptor-v1` with: + +- `study`: the frozen `biometric-package-study-v1`, including all chronological + night assignments and their input/reference hashes. +- `protocol`: the exact managed protocol whose canonical hash is the study's + `protocolHash`. +- `split` and an `idempotencyKey` for this exact trial. +- `variants`: exactly `default`, `current`, and `candidate`, each specifying + `proposal` (file path), `config`, `seed`, and the operator execution `policy`. +- `nights`: exactly the selected split's night IDs, each with `frames` (JSONL + path) and `referenceImport` (raw reference-import JSON path). + +Failure finalization also runs when the failure report cannot be published (for +example, a full report volume). If the ledger remains writable and the lease is +current, the attempt becomes `failed` with a null report hash. No nonexistent +artifact is claimed. Publication or ledger-finalization failures are returned +alongside the original error in an `AggregateError`; its cause remains the original +failure. Failed attempts retain their exposure history. A stale lease cannot end +or annotate a replacement attempt. A failed ledger write leaves state unconfirmed +and requires subsequent lease recovery; the CLI does not report successful cleanup. + +Relative source paths resolve from the descriptor's directory. The descriptor +is operator-owned input, not candidate-provided instructions. Source files must +pass the bounded regular-file reader. Study reference hashes use the shared +`previewReferenceImport(...).summary.contentHash`; input hashes use +`packageInputHash`. Keep retained sources and proposals alongside the study for +reproducibility; metadata alone does not prove a complete night or subject +identity. + +Night files are opened only after the ledger records exposure. The receipt +contains the winning trial/report identity, report path and current ledger +evidence. Repeating an unchanged completed descriptor verifies and reuses its +report without rereading night files. Reusing its idempotency key with changed +configuration or study fails. Review current exposure again at any later +promotion decision, since other trials can invalidate first-use evidence. +A receipt is not an installation or qualification approval. + +## Recovery after a terminated experiment controller + +```sh +pnpm biometrics:experiment recover recovery-report.json +``` + +Experiment and environment-probe containers carry immutable version, invoking +host UID, kind and expiry labels. Expiry includes the maximum execution deadline +plus two minutes for startup and cleanup. Run this command after restarting an +experiment host and periodically while accepting work. It checks labels, names +and full container IDs on the currently selected Docker daemon, then force-removes +only overdue owned containers by immutable ID. Unexpired, unlabelled and foreign +containers remain untouched. The private report records inspected count and +removed IDs; repeating a sweep is safe. A daemon outage fails the command and +must be retried. Legacy containers without these labels require manual review. + +Recovery does not retroactively qualify an interrupted trial or erase exposure. +The ledger's expired lease permits a fresh fenced attempt, with earlier holdout +access still counted. The container sweep does not delete retained reports; its subsequent staging +sweep is described below. This is not a background supervisor: the external host +must schedule it. Host clock correctness is required for absolute expiry. Aggregate +temporary-staging quotas and periodic recovery scheduling remain separate work. + +## Retained evidence verification + +`readManagedEvidence(ledger, trialId, reportDirectory)` starts from the successful +ledger-selected report hash. It verifies every linked night artifact, exact +study/request/protocol/attempt identities, the complete selected-night set, +reference content and subject/side, all three package/configuration/input/image +identities, and declared resource limits. It recomputes reference comparisons, +resource assessments and whole-night uncertainty, requiring both the per-night +and top-level summaries to match those calculations. Updating hashes around an +altered score does not make that score valid. + +Completed managed runs use this verifier before returning a cached result. It +reads reports only, without rerunning candidate code or reopening retained raw +night inputs. The returned evidence includes a fresh ledger exposure snapshot; +later overlapping access can invalidate a previously unique holdout attempt. +This does not prove that a local administrator has never replaced the ledger or +fabricated raw execution records. The experiment host and its retained history +remain trusted. Qualification must recheck current exposure when publishing its +decision and still requires objective criteria, determinism, representative +fixtures, compatibility, target-pod resources and reviewed release promotion. + +## Frozen accuracy criteria + +Managed protocols may include `accuracyPolicy` using schema +`biometric-package-accuracy-policy-v1`. Choose and review its explicit criteria +before freezing the study: `minimumMaeImprovement`, +`maximumNightMaeRegression`, `maximumNightP95Regression`, +`maximumNightAbsoluteBiasIncrease`, `minimumCommonPairsPerNight`, +`minimumReferenceCoverage`, `maximumReferenceCoverageLoss`, and +`maximumLostTimestampFraction`. Error thresholds use the selected metric's +canonical unit; coverage/loss thresholds are fractions. There are no implicit +production thresholds. Changing criteria changes the protocol hash and cannot +retrofit the original study. Legacy protocols without criteria remain readable +but cannot meet the accuracy gate. + +The managed receipt's `accuracy` assessment is recomputed from verified retained +evidence and current ledger exposure. It requires a unique current holdout +attempt, sufficient common pairs on every selected night, coverage and support +within the frozen limits, no excessive per-night MAE/tail/bias regression, and +an estimated whole-night confidence interval supporting the requested MAE +improvement against both default and current. Missing or degenerate uncertainty, +omitted nights, ambiguous support and absent criteria remain insufficient. +Known violations produce `criteria_not_met` even if other evidence is missing. + +`criteria_met` means only that these reference-agreement criteria passed. It is +not clinical validation or package qualification: determinism, resource/noise +fixtures, compatibility, pod fitness, prospective monitoring and reviewed release +promotion still apply. The assessment retains exact failures and missing evidence +and never installs code or changes live processing. + +## Combined external preflight + +```sh +pnpm biometrics:experiment preflight proposal.json plan.json suite.json operator-policy.json preflight-report.json +``` + +The plan uses `biometric-package-preflight-plan-v1`, `suiteHash` (the canonical +`dataHash` of the parsed fixture suite), `repetitions` (2–5), `configHash` +(`configHash({ config, seed })`), `hardwareId`, `sampleRateHz`, and `side`. +Every fixture must use that exact candidate configuration and seed, hardware, +rate and side. A changed suite or configuration is rejected before execution. +Use reviewed synthetic fixtures here, not undisclosed holdout data. + +The runner snapshots candidate bytes, suite and operator policy, executes the +fixture suite, then repeats every fixture in independent containers to check +full-output/state determinism. Each repeat also receives a resource assessment +under the suite's frozen policy. Execution failures and nondeterminism fail the +preflight. Missing fixture categories or incomplete resource observations keep +it incomplete. Cancellation propagates to active experiments and stops further +fixtures/repeats. Reports retain the exact plan and its hash, fixture report, +per-fixture determinism reports, resource assessments and bounded errors. + +`external_checks_passed` is external preflight only. Managed studies can bind +the preflight artifact as described below; standalone reports cannot prove +holdout independence or pod fitness. The final combined qualification decision +remains separate. The candidate still needs representative fixture review, current +held-out accuracy, target-pod compatibility/resources, and normal reviewed +release promotion before it can become live. + +## Binding preflight to a managed study + +Include `preflightPlan` in the managed protocol and `preflightSuite` (a suite-file +path relative to the descriptor) in the managed descriptor. Both are required +together. The study protocol hash freezes the plan, including suite identity, +repetitions and candidate configuration/seed. Its hardware, rate and side must +match the managed study. The runner snapshots the suite before asynchronous work +and rejects mismatches before claiming a trial. + +During the leased attempt, preflight runs under the same cancellation heartbeat +before night comparisons. Its complete report, including candidate manifest, is +published by content hash and referenced by the final trial report. Failed or +incomplete preflight remains visible alongside the night results; completing a +trial does not imply that preflight passed. Failure reports retain a completed +preflight reference if a later step fails. Legacy trials without a plan have no +preflight evidence and cannot supply that qualification dimension. + +The retained-evidence verifier reopens the bound preflight artifact and checks +its plan, candidate/image, exact fixture set, fixture inputs and output protocol, +resource limits and measurements, repeat count, run identities and determinism +hashes. It recomputes fixture outcomes, resource assessments, missing categories +and overall status. Missing, altered or inconsistent artifacts fail cached reuse. +Completed file workflows do not reread the original suite or night input files. +The managed receipt exposes verified preflight status alongside accuracy; target +pod compatibility, representative-fixture review and release promotion remain +required before an algorithm can become live. + +## Combined external qualification assessment + +`assessManagedQualification(ledger, trialId, reportDirectory)` verifies the full +retained graph once, reads current holdout exposure, assesses frozen accuracy, +and combines it with bound preflight and every night/variant's resource evidence. +The managed command includes this `qualification` object and an `assessmentHash` +in its receipt. The hash binds trial/report/protocol/request identities, current +exposure and all assessment dimensions. Re-reviewing unchanged evidence is stable; +later overlapping holdout use changes the snapshot and invalidates its independent +holdout claim even though the original execution report remains unchanged. + +The combined status is `failed`, `insufficient_evidence`, or +`external_criteria_met`. Known failures dominate missing evidence, with both +reason lists retained. Passing accuracy cannot waive absent/failed preflight, +missing or duplicated resource rows, substituted resource policy, budget excess, +or inadequate measurements for default/current/candidate on any selected night. + +This is an external evidence snapshot, not a durable installation authorization. +`installationAuthorized` remains false, and the receipt lists outstanding +representative-fixture review, target-pod compatibility and resource qualification, +prospective health/reference comparison, and reviewed shipped-registry release. +A future promotion step must recheck authoritative evidence at publication; an +old exported assessment must never substitute for that current check. + + +### Abandoned private staging + +Before freezing inputs, the executor writes a private, fsynced ownership receipt +inside its mode-0700 staging directory. It binds the directory name/device/inode, +host UID, controller PID, exact container name, and wall deadline plus two-minute +grace. These receipts require a Unix host UID (macOS or Linux). Package code cannot +read or modify them through its package-only mount. + +`recover OUTPUT` first performs the existing expired-container sweep, then scans +host temporary staging. Its output is `biometric-experiment-recovery-v1` with +separate `containers` and `staging` results. Staging is removed only when the +receipt is valid and expired, its controller PID is confirmed absent with ESRCH, +and Docker confirms the named container does not exist. A present container or +an unavailable daemon never proves absence. PID reuse/permission errors retain +staging conservatively. Directory and marker identities are rechecked before +removal; markers must be private, owned, bounded regular files with one link. +The sweep is bounded to 16,384 temporary-directory entries and 128 candidates. + +Normal cleanup also preserves staging if container removal fails. Legacy or +unmarked directories, damaged receipts, symlinks, and unproven ownership are left +untouched. A crash between creating the empty directory and writing its receipt +can leave an unmarked empty directory; recovery will not guess its ownership. +Reports and exposure ledgers are not touched. This is an explicit recovery sweep; +a periodic host service and aggregate temporary-staging quotas remain separate. diff --git a/scripts/algorithm-experiments/accuracy-policy.ts b/scripts/algorithm-experiments/accuracy-policy.ts new file mode 100644 index 000000000..d9414ffd5 --- /dev/null +++ b/scripts/algorithm-experiments/accuracy-policy.ts @@ -0,0 +1,15 @@ +import { z } from 'zod' + +/** Operator-reviewed criteria, frozen into the protocol before holdout access. + * There are deliberately no production defaults or clinical accuracy claims. */ +export const packageAccuracyPolicySchema = z.object({ + schema: z.literal('biometric-package-accuracy-policy-v1'), + minimumMaeImprovement: z.number().finite().positive(), + maximumNightMaeRegression: z.number().finite().nonnegative(), + maximumNightP95Regression: z.number().finite().nonnegative(), + maximumNightAbsoluteBiasIncrease: z.number().finite().nonnegative(), + minimumCommonPairsPerNight: z.number().int().positive().max(100000), + minimumReferenceCoverage: z.number().positive().max(1), + maximumReferenceCoverageLoss: z.number().min(0).max(1), + maximumLostTimestampFraction: z.number().min(0).max(1), +}).strict() diff --git a/scripts/algorithm-experiments/accuracy.test.ts b/scripts/algorithm-experiments/accuracy.test.ts new file mode 100644 index 000000000..b05be08e8 --- /dev/null +++ b/scripts/algorithm-experiments/accuracy.test.ts @@ -0,0 +1,82 @@ +// @vitest-environment node +import { createHash } from 'node:crypto' +import { describe, expect, it } from 'vitest' +import { compareNumericConfigurations } from '../../src/biometrics/backtest/numeric-comparison' +import { createNightUncertainty } from '../../src/biometrics/backtest/uncertainty-core' +import { normalizeReferenceImport } from '../../src/biometrics/references/normalize-core' +import { assessPackageAccuracy } from './accuracy' +import { managedProtocolSchema } from './managed-protocol' +import { fixtureResourceTestPolicy } from './resource-test-policy' +import { configHash } from '../../src/biometrics/algorithms/canonical' + +type Input = Parameters[0] +const policy = { schema: 'biometric-package-accuracy-policy-v1', minimumMaeImprovement: 0.5, maximumNightMaeRegression: 0, + maximumNightP95Regression: 0, maximumNightAbsoluteBiasIncrease: 0, minimumCommonPairsPerNight: 2, + minimumReferenceCoverage: 1, maximumReferenceCoverageLoss: 0, maximumLostTimestampFraction: 0 } +function fixture(): Input { + const protocol = managedProtocolSchema.parse({ schema: 'biometric-managed-protocol-v1', hardwareId: 'pod4', sampleRateHz: 500, metricId: 'heart_rate', minimumQuality: 0.8, + alignment: { version: 'reference-alignment-v1', pointToleranceMs: 0, clockOffsetMs: 0, offsetSource: 'fixed' }, + uncertainty: { version: 'paired-night-percentile-v1', seed: 'accuracy-test', resamples: 200, minimumNights: 3 }, + resourcePolicy: fixtureResourceTestPolicy, accuracyPolicy: policy }) + const reference = normalizeReferenceImport({ subjectId: 'fixture', side: 'left', source: 'synthetic', + rows: [0, 1].map(index => ({ recordKey: String(index), metric: 'heart_rate', value: 60, unit: 'bpm', timestampMs: 1700000000000 + index * 1000 })) }) + const nights = [0, 1, 2].map((index) => { + const outputs = (['default', 'current', 'candidate'] as const).map((label, variant) => ({ label, configHash: String(variant), + observations: [0, 1].map(point => ({ timestampMs: 1700000000000 + point * 1000, value: variant === 2 ? 60 : 64 + index + (1 - variant) })) })) + const comparison = { schema: 'biometric-package-comparison-report-v1' as const, inputHash: 'a', referenceHash: 'b', metricId: 'heart_rate', + contract: { metric: 'heart_rate' as const, windowMs: null }, alignment: protocol.alignment, startMs: 1700000000000, endMs: 1700000060000, + minimumQuality: 0.8, excludedBoundaryReferences: 0, compatibilityExclusions: {}, referenceSummary: reference.summary, + comparisons: compareNumericConfigurations('heart_rate', null, outputs, reference.measurements, protocol.alignment), + interpretation: 'descriptive_reference_agreement_not_independent_holdout_or_promotion_evidence' as const } + return { nightId: String(index), reportHash: 'c', comparison, resources: [] } + }) + const uncertainty = createNightUncertainty(text => createHash('sha256').update(text).digest(), 'equal_night_mean_common_mae_difference') + return { protocol, nights, uncertainty: (['default', 'current'] as const).map(baseline => ({ baseline, + report: uncertainty(nights.map(night => ({ cohortId: night.nightId, difference: night.comparison.comparisons.find(row => row.baseline === baseline && row.candidate === 'candidate')?.differences.commonMae ?? null })), protocol.uncertainty) })), + currentEvidence: { trialId: 'fixture', state: 'succeeded', reportHash: 'hash', otherExposures: [], uniqueHoldoutAttempt: true } } +} +describe('frozen package accuracy criteria', () => { + it('requires supported improvement against both baselines without authorizing installation', () => { + expect(assessPackageAccuracy(fixture())).toMatchObject({ status: 'criteria_met', failures: [], missing: [], interpretation: expect.stringContaining('requires_full_package') }) + const input = fixture() + const report = input.uncertainty[1].report + if (report.interval) report.interval.upper = 0 + expect(assessPackageAccuracy(input)).toMatchObject({ status: 'criteria_not_met', failures: ['current:improvement_not_supported'] }) + }) + it('cannot backfill criteria into an already frozen protocol or reuse exposed holdout', () => { + const input = fixture(), frozen = configHash(input.protocol) + delete input.protocol.accuracyPolicy + expect(configHash(input.protocol)).not.toBe(frozen) + expect(assessPackageAccuracy(input).missing).toContain('no_frozen_accuracy_policy') + input.currentEvidence.uniqueHoldoutAttempt = false + expect(assessPackageAccuracy(input).missing).toContain('no_unique_current_holdout') + }) + it('rejects coverage loss and individual-night regressions even with favorable aggregate intervals', () => { + const input = fixture() + const row = input.nights[0].comparison.comparisons.find(row => row.candidate === 'candidate' && row.baseline === 'default') + if (!row) throw new Error('Missing fixture comparison') + row.ownSupport.candidate.referenceCoverage = 0.5 + row.differences.referenceCoverage = -0.5 + row.lostObservationTimestamps = 1 + row.differences.commonMae = 1 + row.differences.commonP95AbsoluteError = 2 + if (row.commonSupport.candidate.metrics) row.commonSupport.candidate.metrics.signedBias = 99 + const result = assessPackageAccuracy(input) + expect(result.status).toBe('criteria_not_met') + expect(result.failures).toEqual(expect.arrayContaining(['0:default:reference_coverage_below_minimum', '0:default:reference_coverage_loss', + '0:default:observation_support_loss', '0:default:night_mae_regression', '0:default:night_tail_regression', '0:default:night_bias_regression'])) + }) + it('does not treat omitted nights, missing pairs or degenerate intervals as successful evidence', () => { + const input = fixture() + input.uncertainty[0].report.omittedNights = 1 + input.uncertainty[1].report = { ...input.uncertainty[1].report, status: 'degenerate', interval: null } + const row = input.nights[0].comparison.comparisons.find(row => row.candidate === 'candidate' && row.baseline === 'default') + if (!row) throw new Error('Missing fixture comparison') + row.commonSupport.candidate.pairedCount = 0 + row.ambiguousObservationTimestamps.candidate = 1 + expect(assessPackageAccuracy(input)).toMatchObject({ status: 'insufficient_evidence', missing: expect.arrayContaining([ + 'default:insufficient_whole_night_uncertainty', 'current:insufficient_whole_night_uncertainty', + '0:default:insufficient_common_pairs', '0:default:ambiguous_observation_support', + ]) }) + }) +}) diff --git a/scripts/algorithm-experiments/accuracy.ts b/scripts/algorithm-experiments/accuracy.ts new file mode 100644 index 000000000..900052d73 --- /dev/null +++ b/scripts/algorithm-experiments/accuracy.ts @@ -0,0 +1,66 @@ +import { configHash } from '../../src/biometrics/algorithms/canonical' +import { packageAccuracyPolicySchema } from './accuracy-policy' +import { readManagedEvidence } from './managed-evidence' + +type Evidence = ReturnType +type AccuracyInput = Pick + +/** Pure assessment of already verified evidence. The authoritative entry point + * below reads the retained graph and fresh ledger state first. */ +export function assessPackageAccuracy(evidence: AccuracyInput) { + const failures: string[] = [], missing: string[] = [] + const rawPolicy = evidence.protocol.accuracyPolicy + const policy = rawPolicy === undefined ? null : packageAccuracyPolicySchema.parse(rawPolicy) + if (!policy) missing.push('no_frozen_accuracy_policy') + if (!evidence.currentEvidence.uniqueHoldoutAttempt) missing.push('no_unique_current_holdout') + if (!evidence.nights.length) missing.push('no_nights') + for (const night of evidence.nights) { + for (const baseline of ['default', 'current'] as const) { + const key = `${night.nightId}:${baseline}` + const comparison = night.comparison.comparisons.find(row => row.candidate === 'candidate' && row.baseline === baseline) + if (!comparison) { + missing.push(`${key}:missing_comparison`) + continue + } + if (comparison.commonSupport.candidate.status === 'incompatible' || comparison.commonSupport.baseline.status === 'incompatible') { + failures.push(`${key}:incompatible_metric`) + continue + } + if (comparison.ambiguousObservationTimestamps.baseline || comparison.ambiguousObservationTimestamps.candidate) missing.push(`${key}:ambiguous_observation_support`) + if (!policy) continue + const candidate = comparison.commonSupport.candidate, previous = comparison.commonSupport.baseline + if (!candidate.metrics || !previous.metrics || candidate.pairedCount < policy.minimumCommonPairsPerNight + || previous.pairedCount < policy.minimumCommonPairsPerNight) missing.push(`${key}:insufficient_common_pairs`) + const own = comparison.ownSupport.candidate + if (own.referenceCoverage === null) missing.push(`${key}:missing_reference_coverage`) + else if (own.referenceCoverage < policy.minimumReferenceCoverage) failures.push(`${key}:reference_coverage_below_minimum`) + const coverageChange = comparison.differences.referenceCoverage + if (coverageChange === null) missing.push(`${key}:missing_coverage_difference`) + else if (coverageChange < -policy.maximumReferenceCoverageLoss) failures.push(`${key}:reference_coverage_loss`) + const baselineCount = comparison.ownSupport.baseline.validObservationCount + if (!baselineCount) missing.push(`${key}:missing_baseline_observations`) + else if (comparison.lostObservationTimestamps / baselineCount > policy.maximumLostTimestampFraction) failures.push(`${key}:observation_support_loss`) + if (comparison.differences.commonMae !== null && comparison.differences.commonMae > policy.maximumNightMaeRegression) failures.push(`${key}:night_mae_regression`) + if (comparison.differences.commonP95AbsoluteError !== null && comparison.differences.commonP95AbsoluteError > policy.maximumNightP95Regression) failures.push(`${key}:night_tail_regression`) + if (candidate.metrics && previous.metrics + && Math.abs(candidate.metrics.signedBias) - Math.abs(previous.metrics.signedBias) > policy.maximumNightAbsoluteBiasIncrease) failures.push(`${key}:night_bias_regression`) + } + } + for (const baseline of ['default', 'current'] as const) { + const report = evidence.uncertainty.find(row => row.baseline === baseline)?.report + if (!report || report.status !== 'estimated' || !report.interval || report.omittedNights > 0 + || report.scoredNights !== evidence.nights.length) missing.push(`${baseline}:insufficient_whole_night_uncertainty`) + else if (policy && (report.estimate === null || report.estimate > -policy.minimumMaeImprovement + || report.interval.upper > -policy.minimumMaeImprovement)) failures.push(`${baseline}:improvement_not_supported`) + } + return { schema: 'biometric-package-accuracy-assessment-v1' as const, + status: failures.length ? 'criteria_not_met' as const : missing.length ? 'insufficient_evidence' as const : 'criteria_met' as const, + policyHash: policy ? configHash(policy) : null, failures, missing, + interpretation: 'reference_agreement_criteria_only_requires_full_package_qualification_and_review' as const } +} + +export function assessManagedAccuracy(...args: Parameters) { + const evidence = readManagedEvidence(...args) + return { trialId: evidence.trialId, reportHash: evidence.reportHash, currentEvidence: evidence.currentEvidence, + assessment: assessPackageAccuracy(evidence), preflight: evidence.preflight } +} diff --git a/scripts/algorithm-experiments/cli.ts b/scripts/algorithm-experiments/cli.ts new file mode 100644 index 000000000..bbd3bbe08 --- /dev/null +++ b/scripts/algorithm-experiments/cli.ts @@ -0,0 +1,87 @@ +import { recoverExperimentStaging } from './staging-recovery' +import { runPackagePreflight } from './preflight' +import { recoverExpiredContainers } from './recovery' +import { pathToFileURL } from 'node:url' +import { z } from 'zod' +import { runManagedFiles } from './managed-files' +import { experimentPolicySchema as policySchema } from './policy' +import { comparePackageOutputs } from './comparison' +import { runPackageFixtures } from './fixtures' +import { packageInputHash } from './input' +import { inspectPythonEnvironment } from './environment' +import { executePackageExperiment } from './container' +import { checkPackageDeterminism } from './determinism' +import { publishJson, readJsonFile, readJsonLines, readPackageProposal } from './files' + +const positive = z.number().int().positive().max(Number.MAX_SAFE_INTEGER) +const usage = 'Usage: preflight PROPOSAL PLAN SUITE POLICY OUTPUT | recover OUTPUT | managed WORKSPACE DESCRIPTOR RECEIPT [STORAGE_POLICY] | compare INPUT OUTPUT | fixtures PROPOSAL SUITE POLICY OUTPUT | lock IMAGE OUTPUT | input-hash FRAMES MAX_BYTES OUTPUT | inspect PROPOSAL OUTPUT | run PROPOSAL REQUEST FRAMES POLICY OUTPUT | determinism PROPOSAL REQUEST FRAMES POLICY REPETITIONS OUTPUT' + +export async function runExperimentCli(args: string[]) { + const [command, ...paths] = args + if (command === 'preflight' && paths.length === 5) { + const proposal = readPackageProposal(paths[0]) + const plan = readJsonFile(paths[1], 16384) + const suite = readJsonFile(paths[2], 32 * 1024 * 1024) + const policy = policySchema.parse(readJsonFile(paths[3], 16384)) + await publishJson(paths[4], await runPackagePreflight({ ...proposal, policy }, plan, suite)) + return + } + if (command === 'recover' && paths.length === 1) { + const containers = await recoverExpiredContainers() + const staging = await recoverExperimentStaging() + await publishJson(paths[0], { schema: 'biometric-experiment-recovery-v1', containers, staging }) + return + } + if (command === 'managed' && (paths.length === 3 || paths.length === 4)) { + await publishJson(paths[2], await runManagedFiles(paths[0], paths[1], paths[3] ? readJsonFile(paths[3], 16384) : undefined)) + return + } + if (command === 'compare' && paths.length === 2) { + await publishJson(paths[1], comparePackageOutputs(readJsonFile(paths[0], 32 * 1024 * 1024))) + return + } + if (command === 'fixtures' && paths.length === 4) { + const proposal = readPackageProposal(paths[0]) + const suite = readJsonFile(paths[1], 32 * 1024 * 1024) + const policy = policySchema.parse(readJsonFile(paths[2], 16384)) + const report = await runPackageFixtures({ manifest: proposal.manifest, files: proposal.files, policy }, suite) + await publishJson(paths[3], report) + return + } + if (command === 'lock' && paths.length === 2) { + await publishJson(paths[1], await inspectPythonEnvironment(paths[0])) + return + } + if (command === 'input-hash' && paths.length === 3) { + const maximumBytes = positive.parse(Number(paths[1])) + const hash = packageInputHash(readJsonLines(paths[0], maximumBytes)) + await publishJson(paths[2], { schema: 'biometric-package-input-v1', hash }) + return + } + if (command === 'inspect' && paths.length === 2) { + const proposal = readPackageProposal(paths[0]) + await publishJson(paths[1], { schema: 'biometric-package-intake-v1', packageHash: proposal.packageHash, + status: proposal.status, manifest: proposal.manifest, totalBytes: proposal.totalBytes }) + return + } + if ((command === 'run' && paths.length === 5) || (command === 'determinism' && paths.length === 6)) { + const policy = policySchema.parse(readJsonFile(paths[3], 16384)) + const proposal = readPackageProposal(paths[0]) + const request = readJsonFile(paths[1], 65536) + const frames = () => readJsonLines(paths[2], policy.maximumInputBytes) + const options = { manifest: proposal.manifest, files: proposal.files, request, policy } + const report = command === 'run' + ? { schema: 'biometric-package-experiment-v1', status: 'experiment_only', ...await executePackageExperiment({ ...options, frames: frames() }) } + : await checkPackageDeterminism({ ...options, frames }, Number(paths[4])) + await publishJson(paths[paths.length - 1], report) + return + } + throw new Error(usage) +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + runExperimentCli(process.argv.slice(2)).catch((error: unknown) => { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) + process.exitCode = 1 + }) +} diff --git a/scripts/algorithm-experiments/comparison.test.ts b/scripts/algorithm-experiments/comparison.test.ts new file mode 100644 index 000000000..a645e6f13 --- /dev/null +++ b/scripts/algorithm-experiments/comparison.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from 'vitest' +import { configHash } from '../../src/biometrics/algorithms/canonical' +import { packageFixture } from '../../src/biometrics/packages/tests/fixture' +import { comparePackageOutputs } from './comparison' +const start = 1791673200000 +function fixture() { + const outputs = (['default', 'current', 'candidate'] as const).map((label, index) => { + const { manifest } = packageFixture() + const declared = { ...manifest, metrics: manifest.metrics.map(metric => ({ ...metric, reference: { metric: 'heart_rate', windowMs: null as number | null } })) } + const packageHash = configHash(declared) + const request = { schema: 'biometric-package-run-v1', packageHash, inputHash: 'a'.repeat(64), hardwareId: 'pod4', sampleRateHz: 500, side: 'left', seed: 1, config: { offset: index } } + const runHash = configHash(request) + return { label, manifest: declared, request, result: { packageHash, runHash, + frozenInput: { schema: 'biometric-package-input-v1', hash: request.inputHash, bytes: 1000, frameCount: 1 }, + results: [{ schema: 'biometric-metric-frame-v1', runHash, frameHash: 'b'.repeat(64), sequence: 0, state: {}, + readings: [0, 60000].map(offset => ({ timestampMs: start + offset, side: 'left', metric: 'heart_rate', value: 64 - index * 2 as number | null, quality: 1, flags: [] })) }] } } + }) + return { schema: 'biometric-package-comparison-v1', metricId: 'heart_rate', startMs: start, endMs: start + 120000, minimumQuality: 0.8, + referenceImport: { side: 'left', subjectId: 'fixture', source: 'synthetic', rows: [0, 60000].map((offset, index) => ({ recordKey: String(index), metric: 'heart_rate', value: 60, unit: 'bpm', timestampMs: start + offset })) }, + alignment: { version: 'reference-alignment-v1', pointToleranceMs: 0, clockOffsetMs: 0, offsetSource: 'fixed' }, outputs } +} +function rebind(output: ReturnType['outputs'][number]) { + output.request.packageHash = configHash(output.manifest) + output.result.packageHash = output.request.packageHash + output.result.runHash = configHash(output.request) + output.result.results.forEach((frame) => { + frame.runHash = output.result.runHash + }) +} +describe('package reference agreement', () => { + it('compares both baselines with shared pairing without granting promotion', () => { + const result = comparePackageOutputs(fixture()) + expect(result.comparisons.filter(row => row.candidate === 'candidate').map(row => row.differences.commonMae)).toEqual([-4, -2]) + expect(result.interpretation).toContain('not_independent_holdout_or_promotion') + }) + it('exposes dropped and low-quality values as lost support', () => { + for (const lowQuality of [false, true]) { + const input = fixture() + const row = input.outputs[2].result.results[0].readings[0] + if (lowQuality) row.quality = 0.1 + else row.value = null + const comparison = comparePackageOutputs(input).comparisons.find(row => row.candidate === 'candidate' && row.baseline === 'default') + expect(comparison).toMatchObject({ coverageReduced: true, lostObservationTimestamps: 1, commonObservationCount: 1 }) + } + }) + it('rejects substituted run, frame, input, side and duplicate baseline identities', () => { + for (const change of [ + (input: ReturnType) => { input.outputs[2].result.runHash = 'c'.repeat(64) }, + (input: ReturnType) => { input.outputs[2].result.results[0].frameHash = 'c'.repeat(64) }, + (input: ReturnType) => { input.outputs[2].request.side = 'right' }, + (input: ReturnType) => { input.outputs[2].request.inputHash = 'c'.repeat(64) }, + (input: ReturnType) => { input.outputs[2].label = 'default' }, + ]) { + const input = fixture() + change(input) + expect(() => comparePackageOutputs(input)).toThrow() + } + }) + it('rejects incompatible package definitions before scoring', () => { + const input = fixture() + input.outputs[2].manifest.metrics[0].reference = { metric: 'hrv_rmssd', windowMs: 300000 } + input.outputs[2].manifest.metrics[0].unit = 'ms' + rebind(input.outputs[2]) + expect(() => comparePackageOutputs(input)).toThrow('Incompatible package metric') + }) + it('does not score external RMSSD against the window IBI variability index', () => { + const input = fixture() + for (const output of input.outputs) { + output.manifest.metrics[0].reference = { metric: 'window_ibi_variability_v1', windowMs: 300000 } + output.manifest.metrics[0].unit = 'ms' + rebind(output) + } + input.referenceImport.rows = input.referenceImport.rows.map(row => ({ ...row, metric: 'hrv_rmssd', unit: 'ms', windowMs: 300000 })) + const report = comparePackageOutputs(input) + expect(report.compatibilityExclusions).toEqual({ incompatible_metric_definition: 2 }) + expect(report.comparisons.every(row => row.commonSupport.candidate.status === 'incompatible' && row.commonSupport.candidate.metrics === null)).toBe(true) + }) +}) diff --git a/scripts/algorithm-experiments/comparison.ts b/scripts/algorithm-experiments/comparison.ts new file mode 100644 index 000000000..f1c4a5cf4 --- /dev/null +++ b/scripts/algorithm-experiments/comparison.ts @@ -0,0 +1,76 @@ +import { z } from 'zod' +import { configHash, dataHash } from '../../src/biometrics/algorithms/canonical' +import { compareNumericConfigurations } from '../../src/biometrics/backtest/numeric-comparison' +import { algorithmPackageManifestSchema } from '../../src/biometrics/packages/manifest' +import { metricFrameSchema, packageRunRequestSchema } from '../../src/biometrics/packages/protocol' +import { referenceCompatibility } from '../../src/biometrics/references/alignment' +import { normalizeReferenceImport } from '../../src/biometrics/references/normalize-core' +import { referenceImportSchema, timestampMsSchema } from '../../src/biometrics/references/contracts' + +const hash = z.string().regex(/^[a-f0-9]{64}$/) +export const packageComparisonInputSchema = z.object({ + schema: z.literal('biometric-package-comparison-v1'), metricId: z.string().min(1).max(100), + startMs: timestampMsSchema, endMs: timestampMsSchema, minimumQuality: z.number().min(0).max(1), + referenceImport: referenceImportSchema, + alignment: z.object({ version: z.literal('reference-alignment-v1'), pointToleranceMs: z.number().int().min(0).max(300000), + clockOffsetMs: z.number().int().min(-3600000).max(3600000), offsetSource: z.enum(['fixed', 'training']) }).strict(), + outputs: z.array(z.object({ label: z.enum(['default', 'current', 'candidate']), manifest: algorithmPackageManifestSchema, + request: packageRunRequestSchema, + result: z.object({ packageHash: hash, runHash: hash, + frozenInput: z.object({ schema: z.literal('biometric-package-input-v1'), hash, bytes: z.number().int().positive(), frameCount: z.number().int().positive() }).strict(), + results: z.array(metricFrameSchema).max(100000), + }), + }).strict()).length(3), +}).strict().refine(value => value.endMs > value.startMs && value.endMs - value.startMs <= 86400000, 'Comparison requires a bounded whole-night interval') + +/** Descriptive scoring of host-produced experiment reports. This does not + * attest execution provenance or independent holdout use; promotion must use + * the authoritative experiment store and reviewed qualification pipeline. */ +export function comparePackageOutputs(raw: unknown) { + const input = packageComparisonInputSchema.parse(raw) + if (new Set(input.outputs.map(output => output.label)).size !== 3) throw new Error('Default, current and candidate outputs are required') + const reference = normalizeReferenceImport(input.referenceImport) + let commonInput: string | undefined + let frameIdentity: string | undefined + let contract: { metric: 'heart_rate' | 'breathing_rate' | 'window_ibi_variability_v1' | 'hrv_rmssd' | 'hrv_sdnn', windowMs: number | null } | undefined + const outputs = input.outputs.map((output) => { + const { manifest, request, result } = output + const packageHash = configHash(manifest) + if (packageHash !== request.packageHash || packageHash !== result.packageHash || configHash(request) !== result.runHash + || result.frozenInput.hash !== request.inputHash || result.frozenInput.frameCount !== result.results.length) throw new Error('Package comparison identity mismatch') + if (request.side !== input.referenceImport.side) throw new Error('Reference side mismatch') + const identity = configHash({ inputHash: request.inputHash, side: request.side, hardwareId: request.hardwareId, sampleRateHz: request.sampleRateHz }) + if (commonInput !== undefined && commonInput !== identity) throw new Error('Packages did not process the same input and side') + commonInput = identity + const frameHash = dataHash(result.results.map(frame => frame.frameHash)) + if (frameIdentity !== undefined && frameIdentity !== frameHash) throw new Error('Packages received different frames') + frameIdentity = frameHash + const metric = manifest.metrics.find(metric => metric.id === input.metricId) + if (!metric?.reference) throw new Error('Package lacks a comparable metric contract') + if (contract && configHash(contract) !== configHash(metric.reference)) throw new Error('Incompatible package metric definitions or windows') + contract = metric.reference + const observations = [] + for (const [index, frame] of result.results.entries()) { + if (frame.runHash !== result.runHash || frame.sequence !== index) throw new Error('Package output identity mismatch') + for (const reading of frame.readings) { + if (reading.side !== request.side) throw new Error('Package output side mismatch') + if (reading.metric !== input.metricId) continue + if (reading.value !== null && (reading.value < metric.minimum || reading.value > metric.maximum)) throw new Error('Package metric outside declared range') + if (reading.timestampMs >= input.startMs && reading.timestampMs < input.endMs) observations.push({ timestampMs: reading.timestampMs, value: reading.quality >= input.minimumQuality ? reading.value : null }) + } + } + return { label: output.label, configHash: result.runHash, observations } + }) + if (!contract) throw new Error('No comparable metric contract') + const selected = reference.measurements.filter(row => row.startMs >= input.startMs && row.startMs < input.endMs && row.endMs <= input.endMs) + const compatibilityExclusions: Record = {} + for (const row of selected) { + const reason = referenceCompatibility(contract.metric, contract.windowMs, row) + if (reason) compatibilityExclusions[reason] = (compatibilityExclusions[reason] ?? 0) + 1 + } + const comparisons = compareNumericConfigurations(contract.metric, contract.windowMs, outputs, selected, input.alignment) + return { schema: 'biometric-package-comparison-report-v1' as const, inputHash: dataHash(input), referenceHash: dataHash(reference.measurements), + metricId: input.metricId, contract, alignment: input.alignment, startMs: input.startMs, endMs: input.endMs, + minimumQuality: input.minimumQuality, excludedBoundaryReferences: reference.measurements.length - selected.length, compatibilityExclusions, + referenceSummary: reference.summary, comparisons, interpretation: 'descriptive_reference_agreement_not_independent_holdout_or_promotion_evidence' as const } +} diff --git a/scripts/algorithm-experiments/container.test.ts b/scripts/algorithm-experiments/container.test.ts new file mode 100644 index 000000000..fc75db6f3 --- /dev/null +++ b/scripts/algorithm-experiments/container.test.ts @@ -0,0 +1,221 @@ +import { fixtureResourceTestPolicy } from './resource-test-policy' +// @vitest-environment node +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { dirname, join } from 'node:path' +import { statSync } from 'node:fs' +import * as frozenInput from './input' +import * as resourceReader from './resources' +import { tmpdir } from 'node:os' +import { runPackageFixtures } from './fixtures' +import { runExperimentCli } from './cli' +import { createHash } from 'node:crypto' +import { beforeAll, describe, expect, it, vi } from 'vitest' +import { canonicalJson, configHash } from '../../src/biometrics/algorithms/canonical' +import { packageFixture } from '../../src/biometrics/packages/tests/fixture' +import { inspectPythonEnvironment } from './environment' +import { checkPackageDeterminism } from './determinism' +import { packageInputHash } from './input' +import { executePackageExperiment } from './container' + +const image = process.env.BIOMETRIC_EXPERIMENT_TEST_IMAGE +let lockBytes: Buffer +beforeAll(async () => { + if (image) lockBytes = Buffer.from(JSON.stringify(await inspectPythonEnvironment(image))) +}) +const program = `import json, sys +init = json.loads(sys.stdin.readline()) +for line in sys.stdin: + message = json.loads(line) + frame = message['frame'] + print(json.dumps({'schema': 'biometric-metric-frame-v1', 'runHash': init['runHash'], 'frameHash': message['frameHash'], 'sequence': frame['sequence'], 'readings': [], 'state': {}}), flush=True) +` +function options(code: string) { + const { manifest, files } = packageFixture() + files.set('engine.py', Buffer.from(code)) + files.set(manifest.dependencyLock, lockBytes) + manifest.artifacts = [...files].map(([path, bytes]) => ({ path, bytes: bytes.length, sha256: createHash('sha256').update(bytes).digest('hex') })) + manifest.resources = { cpuMs: 5000, wallMs: 5000, memoryBytes: 64 * 1024 * 1024, processes: 1, outputBytes: 16384 } + const frames = [{ schema: 'biometric-signal-frame-v1', sequence: 0, timestampMs: 1000, samples: { left: [1, 2, 3], right: null }, context: { occupied: null, pumpActive: null } }] + frames.push({ ...frames[0], sequence: 1, timestampMs: 1006 }) + return { manifest, files, + request: { schema: 'biometric-package-run-v1', packageHash: configHash(manifest), inputHash: packageInputHash(frames), hardwareId: 'pod4', sampleRateHz: 500, side: 'left', seed: 123, config: {} }, + frames, + policy: { image: image ?? '', maximumWallMs: 10000, maximumMemoryBytes: 128 * 1024 * 1024, maximumProcesses: 4, maximumOutputBytes: 32768, maximumInputBytes: 32768 } } +} + +describe.skipIf(!image)('real external Docker experiments', () => { + it('runs the frame protocol with read-only input, no network, no extra processes or host environment', async () => { + const probes = `import os, socket +assert os.getuid() == 65534 +assert 'BIOMETRIC_EXPERIMENT_TEST_IMAGE' not in os.environ +assert not os.path.exists('/var/run/docker.sock') +assert not os.path.exists('/root/.ssh') +for path in ['/package/engine.py', '/tmp/write-attempt']: + try: + open(path, 'w') + except OSError: + pass + else: + raise AssertionError('Writable filesystem') +try: + os.fork() +except OSError: + pass +else: + raise AssertionError('Fork succeeded') +s = socket.socket() +s.settimeout(0.2) +try: + s.connect(('1.1.1.1', 443)) +except OSError: + pass +else: + raise AssertionError('Network available') +s.close() +` + const input = options(probes + program) + const execution = executePackageExperiment(input) + input.policy.image = 'sha256:' + 'b'.repeat(64) + const result = await execution + expect(result.execution.environment.image).toBe(image) + expect(result.execution.measurements.samples).toBeGreaterThan(0) + expect(result.execution.measurements.maximumObservedCpuMs).toBeGreaterThan(0) + expect(result.execution.measurements.peakObservedMemoryBytes).toBeGreaterThan(0) + expect(result.execution.measurements).toMatchObject({ source: 'host_docker_engine_stats', exactPeakMemory: false, exactLifetimeCpu: false }) + expect(result.results).toHaveLength(2) + expect(result.execution.isolation).toBe('docker-restricted-v1') + }, 20000) + it('keeps the host staging parent private while the unprivileged container reads its package mount', async () => { + let parent: string | undefined + let modes: number[] | undefined + const freeze = frozenInput.freezePackageInput + const readResources = resourceReader.dockerResourceReader + const capture = vi.spyOn(frozenInput, 'freezePackageInput').mockImplementation(async (path, frames, maximumBytes) => { + parent = dirname(path) + return freeze(path, frames, maximumBytes) + }) + const observe = vi.spyOn(resourceReader, 'dockerResourceReader').mockImplementation(async (id) => { + if (!parent) throw new Error('Missing staged input') + // This runs after Docker create, when all host permissions are final. + modes = [parent, join(parent, 'input.jsonl'), join(parent, 'package')].map(path => statSync(path).mode & 0o777) + return readResources(id) + }) + try { + expect((await executePackageExperiment(options(program))).results).toHaveLength(2) + expect(observe).toHaveBeenCalledOnce() + expect(modes).toEqual([0o700, 0o600, 0o755]) + const stagedParent = parent + if (!stagedParent) throw new Error('Missing staging') + expect(() => statSync(stagedParent)).toThrow() + } + finally { + capture.mockRestore() + observe.mockRestore() + } + }, 20000) + it('runs a complete file-based CLI experiment and publishes a reviewable report', async () => { + const root = await mkdtemp(join(tmpdir(), 'package-cli-test-')) + try { + const input = options(program) + const proposal = { schema: 'biometric-package-proposal-v1', manifest: input.manifest, + files: [...input.files].map(([path, bytes]) => ({ path, base64: bytes.toString('base64') })) } + for (const [name, value] of [['proposal', proposal], ['request', input.request], ['policy', input.policy]] as const) { + await writeFile(join(root, name), JSON.stringify(value)) + } + await writeFile(join(root, 'frames'), input.frames.map(frame => JSON.stringify(frame)).join('\n') + '\n') + await runExperimentCli(['run', ...['proposal', 'request', 'frames', 'policy', 'report'].map(name => join(root, name))]) + const report = JSON.parse(await readFile(join(root, 'report'), 'utf8')) + expect(report).toMatchObject({ schema: 'biometric-package-experiment-v1', status: 'experiment_only', packageHash: input.request.packageHash, + frozenInput: { hash: input.request.inputHash, frameCount: 2 } }) + expect(report.results).toHaveLength(2) + } + finally { await rm(root, { recursive: true, force: true }) } + }, 20000) + it('checks numeric coverage and catches false vitals during an empty-bed fixture', async () => { + const reading = '[{\'timestampMs\': frame[\'timestampMs\'], \'side\': \'left\', \'metric\': \'heart_rate\', \'value\': 60, \'quality\': 1, \'flags\': []}]' + const makeSuite = (input: ReturnType) => ({ schema: 'biometric-package-fixtures-v1', resourcePolicy: fixtureResourceTestPolicy, fixtures: [true, false].map((occupied) => { + const frames = input.frames.map(frame => ({ ...frame, context: { ...frame.context, occupied } })) + return { id: occupied ? 'clean' : 'empty', category: occupied ? 'clean_signal' : 'empty_bed', + request: { ...input.request, inputHash: packageInputHash(frames) }, frames, + expectations: [{ metric: 'heart_rate', startMs: 1000, endMs: 1012, ...(occupied + ? { kind: 'numeric', minimum: 59, maximum: 61, minimumReadings: 2, minimumQuality: 0.8 } + : { kind: 'abstain' }) }] } + }) }) + const candidate = options(program.replace('\'readings\': []', `'readings': ${reading} if frame['context']['occupied'] else []`)) + const suite = makeSuite(candidate) + const running = runPackageFixtures(candidate, suite) + suite.fixtures[0].frames[0].samples.left[0] = 999 + const report = await running + expect(report.suite.fixtures[0].frames[0].samples.left?.[0]).toBe(1) + expect(report.status).toBe('incomplete') + expect(report.results.every(result => result.assessment.passed)).toBe(true) + expect(report.missingCategories).toContain('pump') + const strictBudget = makeSuite(candidate) + strictBudget.resourcePolicy = { ...fixtureResourceTestPolicy, maximumWallMs: 1 } + const overBudget = await runPackageFixtures(candidate, strictBudget) + expect(overBudget.results.every(result => result.assessment.passed)).toBe(true) + expect(overBudget.results.every(result => result.resourceAssessment?.status === 'failed')).toBe(true) + expect(overBudget.status).toBe('failed') + const bad = options(program.replace('\'readings\': []', `'readings': ${reading}`)) + const regression = await runPackageFixtures(bad, makeSuite(bad)) + expect(regression.status).toBe('failed') + expect(regression.results[1].assessment.failures).toContain('failed_abstention:heart_rate:1000') + }, 30000) + it('repeats in fresh containers and detects changing state even with identical readings', async () => { + const input = options(program) + const stable = await checkPackageDeterminism({ ...input, frames: () => input.frames }) + expect(stable.deterministic).toBe(true) + expect(stable.runs).toHaveLength(2) + const unstable = options(program.replace('import json, sys', 'import json, sys, secrets').replace('\'state\': {}', '\'state\': {\'nonce\': secrets.token_hex(16)}')) + expect((await checkPackageDeterminism({ ...unstable, frames: () => unstable.frames })).deterministic).toBe(false) + }, 20000) + it('rejects a locked Python version that differs from the actual image before running code', async () => { + const input = options('raise AssertionError("candidate must not run")') + const wrong = JSON.parse(lockBytes.toString()) + wrong.pythonVersion = '0.0.0' + const bytes = Buffer.from(JSON.stringify(wrong)) + input.files.set(input.manifest.dependencyLock, bytes) + input.manifest.artifacts = input.manifest.artifacts.map(artifact => artifact.path === input.manifest.dependencyLock + ? { ...artifact, bytes: bytes.length, sha256: createHash('sha256').update(bytes).digest('hex') } + : artifact) + input.request.packageHash = configHash(input.manifest) + await expect(executePackageExperiment(input)).rejects.toThrow('differs from dependency lock') + }, 20000) + it('streams an input whose frozen size exactly meets the operator limit', async () => { + const input = options(program) + input.policy.maximumInputBytes = input.frames.reduce((bytes, frame) => bytes + Buffer.byteLength(canonicalJson(frame) + '\n'), 0) + const result = await executePackageExperiment(input) + expect(result.frozenInput.bytes).toBe(input.policy.maximumInputBytes) + expect(result.execution.inputBytes).toBeGreaterThan(input.policy.maximumInputBytes) + expect(result.results).toHaveLength(2) + }, 20000) + it('terminates an output flood without buffering it', async () => { + await expect(executePackageExperiment(options('import sys\nsys.stdout.write(\'x\' * 1000000)\nsys.stdout.flush()\n'))).rejects.toThrow('Output transport limit') + }, 20000) + it('terminates a package that never responds', async () => { + const input = options('while True: pass\n') + input.manifest.resources.cpuMs = 1000 + input.request.packageHash = configHash(input.manifest) + await expect(executePackageExperiment(input)).rejects.toThrow('deadline') + }, 20000) + it('stops a cancelled experiment and removes its container', async () => { + const controller = new AbortController() + const timer = setTimeout(() => controller.abort(), 1000) + try { + await expect(executePackageExperiment({ ...options('import time\ntime.sleep(60)\n'), signal: controller.signal })).rejects.toThrow(/abort/i) + } + finally { clearTimeout(timer) } + }, 20000) + it('terminates memory exhaustion', async () => { + await expect(executePackageExperiment(options('data = bytearray(256 * 1024 * 1024)\n'))).rejects.toThrow() + }, 20000) + it('rejects invalid output identities and operator limit excess', async () => { + await expect(executePackageExperiment(options(program.replace('init[\'runHash\']', '\'b\' * 64')))).rejects.toThrow('identity') + const substituted = options(program) + substituted.frames[0].samples.left[0]++ + await expect(executePackageExperiment(substituted)).rejects.toThrow('input identity mismatch') + const input = options(program) + input.policy.maximumMemoryBytes = 1 + await expect(executePackageExperiment(input)).rejects.toThrow('operator resource policy') + }, 20000) +}) diff --git a/scripts/algorithm-experiments/container.ts b/scripts/algorithm-experiments/container.ts new file mode 100644 index 000000000..45d394291 --- /dev/null +++ b/scripts/algorithm-experiments/container.ts @@ -0,0 +1,200 @@ +import { recordExperimentStaging } from './staging-recovery' +import { recoveryLabelArguments } from './recovery' +import { spawn, execFile } from 'node:child_process' +import { existsSync } from 'node:fs' +import { randomUUID } from 'node:crypto' +import { mkdtemp, mkdir, writeFile, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' +import { promisify } from 'node:util' +import { performance } from 'node:perf_hooks' +import { verifyAlgorithmPackage } from '../../src/biometrics/packages/manifest' +import { PackageFrameValidator } from '../../src/biometrics/packages/protocol' +import { dockerResourceReader, startResourceMonitor } from './resources' +import { inspectPythonEnvironment, matchPythonEnvironment, parsePythonEnvironmentLock } from './environment' +import { freezePackageInput, packageTransportCeiling, readFrozenPackageInput } from './input' +import { dataHash } from '../../src/biometrics/algorithms/canonical' + +const exec = promisify(execFile) +async function removeContainer(name: string) { + try { + await exec('docker', ['rm', '-f', name], { timeout: 10000, maxBuffer: 65536 }) + } + catch (error) { + if (!(error instanceof Error) || !error.message.includes('No such container:')) throw error + } +} +/** Operator-owned policy, never accepted from a candidate or pod settings. */ +export interface ExperimentPolicy { + image: string + maximumWallMs: number + maximumMemoryBytes: number + maximumProcesses: number + maximumOutputBytes: number + maximumInputBytes: number +} + +/** Off-pod CLI only. Do not import from the web application or run on a pod. + * The image must be a reviewed Python runtime with locked dependencies already + * installed; dependency installation and network access are not permitted here. */ +export async function executePackageExperiment(options: { + manifest: unknown + files: ReadonlyMap + request: unknown + frames: Iterable + policy: ExperimentPolicy + signal?: AbortSignal +}) { + options.signal?.throwIfAborted() + if (existsSync('/persistent/deviceinfo/dac.sock') || existsSync('/deviceinfo/dac.sock')) throw new Error('Algorithm experiments must run off-pod') + const policy = structuredClone(options.policy) + if (!/^sha256:[a-f0-9]{64}$/.test(policy.image)) throw new Error('A local immutable image ID is required') + for (const value of [policy.maximumWallMs, policy.maximumMemoryBytes, policy.maximumProcesses, policy.maximumOutputBytes, policy.maximumInputBytes]) { + if (!Number.isSafeInteger(value) || value <= 0) throw new Error('Invalid experiment policy') + } + verifyAlgorithmPackage(options.manifest, options.files) + const files = new Map([...options.files].map(([path, bytes]) => [path, Buffer.from(bytes)])) + const verified = verifyAlgorithmPackage(options.manifest, files) + const validator = new PackageFrameValidator(verified.manifest, options.request) + const resources = verified.manifest.resources + if (resources.wallMs > policy.maximumWallMs || resources.memoryBytes > policy.maximumMemoryBytes + || resources.processes > policy.maximumProcesses || resources.outputBytes > policy.maximumOutputBytes) throw new Error('Package exceeds operator resource policy') + if (resources.wallMs > 3600000) throw new Error('Experiment deadline exceeds one hour') + const info = JSON.parse((await exec('docker', ['info', '--format', '{{json .}}'], { timeout: 10000, maxBuffer: 1024 * 1024 })).stdout) + if (info.OSType !== 'linux' || !info.MemoryLimit || !info.PidsLimit || !info.CpuCfsQuota || !info.SecurityOptions?.some((item: string) => item.startsWith('name=seccomp'))) throw new Error('Docker lacks required isolation controls') + const lockBytes = files.get(verified.manifest.dependencyLock) + if (!lockBytes) throw new Error('Missing dependency lock') + parsePythonEnvironmentLock(lockBytes, policy.image) + const environment = matchPythonEnvironment(lockBytes, await inspectPythonEnvironment(policy.image), policy.image) + const root = await mkdtemp(join(tmpdir(), 'sleepypod-package-')) + const name = `sleepypod-experiment-${randomUUID()}` + let created = false + try { + await recordExperimentStaging(root, name, Math.min(resources.wallMs, resources.cpuMs)) + const inputPath = join(root, 'input.jsonl') + const frozenInput = await freezePackageInput(inputPath, options.frames, policy.maximumInputBytes) + if (frozenInput.hash !== validator.request.inputHash) throw new Error('Experiment input identity mismatch') + const initialize = { type: 'initialize', request: validator.request, runHash: validator.runHash } + const transportLimit = packageTransportCeiling(frozenInput, Buffer.byteLength(JSON.stringify(initialize) + '\n')) + const packagePath = join(root, 'package') + await mkdir(packagePath, { mode: 0o755 }) + // Docker mounts only packagePath; its container UID needs access inside that + // mount, not traversal of the host parent. Keep the mkdtemp root at 0700 so + // host users cannot list package source or the private frozen sensor input. + for (const artifact of verified.manifest.artifacts) { + const path = join(packagePath, artifact.path) + await mkdir(dirname(path), { recursive: true, mode: 0o755 }) + // Copy to privately owned staging so caller mutation cannot alter the mount. + await writeFile(path, files.get(artifact.path) as Uint8Array, { flag: 'wx', mode: 0o444 }) + } + // Bound aggregate CPU conservatively: one CPU and a wall deadline no longer + // than the CPU budget. Idle time consumes that budget too. This is not a + // measured CPU receipt and cannot qualify a package's runtime performance. + const deadlineMs = Math.min(resources.wallMs, resources.cpuMs) + options.signal?.throwIfAborted() + created = true + const creation = await exec('docker', ['create', '--name', name, ...recoveryLabelArguments('experiment', deadlineMs), '--network', 'none', '--ipc', 'none', '--read-only', + '--cap-drop', 'ALL', '--security-opt', 'no-new-privileges=true', '--user', '65534:65534', + '--pids-limit', String(resources.processes), '--memory', String(resources.memoryBytes), + '--memory-swap', String(resources.memoryBytes), '--cpus', '1', '--ulimit', 'core=0', + '--log-driver', 'none', '--restart', 'no', '--no-healthcheck', '-i', + '--mount', `type=bind,source=${packagePath},target=/package,readonly`, '--workdir', '/package', + '--entrypoint', 'python', policy.image, '-I', '-B', `/package/${verified.manifest.entrypoint}`], + { timeout: 10000, maxBuffer: 65536 }) + const resourceReader = await dockerResourceReader(creation.stdout.trim()).catch(() => null) + const started = performance.now() + const resourceMonitor = startResourceMonitor(resourceReader, started) + const child = spawn('docker', ['start', '-ai', name], { stdio: ['pipe', 'pipe', 'pipe'] }) + let failure: Error | null = null + let pending = Buffer.alloc(0), outputBytes = 0, inputBytes = 0 + let waiting: { resolve: (line: string) => void, reject: (error: Error) => void } | null = null + let stopPromise: Promise | null = null + const fail = (error: Error) => { + if (failure) return + failure = error + waiting?.reject(error) + waiting = null + child.stdin.destroy() + stopPromise = exec('docker', ['kill', name], { timeout: 10000, maxBuffer: 65536 }).catch(() => undefined) + child.kill('SIGKILL') + } + const abort = () => fail(new Error('Experiment aborted')) + options.signal?.addEventListener('abort', abort, { once: true }) + if (options.signal?.aborted) abort() + child.stdin.on('error', error => fail(error)) + child.on('error', error => fail(error)) + child.stdout.on('data', (chunk: Buffer) => { + if (failure) return + outputBytes += chunk.length + if (outputBytes > resources.outputBytes) return fail(new Error('Output transport limit exceeded')) + pending = Buffer.concat([pending, chunk]) + let newline: number + while ((newline = pending.indexOf(10)) >= 0) { + if (!waiting) return fail(new Error('Unexpected package output')) + const line = pending.subarray(0, newline).toString('utf8') + pending = pending.subarray(newline + 1) + const receiver = waiting + waiting = null + receiver.resolve(line) + } + }) + child.stderr.on('data', (chunk: Buffer) => { + outputBytes += chunk.length + if (outputBytes > resources.outputBytes) fail(new Error('Output transport limit exceeded')) + }) + const completion = new Promise((resolve, reject) => { + child.once('close', (code) => { + if (waiting) fail(new Error('Package exited before responding')) + if (failure) reject(failure) + else if (code !== 0 || pending.length) reject(new Error(`Package exited with invalid output (exit ${code})`)) + else resolve() + }) + }) + // Observe immediately, even if a frame iterator is still pending. + void completion.catch(() => undefined) + const timer = setTimeout(() => fail(new Error('Package deadline exceeded')), deadlineMs) + const send = async (value: unknown) => { + if (failure) throw failure + const bytes = JSON.stringify(value) + '\n' + inputBytes += Buffer.byteLength(bytes) + if (inputBytes > transportLimit) throw new Error('Input transport limit exceeded') + await new Promise((resolve, reject) => child.stdin.write(bytes, error => error ? reject(error) : resolve())) + } + const results = [] + try { + await send(initialize) + for await (const input of readFrozenPackageInput(inputPath)) { + const { frame } = validator.validateFrame(input) + const response = new Promise((resolve, reject) => { + waiting = { resolve, reject } + }) + void response.catch(() => undefined) + await send({ type: 'frame', frame, frameHash: dataHash(frame) }) + const result = validator.validateStep(frame, JSON.parse(await response)).result + results.push(result) + } + await resourceMonitor.sample() + child.stdin.end() + await completion + const measurements = await resourceMonitor.stop() + return { packageHash: verified.packageHash, runHash: validator.runHash, frozenInput, results, + execution: { limits: resources, measurements, environment, image: policy.image, wallMs: performance.now() - started, outputBytes, inputBytes, isolation: 'docker-restricted-v1' as const } } + } + catch (error) { + fail(error instanceof Error ? error : new Error(String(error))) + await completion.catch(() => undefined) + throw error + } + finally { + options.signal?.removeEventListener('abort', abort) + clearTimeout(timer) + await resourceMonitor.stop() + if (stopPromise) await stopPromise + } + } + finally { + // Keep ownership metadata and files if Docker cleanup is unconfirmed. + if (created) await removeContainer(name) + await rm(root, { recursive: true, force: true }) + } +} diff --git a/scripts/algorithm-experiments/determinism.ts b/scripts/algorithm-experiments/determinism.ts new file mode 100644 index 000000000..85b6ff0c2 --- /dev/null +++ b/scripts/algorithm-experiments/determinism.ts @@ -0,0 +1,28 @@ +import { createHash } from 'node:crypto' +import { canonicalJson } from '../../src/biometrics/algorithms/canonical' +import { verifyAlgorithmPackage } from '../../src/biometrics/packages/manifest' +import { executePackageExperiment } from './container' + +type ExperimentOptions = Parameters[0] +/** Independent containers provide a fresh process and filesystem for every run. + * The trusted caller supplies a fresh iterator over the same retained inputs. */ +export async function checkPackageDeterminism(options: Omit & { + frames: () => Iterable +}, repetitions = 2) { + if (!Number.isInteger(repetitions) || repetitions < 2 || repetitions > 5) throw new Error('Determinism requires 2–5 independent runs') + verifyAlgorithmPackage(options.manifest, options.files) + const stable = { manifest: structuredClone(options.manifest), request: structuredClone(options.request), + policy: structuredClone(options.policy), signal: options.signal, files: new Map([...options.files].map(([path, bytes]) => [path, Buffer.from(bytes)])) } + type Result = Awaited> + const runs: (Pick & { input: Result['frozenInput'], outputHash: string, responseCount: number })[] = [] + for (let index = 0; index < repetitions; index++) { + const result = await executePackageExperiment({ ...stable, frames: options.frames() }) + const hash = createHash('sha256').update('biometric-package-output-v1\n') + for (const frame of result.results) hash.update(canonicalJson(frame) + '\n') + runs.push({ packageHash: result.packageHash, runHash: result.runHash, input: result.frozenInput, + outputHash: hash.digest('hex'), responseCount: result.results.length, execution: result.execution }) + } + const deterministic = runs.every(run => run.runHash === runs[0].runHash && run.outputHash === runs[0].outputHash) + return { schema: 'biometric-package-determinism-v1' as const, deterministic, runs, + qualification: 'requires_resource_noise_holdout_compatibility_checks_and_review' as const } +} diff --git a/scripts/algorithm-experiments/environment.test.ts b/scripts/algorithm-experiments/environment.test.ts new file mode 100644 index 000000000..c258e3464 --- /dev/null +++ b/scripts/algorithm-experiments/environment.test.ts @@ -0,0 +1,26 @@ +// @vitest-environment node +import { describe, expect, it } from 'vitest' +import { matchPythonEnvironment, parsePythonEnvironmentLock, pythonEnvironmentLockSchema } from './environment' +const image = 'sha256:' + 'a'.repeat(64) +const lock = { schema: 'biometric-python-environment-v1', image, implementation: 'cpython', pythonVersion: '3.12.13', system: 'Linux', machine: 'aarch64', distributions: [{ name: 'numpy', version: '2.0.0' }, { name: 'scipy', version: '1.0.0' }] } +const bytes = Buffer.from(JSON.stringify(lock)) +describe('locked Python environment', () => { + it('binds the image, interpreter, architecture and complete distribution inventory', () => { + expect(matchPythonEnvironment(bytes, lock, image)).toMatchObject({ image, schema: 'biometric-python-environment-verification-v1' }) + for (const actual of [{ ...lock, image: 'sha256:' + 'b'.repeat(64) }, { ...lock, machine: 'x86_64' }, { ...lock, pythonVersion: '3.13.0' }, + { ...lock, distributions: lock.distributions.slice(0, 1) }, { ...lock, distributions: [{ name: 'numpy', version: '2.1.0' }, lock.distributions[1]] }]) { + expect(() => matchPythonEnvironment(bytes, actual, image)).toThrow('differs') + } + }) + it('rejects ambiguous names, ordering, duplicates and extra lock fields', () => { + for (const invalid of [{ ...lock, distributions: [...lock.distributions].reverse() }, { ...lock, distributions: [lock.distributions[0], lock.distributions[0]] }, + { ...lock, distributions: [{ name: 'Num_Py', version: '2.0.0' }] }, { ...lock, command: 'pip install latest' }]) { + expect(pythonEnvironmentLockSchema.safeParse(invalid).success).toBe(false) + } + }) + it('rejects unpinned images, foreign image locks, invalid encoding and oversized locks', () => { + expect(() => parsePythonEnvironmentLock(bytes, 'python:latest')).toThrow('operator image') + expect(() => parsePythonEnvironmentLock(Buffer.alloc(256 * 1024 + 1), image)).toThrow('byte limit') + expect(() => parsePythonEnvironmentLock(Buffer.from([255]), image)).toThrow() + }) +}) diff --git a/scripts/algorithm-experiments/environment.ts b/scripts/algorithm-experiments/environment.ts new file mode 100644 index 000000000..b15de5795 --- /dev/null +++ b/scripts/algorithm-experiments/environment.ts @@ -0,0 +1,65 @@ +import { recoveryLabelArguments } from './recovery' +import { execFile } from 'node:child_process' +import { createHash, randomUUID } from 'node:crypto' +import { promisify } from 'node:util' +import { z } from 'zod' +import { configHash } from '../../src/biometrics/algorithms/canonical' + +const exec = promisify(execFile) +const imageId = z.string().regex(/^sha256:[a-f0-9]{64}$/) +export const pythonEnvironmentLockSchema = z.object({ + schema: z.literal('biometric-python-environment-v1'), image: imageId, + implementation: z.literal('cpython'), pythonVersion: z.string().regex(/^\d+\.\d+\.\d+$/), + system: z.literal('Linux'), machine: z.string().regex(/^[a-zA-Z0-9_+-]{1,40}$/), + distributions: z.array(z.object({ name: z.string().regex(/^[a-z0-9]+(?:-[a-z0-9]+)*$/), version: z.string().min(1).max(100) }).strict()).max(512), +}).strict().superRefine((value, context) => { + const names = value.distributions.map(item => item.name) + if (names.some((name, index) => index > 0 && name <= names[index - 1])) context.addIssue({ code: 'custom', message: 'Distributions must be unique and sorted by normalized name' }) +}) + +const probe = `import importlib.metadata, json, platform, re, sys +packages = sorted([{'name': re.sub(r'[-_.]+', '-', dist.metadata['Name']).lower(), 'version': dist.version} for dist in importlib.metadata.distributions()], key=lambda item: item['name']) +print(json.dumps({'implementation': sys.implementation.name, 'pythonVersion': platform.python_version(), 'system': platform.system(), 'machine': platform.machine(), 'distributions': packages})) +` + +/** Probe an operator-reviewed immutable image, with no candidate code or mounts. + * Captures installed versions and pins all runtime bytes through the image ID. + * This is not a dependency resolver or an approval of the image's contents. */ +export async function inspectPythonEnvironment(image: string) { + imageId.parse(image) + const name = `sleepypod-environment-${randomUUID()}` + try { + await exec('docker', ['create', '--name', name, ...recoveryLabelArguments('environment', 10000), '--network', 'none', '--ipc', 'none', '--read-only', + '--cap-drop', 'ALL', '--security-opt', 'no-new-privileges=true', '--user', '65534:65534', + '--pids-limit', '16', '--memory', '134217728', '--memory-swap', '134217728', '--cpus', '1', + '--ulimit', 'core=0', '--ulimit', 'cpu=5', '--log-driver', 'none', '--restart', 'no', '--no-healthcheck', + '--entrypoint', 'python', image, '-I', '-B', '-c', probe], { timeout: 10000, maxBuffer: 65536 }) + const { stdout } = await exec('docker', ['start', '-a', name], { timeout: 10000, maxBuffer: 256 * 1024 }) + return pythonEnvironmentLockSchema.parse({ ...JSON.parse(stdout), schema: 'biometric-python-environment-v1', image }) + } + finally { + await removeProbe(name) + } +} +async function removeProbe(name: string) { + try { + await exec('docker', ['rm', '-f', name], { timeout: 10000, maxBuffer: 65536 }) + } + catch (error) { + if (!(error instanceof Error) || !error.message.includes('No such container:')) throw error + } +} + +export function parsePythonEnvironmentLock(bytes: Uint8Array, image: string) { + if (bytes.byteLength > 256 * 1024) throw new Error('Environment lock byte limit exceeded') + const lock = pythonEnvironmentLockSchema.parse(JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes))) + if (lock.image !== image) throw new Error('Environment lock does not match operator image') + return lock +} +export function matchPythonEnvironment(bytes: Uint8Array, actual: unknown, image: string) { + const locked = parsePythonEnvironmentLock(bytes, image) + const observed = pythonEnvironmentLockSchema.parse(actual) + if (configHash(locked) !== configHash(observed)) throw new Error('Installed Python environment differs from dependency lock') + return { schema: 'biometric-python-environment-verification-v1' as const, image, + lockHash: createHash('sha256').update(bytes).digest('hex'), environmentHash: configHash(observed) } +} diff --git a/scripts/algorithm-experiments/files.test.ts b/scripts/algorithm-experiments/files.test.ts new file mode 100644 index 000000000..2ede06f8a --- /dev/null +++ b/scripts/algorithm-experiments/files.test.ts @@ -0,0 +1,62 @@ +// @vitest-environment node +import { mkdtemp, readFile, readdir, rm, stat, symlink, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { packageFixture } from '../../src/biometrics/packages/tests/fixture' +import { publishJson, readJsonFile, readJsonLines, readPackageProposal } from './files' +import { runExperimentCli } from './cli' +let root: string +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'package-files-test-')) +}) +afterEach(async () => { + await rm(root, { recursive: true, force: true }) +}) +function proposal() { + const { files, manifest } = packageFixture() + return { schema: 'biometric-package-proposal-v1', manifest, files: [...files].map(([path, bytes]) => ({ path, base64: bytes.toString('base64') })) } +} +describe('experiment CLI file boundary', () => { + it('inspects an inert proposal and creates a complete private report without running its code', async () => { + const path = join(root, 'proposal.json'), report = join(root, 'report.json') + await writeFile(path, JSON.stringify(proposal())) + await runExperimentCli(['inspect', path, report]) + expect(JSON.parse(await readFile(report, 'utf8'))).toMatchObject({ status: 'inert_proposal', manifest: { engine: { id: 'test.hr' } } }) + expect((await stat(report)).mode & 0o777).toBe(0o600) + await expect(publishJson(report, { replaced: true })).rejects.toThrow() + expect(await readdir(root)).toEqual(['proposal.json', 'report.json']) + }) + it('rejects changed bytes, duplicate files and noncanonical base64', async () => { + const original = proposal() + for (const changed of [{ ...original, files: [original.files[0], original.files[0]] }, + { ...original, files: [{ ...original.files[0], base64: 'cGFzcw==' }, original.files[1]] }, + { ...original, files: [{ ...original.files[0], base64: original.files[0].base64 + '\n' }, original.files[1]] }]) { + const path = join(root, 'proposal.json') + await writeFile(path, JSON.stringify(changed)) + expect(() => readPackageProposal(path)).toThrow() + } + }) + it('rejects symlinks, nonfiles and byte limits for JSON and JSONL', async () => { + const path = join(root, 'input'), link = join(root, 'symlink') + await writeFile(path, '{"value":1}\n{"value":2}\n') + await symlink(path, link) + expect([...readJsonLines(path, 100)]).toEqual([{ value: 1 }, { value: 2 }]) + for (const invalid of [root, link]) { + expect(() => readJsonFile(invalid, 100)).toThrow() + expect(() => [...readJsonLines(invalid, 100)]).toThrow() + } + expect(() => [...readJsonLines(path, 5)]).toThrow() + expect(() => [...readJsonLines(path, 100, 2)]).toThrow('line byte limit') + expect(() => readJsonFile(path, 5)).toThrow() + }) + it('rejects malformed UTF-8, blank lines and incorrect CLI arguments', async () => { + const path = join(root, 'input') + await writeFile(path, Buffer.from([255])) + expect(() => readJsonFile(path, 100)).toThrow() + expect(() => [...readJsonLines(path, 100)]).toThrow() + await writeFile(path, '\n') + expect(() => [...readJsonLines(path, 100)]).toThrow('Empty JSON') + await expect(runExperimentCli(['run'])).rejects.toThrow('Usage:') + }) +}) diff --git a/scripts/algorithm-experiments/files.ts b/scripts/algorithm-experiments/files.ts new file mode 100644 index 000000000..c743add13 --- /dev/null +++ b/scripts/algorithm-experiments/files.ts @@ -0,0 +1,99 @@ +import { constants, closeSync, fstatSync, openSync, readSync } from 'node:fs' +import { link, open, unlink } from 'node:fs/promises' +import { randomUUID } from 'node:crypto' +import { z } from 'zod' +import { algorithmPackageManifestSchema, maximumPackageBytes, verifyAlgorithmPackage } from '../../src/biometrics/packages/manifest' + +function openRegular(path: string, maximumBytes: number) { + if (!Number.isSafeInteger(maximumBytes) || maximumBytes <= 0) throw new Error('Invalid file byte limit') + const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK) + try { + const stat = fstatSync(fd) + if (!stat.isFile() || stat.size > maximumBytes) throw new Error('Expected a bounded regular file') + return fd + } + catch (error) { + closeSync(fd) + throw error + } +} + +/** Symlinks, devices and FIFOs are never followed. Size is checked both before + * and during reads so a growing input cannot bypass its transport limit. */ +export function* readJsonLines(path: string, maximumBytes: number, maximumLineBytes = 8 * 1024 * 1024) { + const fd = openRegular(path, maximumBytes) + const chunk = Buffer.alloc(65536) + let total = 0, pending = Buffer.alloc(0) + try { + let count: number + while ((count = readSync(fd, chunk)) !== 0) { + total += count + if (total > maximumBytes) throw new Error('File byte limit exceeded') + pending = Buffer.concat([pending, chunk.subarray(0, count)]) + let newline: number + while ((newline = pending.indexOf(10)) >= 0) { + if (newline > maximumLineBytes) throw new Error('JSON line byte limit exceeded') + const line = pending.subarray(0, newline) + pending = pending.subarray(newline + 1) + if (!line.length) throw new Error('Empty JSON line') + yield JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(line)) as unknown + } + if (pending.length > maximumLineBytes) throw new Error('JSON line byte limit exceeded') + } + if (pending.length) yield JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(pending)) as unknown + } + finally { closeSync(fd) } +} + +export function readJsonFile(path: string, maximumBytes: number): unknown { + const fd = openRegular(path, maximumBytes) + try { + const bytes = Buffer.alloc(maximumBytes + 1) + let offset = 0, count: number + while ((count = readSync(fd, bytes, offset, bytes.length - offset, null)) !== 0) { + offset += count + if (offset > maximumBytes) throw new Error('File byte limit exceeded') + } + return JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes.subarray(0, offset))) + } + finally { closeSync(fd) } +} + +const proposalSchema = z.object({ schema: z.literal('biometric-package-proposal-v1'), manifest: algorithmPackageManifestSchema, + files: z.array(z.object({ path: z.string().min(1).max(240), base64: z.string().max(Math.ceil(maximumPackageBytes / 3) * 4) }).strict()).max(128), +}).strict() + +/** A single inert JSON envelope avoids archive extraction and filesystem paths + * derived from candidate data. Every byte is verified against the manifest. */ +export function readPackageProposal(path: string) { + const proposal = proposalSchema.parse(readJsonFile(path, 48 * 1024 * 1024)) + if (new Set(proposal.files.map(file => file.path)).size !== proposal.files.length) throw new Error('Duplicate proposal file') + const files = new Map() + let total = 0 + for (const file of proposal.files) { + const bytes = Buffer.from(file.base64, 'base64') + if (bytes.toString('base64') !== file.base64) throw new Error('Expected canonical base64') + total += bytes.length + if (total > maximumPackageBytes) throw new Error('Proposal byte limit exceeded') + files.set(file.path, bytes) + } + const verified = verifyAlgorithmPackage(proposal.manifest, files) + return { ...verified, files } +} + +/** Publish only complete JSON, mode 0600, and never replace an existing report. + * Hard-link publication is atomic and refuses existing files or symlinks. */ +export async function publishJson(path: string, value: unknown) { + const staging = `${path}.${randomUUID()}.tmp` + const file = await open(staging, 'wx', 0o600) + try { + await file.writeFile(JSON.stringify(value, null, 2) + '\n') + await file.sync() + await file.close() + await link(staging, path) + } + finally { + await file.close() + await unlink(staging) + } +} diff --git a/scripts/algorithm-experiments/fixtures.test.ts b/scripts/algorithm-experiments/fixtures.test.ts new file mode 100644 index 000000000..b7f4e791c --- /dev/null +++ b/scripts/algorithm-experiments/fixtures.test.ts @@ -0,0 +1,39 @@ +import { fixtureResourceTestPolicy } from './resource-test-policy' +import { describe, expect, it } from 'vitest' +import { assessFixtureReadings, packageFixtureSuiteSchema } from './fixtures' +const numeric = { kind: 'numeric' as const, metric: 'heart_rate', startMs: 1000, endMs: 2000, minimum: 59, maximum: 61, minimumReadings: 2, minimumQuality: 0.8 } +const reading = { timestampMs: 1000, side: 'left' as const, metric: 'heart_rate', value: 60, quality: 1, flags: [] } +describe('artifact/noise fixture scoring', () => { + it('requires coverage and rejects dropping difficult readings', () => { + expect(assessFixtureReadings([numeric], [reading, { ...reading, timestampMs: 1500 }]).passed).toBe(true) + expect(assessFixtureReadings([numeric], [reading]).failures).toContain('insufficient_coverage:heart_rate:0') + expect(assessFixtureReadings([numeric], [{ ...reading, value: null }]).passed).toBe(false) + expect(assessFixtureReadings([numeric], [reading, reading]).passed).toBe(false) + }) + it('rejects out-of-range, low quality, unknown metric and out-of-window readings', () => { + for (const changed of [{ ...reading, value: 120 }, { ...reading, quality: 0.2 }, { ...reading, timestampMs: 2000 }, { ...reading, metric: 'breathing_rate' }]) { + expect(assessFixtureReadings([{ ...numeric, minimumReadings: 1 }], [changed]).passed).toBe(false) + } + }) + it('accepts explicit null or absent output for abstention, but rejects invented vitals', () => { + const expected = { kind: 'abstain' as const, metric: 'heart_rate', startMs: 1000, endMs: 2000 } + expect(assessFixtureReadings([expected], []).passed).toBe(true) + expect(assessFixtureReadings([expected], [{ ...reading, value: null }]).passed).toBe(true) + expect(assessFixtureReadings([expected], [reading]).failures).toContain('failed_abstention:heart_rate:1000') + }) + it('rejects overlapping, vacuous, out-of-input and duplicate fixture definitions', () => { + const fixture = { id: 'clean', category: 'clean_signal', + request: { schema: 'biometric-package-run-v1', packageHash: 'a'.repeat(64), inputHash: 'b'.repeat(64), hardwareId: 'pod4', sampleRateHz: 1, side: 'left', seed: 1, config: {} }, + frames: [{ schema: 'biometric-signal-frame-v1', sequence: 0, timestampMs: 1000, samples: { left: [1], right: null }, context: { occupied: true, pumpActive: false } }], expectations: [numeric] } + const suite = (fixtures: unknown[]) => ({ schema: 'biometric-package-fixtures-v1', resourcePolicy: fixtureResourceTestPolicy, fixtures }) + expect(packageFixtureSuiteSchema.safeParse(suite([fixture])).success).toBe(true) + for (const fixtures of [[fixture, fixture], [{ ...fixture, expectations: [numeric, numeric] }], + [{ ...fixture, expectations: [{ ...numeric, startMs: 0 }] }], [{ ...fixture, category: 'empty_bed' }], + [{ ...fixture, expectations: [{ kind: 'abstain', metric: 'heart_rate', startMs: 1000, endMs: 2000 }] }]]) { + expect(packageFixtureSuiteSchema.safeParse(suite(fixtures)).success).toBe(false) + } + }) + it('rejects empty suites', () => { + expect(packageFixtureSuiteSchema.safeParse({ schema: 'biometric-package-fixtures-v1', resourcePolicy: fixtureResourceTestPolicy, fixtures: [] }).success).toBe(false) + }) +}) diff --git a/scripts/algorithm-experiments/fixtures.ts b/scripts/algorithm-experiments/fixtures.ts new file mode 100644 index 000000000..ca8927765 --- /dev/null +++ b/scripts/algorithm-experiments/fixtures.ts @@ -0,0 +1,110 @@ +import { z } from 'zod' +import { dataHash } from '../../src/biometrics/algorithms/canonical' +import { packageRunRequestSchema, signalFrameSchema } from '../../src/biometrics/packages/protocol' +import { verifyAlgorithmPackage } from '../../src/biometrics/packages/manifest' +import { assessPackageResources, packageResourcePolicySchema } from './resource-assessment' +import { packageInputHash } from './input' +import { executePackageExperiment } from './container' + +const time = z.number().finite().nonnegative().max(Number.MAX_SAFE_INTEGER) +const common = { metric: z.string().min(1).max(100), startMs: time, endMs: time } +const expectationSchema = z.discriminatedUnion('kind', [ + z.object({ ...common, kind: z.literal('numeric'), minimum: z.number().finite(), maximum: z.number().finite(), + minimumReadings: z.number().int().positive().max(20000), minimumQuality: z.number().min(0).max(1) }).strict(), + z.object({ ...common, kind: z.literal('abstain') }).strict(), +]) +export const fixtureCategories = ['clean_signal', 'pump', 'empty_bed', 'motion', 'cross_side', 'harmonics', 'gaps', 'warmup', 'boundary_reset'] as const +const fixtureSchema = z.object({ + id: z.string().regex(/^[a-z0-9][a-z0-9_-]{0,99}$/), category: z.enum(fixtureCategories), + request: packageRunRequestSchema, frames: z.array(signalFrameSchema).min(1).max(256), + expectations: z.array(expectationSchema).min(1).max(256), +}).strict().superRefine((fixture, context) => { + if (!fixture.frames.length) return + const firstMs = fixture.frames[0].timestampMs + const last = fixture.frames[fixture.frames.length - 1] + const endMs = last.timestampMs + (last.samples[fixture.request.side]?.length ?? 0) * 1000 / fixture.request.sampleRateHz + if (fixture.category === 'clean_signal' && !fixture.expectations.some(expected => expected.kind === 'numeric')) context.addIssue({ code: 'custom', message: 'Clean-signal fixture requires numeric coverage' }) + if (fixture.category === 'empty_bed' && fixture.expectations.some(expected => expected.kind !== 'abstain')) context.addIssue({ code: 'custom', message: 'Empty-bed fixture must require abstention' }) + for (let index = 0; index < fixture.expectations.length; index++) { + const expected = fixture.expectations[index] + if (expected.startMs < firstMs || expected.endMs > endMs) context.addIssue({ code: 'custom', message: 'Expectation outside fixture input interval' }) + if (expected.startMs >= expected.endMs || (expected.kind === 'numeric' && expected.minimum > expected.maximum)) { + context.addIssue({ code: 'custom', message: 'Invalid fixture expectation range' }) + } + if (fixture.expectations.slice(0, index).some(other => other.metric === expected.metric && other.startMs < expected.endMs && expected.startMs < other.endMs)) { + context.addIssue({ code: 'custom', message: 'Overlapping fixture expectations' }) + } + } +}) +export const packageFixtureSuiteSchema = z.object({ schema: z.literal('biometric-package-fixtures-v1'), resourcePolicy: packageResourcePolicySchema, + fixtures: z.array(fixtureSchema).min(1).max(64), +}).strict().refine(suite => new Set(suite.fixtures.map(fixture => fixture.id)).size === suite.fixtures.length, 'Duplicate fixture identity') + +type Expectation = z.infer +type Reading = Awaited>['results'][number]['readings'][number] +/** Expectations come from reviewed fixtures, never from package responses. + * Every returned reading must belong to a declared expectation window. */ +export function assessFixtureReadings(expectations: Expectation[], readings: Reading[]) { + const failures: string[] = [] + const counts = expectations.map(() => 0) + const identities = new Set() + for (const reading of readings) { + const identity = `${reading.metric}:${reading.timestampMs}` + if (identities.has(identity)) failures.push(`duplicate_reading:${identity}`) + identities.add(identity) + const index = expectations.findIndex(expected => expected.metric === reading.metric && expected.startMs <= reading.timestampMs && reading.timestampMs < expected.endMs) + if (index < 0) { + failures.push(`unexpected_reading:${identity}`) + continue + } + const expected = expectations[index] + if (expected.kind === 'abstain') { + if (reading.value !== null) failures.push(`failed_abstention:${identity}`) + } + else if (reading.value !== null) { + if (reading.value < expected.minimum || reading.value > expected.maximum || reading.quality < expected.minimumQuality) failures.push(`out_of_range_or_low_quality:${identity}`) + else counts[index]++ + } + } + expectations.forEach((expected, index) => { + if (expected.kind === 'numeric' && counts[index] < expected.minimumReadings) failures.push(`insufficient_coverage:${expected.metric}:${index}`) + }) + return { passed: failures.length === 0, failures, usableReadings: counts } +} + +/** This is one qualification dimension, not a held-out improvement claim. + * Fresh containers also prevent state leaking between independent fixtures. */ +export async function runPackageFixtures(options: Pick[0], 'manifest' | 'files' | 'policy' | 'signal'>, suiteInput: unknown) { + const suite = packageFixtureSuiteSchema.parse(structuredClone(suiteInput)) + const suiteHash = dataHash(suite) + const verified = verifyAlgorithmPackage(options.manifest, options.files) + const stable = { manifest: verified.manifest, files: new Map([...options.files].map(([path, bytes]) => [path, Buffer.from(bytes)])), policy: structuredClone(options.policy), signal: options.signal } + for (const fixture of suite.fixtures) { + for (const expected of fixture.expectations) { + if (!verified.manifest.metrics.some(metric => metric.id === expected.metric)) throw new Error('Fixture expects an undeclared metric') + } + if (fixture.request.packageHash !== verified.packageHash || fixture.request.inputHash !== packageInputHash(fixture.frames)) throw new Error('Fixture input/package identity mismatch') + } + const results = [] + for (const fixture of suite.fixtures) { + options.signal?.throwIfAborted() + const identity = { fixtureId: fixture.id, category: fixture.category, fixtureHash: dataHash(fixture) } + try { + const execution = await executePackageExperiment({ ...stable, request: fixture.request, frames: fixture.frames }) + const assessment = assessFixtureReadings(fixture.expectations, execution.results.flatMap(frame => frame.readings)) + results.push({ ...identity, assessment, resourceAssessment: assessPackageResources(execution.execution, suite.resourcePolicy), execution, error: null }) + } + catch (error) { + options.signal?.throwIfAborted() + results.push({ ...identity, assessment: { passed: false, failures: ['execution_failed'], usableReadings: [] }, resourceAssessment: null, execution: null, + error: error instanceof Error ? error.message.slice(0, 1000) : 'Experiment failed' }) + } + } + const covered = new Set(suite.fixtures.map(fixture => fixture.category)) + const missingCategories = fixtureCategories.filter(category => !covered.has(category)) + return { schema: 'biometric-package-fixture-report-v1' as const, packageHash: verified.packageHash, suiteHash, suite, + status: results.some(result => !result.assessment.passed || result.resourceAssessment?.status === 'failed') + ? 'failed' + : missingCategories.length || results.some(result => result.resourceAssessment?.status !== 'within_observed_limits') ? 'incomplete' : 'fixture_checks_passed', + missingCategories, results, promotion: 'requires_determinism_resource_holdout_compatibility_and_release_review' as const } +} diff --git a/scripts/algorithm-experiments/input.test.ts b/scripts/algorithm-experiments/input.test.ts new file mode 100644 index 000000000..2a475c9ce --- /dev/null +++ b/scripts/algorithm-experiments/input.test.ts @@ -0,0 +1,56 @@ +// @vitest-environment node +import { mkdtemp, rm } from 'node:fs/promises' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { describe, expect, it } from 'vitest' +import { dataHash } from '../../src/biometrics/algorithms/canonical' +import { signalFrameSchema } from '../../src/biometrics/packages/protocol' +import { freezePackageInput, packageInputHash, packageTransportCeiling, readFrozenPackageInput } from './input' + +const frame = { schema: 'biometric-signal-frame-v1', sequence: 0, timestampMs: 1000, + samples: { left: [1, 2, 3], right: null }, context: { occupied: null, pumpActive: null } } + +describe('frozen experiment input', () => { + it('binds sample bytes, context, ordering and timestamps to a streamed identity', () => { + const original = packageInputHash([frame]) + for (const changed of [{ ...frame, timestampMs: 1001 }, { ...frame, context: { occupied: true, pumpActive: null } }, + { ...frame, samples: { left: [1, 2, 4], right: null } }]) expect(packageInputHash([changed])).not.toBe(original) + expect(packageInputHash([frame, { ...frame, sequence: 1 }])).not.toBe(packageInputHash([{ ...frame, sequence: 1 }, frame])) + expect(packageInputHash([{ context: frame.context, samples: frame.samples, timestampMs: 1000, sequence: 0, schema: frame.schema }])).toBe(original) + }) + it('reads the frozen copy after source mutation and enforces storage bounds', async () => { + const root = await mkdtemp(join(tmpdir(), 'package-input-test-')) + try { + const source = structuredClone(frame) + const next = { ...frame, sequence: 1, timestampMs: 1006 } + const frozen = await freezePackageInput(join(root, 'input'), [source, next], 10000) + source.samples.left[0] = 999 + const collected = [] + for await (const value of readFrozenPackageInput(join(root, 'input'))) collected.push(value) + expect(collected).toEqual([frame, next]) + expect(frozen).toMatchObject({ hash: packageInputHash([frame, next]), frameCount: 2 }) + await expect(freezePackageInput(join(root, 'input'), [frame], 10000)).rejects.toThrow() + await expect(freezePackageInput(join(root, 'small'), [frame], 10)).rejects.toThrow('byte limit') + await expect(freezePackageInput(join(root, 'empty'), [], 10000)).rejects.toThrow('at least one') + } + finally { await rm(root, { recursive: true, force: true }) } + }) + it('derives a transport ceiling that covers the initialize line and every frame envelope of a frozen input', async () => { + const root = await mkdtemp(join(tmpdir(), 'package-input-test-')) + try { + const frames = [frame, { ...frame, sequence: 1, timestampMs: 1006, samples: { left: [0.1, 1e21, 5e-324], right: null } }] + const frozen = await freezePackageInput(join(root, 'input'), frames, 100000) + const initialize = JSON.stringify({ type: 'initialize', request: {}, runHash: 'a'.repeat(64) }) + '\n' + let transport = Buffer.byteLength(initialize) + for await (const input of readFrozenPackageInput(join(root, 'input'))) { + const parsed = signalFrameSchema.parse(input) + transport += Buffer.byteLength(JSON.stringify({ type: 'frame', frame: parsed, frameHash: dataHash(parsed) }) + '\n') + } + const ceiling = packageTransportCeiling(frozen, Buffer.byteLength(initialize)) + expect(ceiling).toBeGreaterThanOrEqual(transport) + expect(packageTransportCeiling(await freezePackageInput(join(root, 'integers'), [frame], 100000), Buffer.byteLength(initialize))) + .toBe(Buffer.byteLength(initialize) + Buffer.byteLength(JSON.stringify({ type: 'frame', frame, frameHash: dataHash(frame) }) + '\n')) + } + finally { await rm(root, { recursive: true, force: true }) } + }) +}) diff --git a/scripts/algorithm-experiments/input.ts b/scripts/algorithm-experiments/input.ts new file mode 100644 index 000000000..322c659ba --- /dev/null +++ b/scripts/algorithm-experiments/input.ts @@ -0,0 +1,59 @@ +import { createHash } from 'node:crypto' +import { open } from 'node:fs/promises' +import { createReadStream } from 'node:fs' +import { createInterface } from 'node:readline' +import { canonicalJson } from '../../src/biometrics/algorithms/canonical' +import { signalFrameSchema } from '../../src/biometrics/packages/protocol' + +const domain = 'biometric-package-input-v1\n' +/** Streaming identity; canonical frame JSON followed by LF, prefixed by domain. + * These are actual experiment frames, not a user-supplied dataset name/hash. */ +export function packageInputHash(frames: Iterable) { + const hash = createHash('sha256').update(domain) + for (const frame of frames) hash.update(canonicalJson(signalFrameSchema.parse(frame)) + '\n') + return hash.digest('hex') +} + +/** Freeze bounded input before any candidate starts. The owned file is never + * mounted into the container: the executor sends one frame at a time. */ +export async function freezePackageInput(path: string, frames: Iterable, maximumBytes: number) { + if (!Number.isSafeInteger(maximumBytes) || maximumBytes <= 0) throw new Error('Invalid input byte limit') + const file = await open(path, 'wx', 0o600) + let bytes = 0, frameCount = 0 + const hash = createHash('sha256').update(domain) + try { + for (const frame of frames) { + const encoded = canonicalJson(signalFrameSchema.parse(frame)) + '\n' + bytes += Buffer.byteLength(encoded) + if (bytes > maximumBytes) throw new Error('Frozen input byte limit exceeded') + hash.update(encoded) + await file.writeFile(encoded) + frameCount++ + } + if (!frameCount) throw new Error('Experiment requires at least one frame') + } + finally { + await file.close() + } + return { schema: 'biometric-package-input-v1' as const, hash: hash.digest('hex'), bytes, frameCount } +} + +/** Transport bytes the executor needs to stream a frozen input: the initialize + * line plus each canonical frame line wrapped in its frame envelope. Canonical + * number expansion is never shorter than JSON.stringify, so this is a ceiling. */ +export function packageTransportCeiling(frozen: { bytes: number, frameCount: number }, initializeBytes: number) { + return initializeBytes + frozen.bytes + frozen.frameCount * frameEnvelopeOverhead +} +const frameEnvelopeOverhead = Buffer.byteLength(JSON.stringify({ type: 'frame', frame: 0, frameHash: '0'.repeat(64) })) - 1 + +export async function* readFrozenPackageInput(path: string) { + const stream = createReadStream(path) + const lines = createInterface({ input: stream, crlfDelay: Infinity }) + try { + for await (const line of lines) yield JSON.parse(line) as unknown + } + finally { + lines.close() + stream.destroy() + } +} diff --git a/scripts/algorithm-experiments/ledger.test.ts b/scripts/algorithm-experiments/ledger.test.ts new file mode 100644 index 000000000..176cece30 --- /dev/null +++ b/scripts/algorithm-experiments/ledger.test.ts @@ -0,0 +1,142 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { mkdtempSync, rmSync } from 'node:fs' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { PackageExperimentLedger, packageStudySchema } from './ledger' +const start = 1791673200000 +function study() { + return { schema: 'biometric-package-study-v1', subjectId: 'fixture', side: 'left', protocolHash: 'a'.repeat(64), nights: [ + { id: 'night', startMs: start, endMs: start + 28800000, inputHash: 'b'.repeat(64), referenceHash: 'c'.repeat(64), split: 'holdout' }, + ] } +} +function trial(studyId: string, split = 'holdout') { + return { studyId, split, variants: ['default', 'current', 'candidate'].map(label => ({ label, packageHash: 'd'.repeat(64), configHash: 'e'.repeat(64), image: 'sha256:' + 'f'.repeat(64) })) } +} +let db: Database.Database, ledger: PackageExperimentLedger, now: number +beforeEach(() => { + db = new Database(':memory:') + now = 1000 + ledger = new PackageExperimentLedger(db, () => now) +}) +afterEach(() => db.close()) +describe('dedicated package experiment ledger', () => { + it('freezes studies and reserves exactly one trial per unchanged request key', () => { + const first = ledger.createStudy(study()) + expect(ledger.createStudy(study()).id).toBe(first.id) + const id = ledger.reserveTrial(trial(first.id), 'request') + expect(ledger.reserveTrial(trial(first.id), 'request')).toBe(id) + const changed = trial(first.id) + changed.variants[2].configHash = '0'.repeat(64) + expect(() => ledger.reserveTrial(changed, 'request')).toThrow('idempotency') + expect(ledger.currentEvidence(id).uniqueHoldoutAttempt).toBe(false) + }) + it('records exposure before execution and fences expired workers and their renewals', () => { + const id = ledger.reserveTrial(trial(ledger.createStudy(study()).id), 'request') + const first = ledger.claim(id, 1000) + expect(first.unexposedHoldoutAtClaim).toBe(true) + expect(() => ledger.claim(id, 1000)).toThrow('leased') + now = 2000 + const retry = ledger.claim(id, 1000) + expect(retry).toMatchObject({ unexposedHoldoutAtClaim: false, priorExposures: [{ nightId: 'night', count: 1 }] }) + expect(() => ledger.finish(first, 'succeeded', '1'.repeat(64))).toThrow('Stale') + expect(() => ledger.renew(first, 1000)).toThrow('Stale') + ledger.renew(retry, 2000) + now = 3500 + ledger.finish(retry, 'succeeded', '2'.repeat(64)) + expect(ledger.currentEvidence(id)).toMatchObject({ reportHash: '2'.repeat(64), uniqueHoldoutAttempt: false }) + expect(() => ledger.claim(id, 1000)).toThrow('completed') + }) + it('detects overlap across changed studies, hashes and splits, including failed attempts', () => { + const tuning = study() + tuning.nights[0].split = 'tuning' + const tuningId = ledger.reserveTrial(trial(ledger.createStudy(tuning).id, 'tuning'), 'tuning') + ledger.finish(ledger.claim(tuningId, 1000), 'failed', '1'.repeat(64)) + const changed = study() + changed.nights[0] = { ...changed.nights[0], id: 'renamed', startMs: start + 1000, inputHash: '0'.repeat(64), referenceHash: '1'.repeat(64) } + const next = ledger.reserveTrial(trial(ledger.createStudy(changed).id), 'new') + expect(ledger.claim(next, 1000)).toMatchObject({ unexposedHoldoutAtClaim: false, priorExposures: [{ nightId: 'renamed', count: 1 }] }) + }) + it('invalidates a previous first-use claim when later overlapping trials run', () => { + const studyId = ledger.createStudy(study()).id + const first = ledger.reserveTrial(trial(studyId), 'first') + ledger.finish(ledger.claim(first, 1000), 'succeeded', '1'.repeat(64)) + expect(ledger.currentEvidence(first).uniqueHoldoutAttempt).toBe(true) + const next = ledger.reserveTrial(trial(studyId), 'next') + ledger.claim(next, 1000) + expect(ledger.currentEvidence(first).uniqueHoldoutAttempt).toBe(false) + expect(new PackageExperimentLedger(db, () => now).currentEvidence(first).otherExposures[0].count).toBe(1) + }) + it('preserves independent sides and disjoint nights', () => { + const original = ledger.reserveTrial(trial(ledger.createStudy(study()).id), 'original') + ledger.claim(original, 1000) + for (const [key, changed] of [['right', { ...study(), side: 'right' }], ['later', { ...study(), nights: study().nights.map(night => ({ ...night, startMs: start + 86400000, endMs: start + 86400000 + 28800000 })) }]] as const) { + const id = ledger.reserveTrial(trial(ledger.createStudy(changed).id), key) + expect(ledger.claim(id, 1000).unexposedHoldoutAtClaim).toBe(true) + } + }) + it('rejects nonchronological and overlapping split plans', () => { + const value = study() + const later = { ...value.nights[0], id: 'later', startMs: start + 86400000, endMs: start + 86400000 + 28800000, split: 'tuning' } + expect(packageStudySchema.safeParse({ ...value, nights: [...value.nights, later] }).success).toBe(false) + expect(packageStudySchema.safeParse({ ...value, nights: [...value.nights, { ...value.nights[0], id: 'overlap' }] }).success).toBe(false) + }) + it('persists exposure and fences competing database connections across reopening', () => { + const directory = mkdtempSync(join(tmpdir(), 'package-ledger-test-')) + const path = join(directory, 'experiments.db') + const firstDb = new Database(path), secondDb = new Database(path) + try { + const first = new PackageExperimentLedger(firstDb, () => now), second = new PackageExperimentLedger(secondDb, () => now) + const studyId = first.createStudy(study()).id + const trialId = first.reserveTrial(trial(studyId), 'shared') + expect(second.reserveTrial(trial(studyId), 'shared')).toBe(trialId) + const lease = first.claim(trialId, 1000) + expect(() => second.claim(trialId, 1000)).toThrow('leased') + now += 1000 + const replacement = second.claim(trialId, 1000) + expect(() => first.finish(lease, 'succeeded', '1'.repeat(64))).toThrow('Stale') + second.finish(replacement, 'succeeded', '2'.repeat(64)) + firstDb.close() + secondDb.close() + const reopened = new Database(path) + try { + expect(new PackageExperimentLedger(reopened).currentEvidence(trialId)).toMatchObject({ uniqueHoldoutAttempt: false, reportHash: '2'.repeat(64) }) + } + finally { reopened.close() } + } + finally { + if (firstDb.open) firstDb.close() + if (secondDb.open) secondDb.close() + rmSync(directory, { recursive: true, force: true }) + } + }) + it('resumes interrupted initial migration branding and rejects unknown future versions', () => { + db.pragma('user_version = 0') + expect(() => new PackageExperimentLedger(db)).not.toThrow() + db.pragma('user_version = 2') + expect(() => new PackageExperimentLedger(db)).toThrow('Unsupported') + expect(db.pragma('user_version', { simple: true })).toBe(2) + }) + it('refuses another application database without changing its schema or data', () => { + const other = new Database(':memory:') + try { + other.exec('CREATE TABLE vitals(value INTEGER); INSERT INTO vitals VALUES (60)') + expect(() => new PackageExperimentLedger(other)).toThrow('non-experiment') + expect(other.prepare('SELECT value FROM vitals').get()).toEqual({ value: 60 }) + expect(other.pragma('application_id', { simple: true })).toBe(0) + } + finally { other.close() } + }) +}) + +it('allows a failed attempt without a report while refusing success without evidence', () => { + const id = ledger.reserveTrial(trial(ledger.createStudy(study()).id), 'missing-report') + const lease = ledger.claim(id, 1000) + expect(() => ledger.finish(lease, 'succeeded', null)).toThrow() + expect(ledger.trial(id).state).toBe('running') + ledger.finish(lease, 'failed', null) + expect(ledger.trial(id)).toMatchObject({ state: 'failed', report_hash: null, lease_until: null }) + expect(ledger.claim(id, 1000).unexposedHoldoutAtClaim).toBe(false) + expect(() => ledger.finish(lease, 'failed', null)).toThrow('Stale') +}) diff --git a/scripts/algorithm-experiments/ledger.ts b/scripts/algorithm-experiments/ledger.ts new file mode 100644 index 000000000..39e9dc290 --- /dev/null +++ b/scripts/algorithm-experiments/ledger.ts @@ -0,0 +1,140 @@ +import type Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { fileURLToPath } from 'node:url' +import { randomUUID } from 'node:crypto' +import { z } from 'zod' +import { configHash, dataHash } from '../../src/biometrics/algorithms/canonical' +import { timestampMsSchema } from '../../src/biometrics/references/contracts' + +const hash = z.string().regex(/^[a-f0-9]{64}$/) +const key = z.string().min(1).max(200) +const split = z.enum(['tuning', 'validation', 'holdout']) +export const packageStudySchema = z.object({ schema: z.literal('biometric-package-study-v1'), subjectId: key, side: z.enum(['left', 'right']), + protocolHash: hash, nights: z.array(z.object({ id: key, startMs: timestampMsSchema, endMs: timestampMsSchema, inputHash: hash, referenceHash: hash, split }).strict()).min(1).max(60), +}).strict().superRefine((study, context) => { + const order = { tuning: 0, validation: 1, holdout: 2 } + if (new Set(study.nights.map(night => night.id)).size !== study.nights.length) context.addIssue({ code: 'custom', message: 'Duplicate night identity' }) + const sorted = [...study.nights].sort((a, b) => a.startMs - b.startMs) + for (const [index, night] of sorted.entries()) { + if (night.endMs <= night.startMs || night.endMs - night.startMs > 86400000) context.addIssue({ code: 'custom', message: 'Invalid whole-night interval' }) + if (index > 0 && (sorted[index - 1].endMs > night.startMs || order[sorted[index - 1].split] > order[night.split])) context.addIssue({ code: 'custom', message: 'Nights must be disjoint chronological splits' }) + } +}) +const trialRequestSchema = z.object({ studyId: z.string().uuid(), split, + variants: z.array(z.object({ label: z.enum(['default', 'current', 'candidate']), packageHash: hash, configHash: hash, image: z.string().regex(/^sha256:[a-f0-9]{64}$/) }).strict()).length(3), +}).strict().refine(request => new Set(request.variants.map(variant => variant.label)).size === 3, 'All three variants are required') +interface StudyRow { id: string, hash: string, payload: string } +interface TrialRow { id: string, request_hash: string, request: string, state: 'pending' | 'running' | 'succeeded' | 'failed', attempt_id: string | null, lease_until: number | null, report_hash: string | null } +export interface PackageTrialLease { trialId: string, attemptId: string } +const applicationId = 0x53504558 + +/** Dedicated off-pod metadata database, never the pod's configuration or vitals + * database. Retain it across experiments: a fresh ledger cannot prove that a + * night was never observed elsewhere. Publication is fenced by attempt ID. */ +export class PackageExperimentLedger { + constructor(private readonly db: Database.Database, private readonly now = Date.now) { + const id = db.pragma('application_id', { simple: true }) + const version = db.pragma('user_version', { simple: true }) + if (id !== applicationId) { + const tables = db.prepare('SELECT name FROM sqlite_master WHERE name NOT LIKE \'sqlite_%\'').all() + if (id !== 0 || version !== 0 || tables.length) throw new Error('Refusing a non-experiment database') + db.pragma(`application_id = ${applicationId}`) + } + else if (version !== 0 && version !== 1) throw new Error('Unsupported experiment ledger version') + migrate(drizzle(db), { migrationsFolder: fileURLToPath(new URL('../../src/db/package-experiment-migrations', import.meta.url)) }) + db.pragma('user_version = 1') + } + + createStudy(input: unknown) { + const study = packageStudySchema.parse(input) + const hash = dataHash(study) + return this.db.transaction(() => { + const existing = this.db.prepare('SELECT * FROM package_studies WHERE hash = ?').get(hash) as StudyRow | undefined + if (existing) return { id: existing.id, hash, study } + const id = randomUUID() + this.db.prepare('INSERT INTO package_studies VALUES (?, ?, ?)').run(id, hash, JSON.stringify(study)) + return { id, hash, study } + }).immediate() + } + + reserveTrial(input: unknown, idempotencyKey: string) { + const request = trialRequestSchema.parse(input) + key.parse(idempotencyKey) + const requestHash = configHash(request) + const study = this.study(request.studyId) + if (!study.nights.some(night => night.split === request.split)) throw new Error('Selected split has no nights') + return this.db.transaction(() => { + const previous = this.db.prepare('SELECT * FROM package_trials WHERE idempotency_key = ?').get(idempotencyKey) as TrialRow | undefined + if (previous) { + if (previous.request_hash !== requestHash) throw new Error('Trial idempotency conflict') + return previous.id + } + const id = randomUUID() + this.db.prepare('INSERT INTO package_trials(id, idempotency_key, request_hash, request, state) VALUES (?, ?, ?, ?, \'pending\')').run(id, idempotencyKey, requestHash, JSON.stringify(request)) + return id + }).immediate() + } + + study(id: string) { + const row = this.db.prepare('SELECT * FROM package_studies WHERE id = ?').get(id) as StudyRow | undefined + if (!row) throw new Error('Unknown experiment study') + const study = packageStudySchema.parse(JSON.parse(row.payload)) + if (dataHash(study) !== row.hash) throw new Error('Study integrity mismatch') + return study + } + + trial(id: string) { + const row = this.db.prepare('SELECT * FROM package_trials WHERE id = ?').get(id) as TrialRow | undefined + if (!row) throw new Error('Unknown package trial') + const request = trialRequestSchema.parse(JSON.parse(row.request)) + if (configHash(request) !== row.request_hash) throw new Error('Trial integrity mismatch') + return { ...row, request } + } + + /** Exposure is committed before opening any selected input. Failed or expired + * attempts remain exposures; changing study/input/reference hashes cannot + * erase prior use of an overlapping night for the same subject and side. */ + claim(trialId: string, leaseMs: number) { + z.number().int().min(1000).max(3600000).parse(leaseMs) + return this.db.transaction(() => { + const trial = this.trial(trialId), now = this.now() + if (trial.state === 'succeeded') throw new Error('Trial already completed; reuse its report') + if (trial.state === 'running' && (trial.lease_until ?? 0) > now) throw new Error('Trial already leased') + const study = this.study(trial.request.studyId) + const nights = study.nights.filter(night => night.split === trial.request.split) + const priorExposures = nights.map(night => ({ nightId: night.id, count: (this.db.prepare(`SELECT COUNT(DISTINCT attempt_id) AS count FROM package_exposures + WHERE subject_id = ? AND side = ? AND start_ms < ? AND end_ms > ?`).get(study.subjectId, study.side, night.endMs, night.startMs) as { count: number }).count })) + const attemptId = randomUUID() + for (const night of nights) this.db.prepare('INSERT INTO package_exposures VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)').run(attemptId, trialId, night.id, study.subjectId, study.side, night.startMs, night.endMs, trial.request.split, now) + this.db.prepare('UPDATE package_trials SET state = \'running\', attempt_id = ?, lease_until = ?, report_hash = NULL WHERE id = ?').run(attemptId, now + leaseMs, trialId) + return { trialId, attemptId, priorExposures, unexposedHoldoutAtClaim: trial.request.split === 'holdout' && priorExposures.every(night => night.count === 0) } + }).immediate() + } + + /** Recomputed when evidence is reviewed; never rely on the claim-time flag + * after other experiments have accessed overlapping nights. */ + currentEvidence(trialId: string) { + return this.db.transaction(() => { + const trial = this.trial(trialId) + const study = this.study(trial.request.studyId) + const otherExposures = study.nights.filter(night => night.split === trial.request.split).map(night => ({ nightId: night.id, + count: (this.db.prepare(`SELECT COUNT(DISTINCT attempt_id) AS count FROM package_exposures + WHERE subject_id = ? AND side = ? AND start_ms < ? AND end_ms > ? AND attempt_id != ?`).get(study.subjectId, study.side, night.endMs, night.startMs, trial.attempt_id ?? '') as { count: number }).count })) + return { trialId, state: trial.state, reportHash: trial.report_hash, otherExposures, + uniqueHoldoutAttempt: trial.state === 'succeeded' && trial.request.split === 'holdout' && otherExposures.every(night => night.count === 0) } + }).deferred() + } + + renew(lease: PackageTrialLease, leaseMs: number) { + z.number().int().min(1000).max(3600000).parse(leaseMs) + const now = this.now() + if (this.db.prepare('UPDATE package_trials SET lease_until = ? WHERE id = ? AND attempt_id = ? AND state = \'running\' AND lease_until > ?').run(now + leaseMs, lease.trialId, lease.attemptId, now).changes !== 1) throw new Error('Stale trial lease') + } + + finish(lease: PackageTrialLease, status: 'succeeded' | 'failed', reportHash: string | null) { + z.enum(['succeeded', 'failed']).parse(status) + if (status === 'succeeded' || reportHash !== null) hash.parse(reportHash) + if (this.db.prepare('UPDATE package_trials SET state = ?, report_hash = ?, lease_until = NULL WHERE id = ? AND attempt_id = ? AND state = \'running\' AND lease_until > ?').run(status, reportHash, lease.trialId, lease.attemptId, this.now()).changes !== 1) throw new Error('Stale trial lease') + } +} diff --git a/scripts/algorithm-experiments/managed-evidence.ts b/scripts/algorithm-experiments/managed-evidence.ts new file mode 100644 index 000000000..7116d854e --- /dev/null +++ b/scripts/algorithm-experiments/managed-evidence.ts @@ -0,0 +1,100 @@ +import { verifyPreflightEvidence } from './preflight-evidence' +import { createHash } from 'node:crypto' +import { join } from 'node:path' +import { z } from 'zod' +import { configHash, dataHash } from '../../src/biometrics/algorithms/canonical' +import { createNightUncertainty } from '../../src/biometrics/backtest/uncertainty-core' +import { previewReferenceImport } from '../../src/biometrics/references/normalize' +import { comparePackageOutputs, packageComparisonInputSchema } from './comparison' +import { readJsonFile } from './files' +import { packageStudySchema, type PackageExperimentLedger } from './ledger' +import { managedProtocolSchema } from './managed-protocol' +import { assessPackageResources } from './resource-assessment' + +const hash = z.string().regex(/^[a-f0-9]{64}$/) +const summarySchema = z.object({ nightId: z.string(), reportHash: hash, comparison: z.unknown(), resources: z.unknown() }).strict() +const trialSchema = z.object({ schema: z.literal('biometric-managed-trial-v1'), trialId: z.string().uuid(), attemptId: z.string().uuid(), + study: packageStudySchema, protocol: managedProtocolSchema, request: z.unknown(), nights: z.array(summarySchema).min(1).max(60), + preflight: z.object({ reportHash: hash, status: z.enum(['failed', 'incomplete', 'external_checks_passed']) }).strict().optional(), + uncertainty: z.unknown(), exposureAtClaim: z.unknown(), + interpretation: z.literal('requires_current_ledger_exposure_check_and_full_qualification_review'), +}).strict() +const outputSchema = packageComparisonInputSchema.shape.outputs.element +const nightSchema = z.object({ schema: z.literal('biometric-managed-night-v1'), trialId: z.string().uuid(), attemptId: z.string().uuid(), + night: packageStudySchema.shape.nights.element, referenceImport: packageComparisonInputSchema.shape.referenceImport, + outputs: z.array(outputSchema.extend({ result: outputSchema.shape.result.extend({ execution: z.object({ + image: z.string(), limits: z.unknown(), measurements: z.unknown(), wallMs: z.unknown(), + }).passthrough() }).passthrough() })).length(3), comparison: z.unknown(), resources: z.unknown(), +}).strict() +const uncertainty = createNightUncertainty(text => createHash('sha256').update(text).digest(), 'equal_night_mean_common_mae_difference') +function retained(directory: string, identity: string) { + hash.parse(identity) + const value = readJsonFile(join(directory, `${identity}.json`), 32 * 1024 * 1024) + if (dataHash(value) !== identity) throw new Error('Retained trial report integrity mismatch') + return value +} +function same(actual: unknown, expected: unknown, description: string) { + if (dataHash(actual) !== dataHash(expected)) throw new Error(`Retained ${description} integrity mismatch`) +} + +/** Verifies the ledger-selected report graph without reopening night inputs or + * executing candidate code. Derived scores are recomputed, never accepted from + * the summary. This is an evidence snapshot, not an installation approval. */ +export function readManagedEvidence(ledger: PackageExperimentLedger, trialId: string, directory: string) { + const trial = ledger.trial(trialId) + if (trial.state !== 'succeeded' || !trial.report_hash) throw new Error('Trial has no successful retained evidence') + const report = trialSchema.parse(retained(directory, trial.report_hash)) + if (report.trialId !== trial.id || report.attemptId !== trial.attempt_id) throw new Error('Retained trial report identity mismatch') + const study = ledger.study(trial.request.studyId) + same(report.study, study, 'study') + same(report.request, trial.request, 'request') + if (configHash(report.protocol) !== study.protocolHash) throw new Error('Retained protocol integrity mismatch') + let preflight: ReturnType | null = null + if (report.protocol.preflightPlan) { + const plan = report.protocol.preflightPlan + const candidate = trial.request.variants.find(variant => variant.label === 'candidate') + if (!candidate || !report.preflight || plan.configHash !== candidate.configHash || plan.side !== study.side + || plan.hardwareId !== report.protocol.hardwareId || plan.sampleRateHz !== report.protocol.sampleRateHz) throw new Error('Retained preflight binding mismatch') + preflight = verifyPreflightEvidence(retained(directory, report.preflight.reportHash), { plan, packageHash: candidate.packageHash, image: candidate.image }) + same(preflight.status, report.preflight.status, 'preflight status') + } + else if (report.preflight) throw new Error('Unexpected retained preflight evidence') + const selected = study.nights.filter(night => night.split === trial.request.split) + if (report.nights.length !== selected.length || new Set(report.nights.map(night => night.nightId)).size !== selected.length + || selected.some(night => !report.nights.some(row => row.nightId === night.id))) throw new Error('Retained night set integrity mismatch') + const nights = report.nights.map((summary) => { + const night = nightSchema.parse(retained(directory, summary.reportHash)) + const expected = selected.find(row => row.id === summary.nightId) + same(night.night, expected, 'night') + if (night.trialId !== trial.id || night.attemptId !== trial.attempt_id) throw new Error('Retained night attempt integrity mismatch') + if (night.referenceImport.side !== study.side || night.referenceImport.subjectId !== study.subjectId + || previewReferenceImport(night.referenceImport).summary.contentHash !== night.night.referenceHash) throw new Error('Retained reference integrity mismatch') + for (const output of night.outputs) { + const variant = trial.request.variants.find(row => row.label === output.label) + if (!variant || output.request.packageHash !== variant.packageHash + || configHash({ config: output.request.config, seed: output.request.seed }) !== variant.configHash + || output.request.inputHash !== night.night.inputHash || output.request.side !== study.side + || output.request.hardwareId !== report.protocol.hardwareId || output.request.sampleRateHz !== report.protocol.sampleRateHz + || output.result.execution.image !== variant.image) throw new Error('Retained variant integrity mismatch') + same(output.result.execution.limits, output.manifest.resources, 'resource limits') + } + const comparison = comparePackageOutputs({ schema: 'biometric-package-comparison-v1', metricId: report.protocol.metricId, + startMs: night.night.startMs, endMs: night.night.endMs, minimumQuality: report.protocol.minimumQuality, + referenceImport: night.referenceImport, alignment: report.protocol.alignment, outputs: night.outputs }) + const resources = night.outputs.map(output => ({ label: output.label, assessment: assessPackageResources(output.result.execution, report.protocol.resourcePolicy) })) + same(night.comparison, comparison, 'night comparison') + same(summary.comparison, comparison, 'summary comparison') + same(night.resources, resources, 'night resources') + same(summary.resources, resources, 'summary resources') + return { nightId: summary.nightId, reportHash: summary.reportHash, comparison, resources } + }) + const uncertaintyReports = (['default', 'current'] as const).map(baseline => ({ baseline, + report: uncertainty(nights.map(night => ({ cohortId: night.nightId, + difference: night.comparison.comparisons.find(row => row.candidate === 'candidate' && row.baseline === baseline)?.differences.commonMae ?? null })), report.protocol.uncertainty) })) + same(report.uncertainty, uncertaintyReports, 'uncertainty') + const currentEvidence = ledger.currentEvidence(trialId) + if (currentEvidence.state !== 'succeeded' || currentEvidence.reportHash !== trial.report_hash) throw new Error('Trial changed during evidence verification') + return { schema: 'biometric-managed-evidence-v1' as const, trialId, reportHash: trial.report_hash, + study, protocol: report.protocol, request: trial.request, preflight, nights, uncertainty: uncertaintyReports, currentEvidence, + interpretation: 'verified_retained_evidence_requires_full_qualification_and_review' as const } +} diff --git a/scripts/algorithm-experiments/managed-files.test.ts b/scripts/algorithm-experiments/managed-files.test.ts new file mode 100644 index 000000000..ba57d87ec --- /dev/null +++ b/scripts/algorithm-experiments/managed-files.test.ts @@ -0,0 +1,77 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { chmod, mkdir, mkdtemp, readFile, rm, symlink, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { configHash } from '../../src/biometrics/algorithms/canonical' +import { packageFixture } from '../../src/biometrics/packages/tests/fixture' +import { fixtureResourceTestPolicy } from './resource-test-policy' +import { managedDescriptorSchema, runManagedFiles } from './managed-files' +let root: string +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'managed-storage-test-')) +}) +afterEach(async () => { + await rm(root, { recursive: true, force: true }) +}) +async function descriptor() { + const { manifest, files } = packageFixture() + await writeFile(join(root, 'proposal.json'), JSON.stringify({ schema: 'biometric-package-proposal-v1', manifest, + files: [...files].map(([path, bytes]) => ({ path, base64: bytes.toString('base64') })) })) + const protocol = { schema: 'biometric-managed-protocol-v1', hardwareId: 'pod4', sampleRateHz: 500, metricId: 'heart_rate', minimumQuality: 0.8, + alignment: { version: 'reference-alignment-v1', pointToleranceMs: 0, clockOffsetMs: 0, offsetSource: 'fixed' }, + uncertainty: { version: 'paired-night-percentile-v1', seed: 'fixture', resamples: 200, minimumNights: 3 }, resourcePolicy: fixtureResourceTestPolicy } + const value = { schema: 'biometric-managed-descriptor-v1', study: { schema: 'biometric-package-study-v1', subjectId: 'fixture', side: 'left', protocolHash: configHash(protocol), + nights: [{ id: 'night', startMs: 1791673200000, endMs: 1791673260000, inputHash: 'a'.repeat(64), referenceHash: 'b'.repeat(64), split: 'holdout' }] }, + protocol, split: 'holdout', idempotencyKey: 'test', variants: ['default', 'current', 'candidate'].map(label => ({ label, proposal: 'proposal.json', config: {}, seed: 1, + policy: { image: 'sha256:' + 'c'.repeat(64), maximumWallMs: 5000, maximumMemoryBytes: 100000000, maximumProcesses: 1, maximumOutputBytes: 10000, maximumInputBytes: 10000 } })), + nights: [{ id: 'night', frames: 'absent-frames', referenceImport: 'absent-reference' }] } + await writeFile(join(root, 'descriptor.json'), JSON.stringify(value)) + return value +} +describe('managed workspace boundary', () => { + it('rejects missing/extra night sources and protocol substitutions before opening inputs', async () => { + const value = await descriptor() + expect(managedDescriptorSchema.safeParse(value).success).toBe(true) + for (const changed of [{ ...value, nights: [] }, { ...value, nights: [...value.nights, value.nights[0]] }, + { ...value, protocol: { ...value.protocol, minimumQuality: 0.1 } }]) { + expect(managedDescriptorSchema.safeParse(changed).success).toBe(false) + } + }) + it('refuses public or symlinked workspaces', async () => { + await descriptor() + const publicPath = join(root, 'public') + await mkdir(publicPath) + await chmod(publicPath, 0o755) + await expect(runManagedFiles(publicPath, join(root, 'descriptor.json'))).rejects.toThrow('private owned directory') + const link = join(root, 'link') + await symlink(root, link) + await expect(runManagedFiles(link, join(root, 'descriptor.json'))).rejects.toThrow('private owned directory') + }) + it('refuses linked databases and sidecars without changing their targets', async () => { + await descriptor() + const target = join(root, 'target') + await writeFile(target, 'preserve me') + for (const name of ['experiments.sqlite', 'experiments.sqlite-wal', 'experiments.sqlite-shm', 'experiments.sqlite-journal']) { + const workspace = join(root, name + '-workspace') + await mkdir(workspace, { mode: 0o700 }) + await symlink(target, join(workspace, name)) + await expect(runManagedFiles(workspace, join(root, 'descriptor.json'))).rejects.toThrow('Unsafe experiment database') + expect(await readFile(target, 'utf8')).toBe('preserve me') + } + }) + it('refuses a foreign database and preserves its contents', async () => { + await descriptor() + const workspace = join(root, 'workspace') + await mkdir(workspace, { mode: 0o700 }) + const path = join(workspace, 'experiments.sqlite') + const database = new Database(path) + database.exec('CREATE TABLE vitals(value INTEGER); INSERT INTO vitals VALUES (60)') + database.close() + await chmod(path, 0o600) + const before = await readFile(path) + await expect(runManagedFiles(workspace, join(root, 'descriptor.json'))).rejects.toThrow('non-experiment database') + expect(await readFile(path)).toEqual(before) + }) +}) diff --git a/scripts/algorithm-experiments/managed-files.ts b/scripts/algorithm-experiments/managed-files.ts new file mode 100644 index 000000000..088a32676 --- /dev/null +++ b/scripts/algorithm-experiments/managed-files.ts @@ -0,0 +1,95 @@ +import { withExperimentWorkspace, defaultWorkspacePolicy } from './workspace' +import { assessManagedQualification } from './qualification' +import Database from 'better-sqlite3' +import { constants } from 'node:fs' +import { lstat, mkdir, open } from 'node:fs/promises' +import { dirname, join, resolve } from 'node:path' +import { z } from 'zod' +import { configHash } from '../../src/biometrics/algorithms/canonical' +import { packageStudySchema, PackageExperimentLedger } from './ledger' +import { managedProtocolSchema, runManagedTrial } from './managed' +import { experimentPolicySchema } from './policy' +import { readJsonFile, readJsonLines, readPackageProposal } from './files' + +const sourcePath = z.string().min(1).max(4096) +export const managedDescriptorSchema = z.object({ schema: z.literal('biometric-managed-descriptor-v1'), + study: packageStudySchema, protocol: managedProtocolSchema, split: z.enum(['tuning', 'validation', 'holdout']), + preflightSuite: sourcePath.optional(), + idempotencyKey: z.string().min(1).max(200), + variants: z.array(z.object({ label: z.enum(['default', 'current', 'candidate']), proposal: sourcePath, + config: z.record(z.string(), z.unknown()), seed: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER), policy: experimentPolicySchema }).strict()).length(3), + nights: z.array(z.object({ id: z.string().min(1).max(200), frames: sourcePath, referenceImport: sourcePath }).strict()).min(1).max(60), +}).strict().superRefine((value, context) => { + if (Boolean(value.protocol.preflightPlan) !== Boolean(value.preflightSuite)) context.addIssue({ code: 'custom', message: 'Preflight plan and suite must be supplied together' }) + if (new Set(value.variants.map(variant => variant.label)).size !== 3) context.addIssue({ code: 'custom', message: 'All three variants are required' }) + const selected = value.study.nights.filter(night => night.split === value.split) + if (new Set(value.nights.map(night => night.id)).size !== value.nights.length || selected.length !== value.nights.length + || selected.some(night => !value.nights.some(source => source.id === night.id))) context.addIssue({ code: 'custom', message: 'Night sources must exactly match the selected split' }) + if (configHash(value.protocol) !== value.study.protocolHash) context.addIssue({ code: 'custom', message: 'Study protocol hash mismatch' }) +}) + +async function privateDirectory(path: string) { + await mkdir(path, { recursive: true, mode: 0o700 }) + const stat = await lstat(path) + if (!stat.isDirectory() || stat.isSymbolicLink() || (stat.mode & 0o077) !== 0 + || (process.getuid && stat.uid !== process.getuid())) throw new Error('Experiment workspace must be a private owned directory, not a symlink') +} + +/** CLI storage boundary. Candidate bytes never choose this workspace or its + * database path; the operator passes the dedicated private workspace argument. */ +export async function runManagedFiles(workspace: string, descriptorPath: string, storagePolicy: unknown = defaultWorkspacePolicy) { + const descriptor = managedDescriptorSchema.parse(readJsonFile(descriptorPath, 2 * 1024 * 1024)) + const base = dirname(resolve(descriptorPath)) + const variants = descriptor.variants.map((variant) => { + const proposal = readPackageProposal(resolve(base, variant.proposal)) + return { ...variant, manifest: proposal.manifest, files: proposal.files, packageHash: proposal.packageHash } + }) + await privateDirectory(workspace) + return withExperimentWorkspace(workspace, storagePolicy, async (owned) => { + const reports = join(workspace, 'reports') + await privateDirectory(reports) + const databasePath = join(workspace, 'experiments.sqlite') + // Refuse symlinked SQLite files and sidecars before opening SQLite. The + // private owned directory excludes mutation by candidate/other-user code. + for (const path of [databasePath, databasePath + '-wal', databasePath + '-shm', databasePath + '-journal']) { + try { + const stat = await lstat(path) + if (!stat.isFile() || stat.isSymbolicLink() || stat.nlink !== 1 || (stat.mode & 0o077) !== 0 || (process.getuid && stat.uid !== process.getuid())) throw new Error('Unsafe experiment database file') + } + catch (error) { + if (!(error instanceof Error) || !('code' in error) || error.code !== 'ENOENT') throw error + } + } + try { + const created = await open(databasePath, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY | constants.O_NOFOLLOW, 0o600) + await created.close() + } + catch (error) { + if (!(error instanceof Error) || !('code' in error) || error.code !== 'EEXIST') throw error + } + const db = new Database(databasePath) + try { + owned.configureDatabase(db) + const ledger = new PackageExperimentLedger(db) + const study = ledger.createStudy(descriptor.study) + const trialId = ledger.reserveTrial({ studyId: study.id, split: descriptor.split, + variants: variants.map(variant => ({ label: variant.label, packageHash: variant.packageHash, + configHash: configHash({ config: variant.config, seed: variant.seed }), image: variant.policy.image })) }, descriptor.idempotencyKey) + const maximumInputBytes = Math.min(...variants.map(variant => variant.policy.maximumInputBytes)) + const result = await runManagedTrial({ ledger, trialId, protocol: descriptor.protocol, variants, reportDirectory: reports, publishReport: owned.publishReport, + preflightSuite: descriptor.preflightSuite && ledger.trial(trialId).state !== 'succeeded' ? readJsonFile(resolve(base, descriptor.preflightSuite), 32 * 1024 * 1024) : undefined, + loadNight: async (id) => { + await owned.checkCapacity() + const source = descriptor.nights.find(night => night.id === id) + if (!source) throw new Error('Missing retained night source') + return { referenceImport: readJsonFile(resolve(base, source.referenceImport), 2 * 1024 * 1024), + frames: () => readJsonLines(resolve(base, source.frames), maximumInputBytes) } + } }) + const reviewed = assessManagedQualification(ledger, trialId, reports) + return { schema: 'biometric-managed-receipt-v1' as const, studyId: study.id, trialId, ...result, + currentEvidence: reviewed.currentEvidence, accuracy: reviewed.qualification.accuracy, preflight: reviewed.qualification.preflight, + qualification: reviewed.qualification, assessmentHash: reviewed.assessmentHash, reportFile: join(reports, `${result.reportHash}.json`) } + } + finally { db.close() } + }) +} diff --git a/scripts/algorithm-experiments/managed-protocol.ts b/scripts/algorithm-experiments/managed-protocol.ts new file mode 100644 index 000000000..74bfc07d8 --- /dev/null +++ b/scripts/algorithm-experiments/managed-protocol.ts @@ -0,0 +1,14 @@ +import { packagePreflightPlanSchema } from './preflight' +import { packageAccuracyPolicySchema } from './accuracy-policy' +import { z } from 'zod' +import { nightBootstrapSchema } from '../../src/biometrics/backtest/uncertainty-policy' +import { packageResourcePolicySchema } from './resource-assessment' + +export const managedProtocolSchema = z.object({ schema: z.literal('biometric-managed-protocol-v1'), + hardwareId: z.string().min(1).max(100), sampleRateHz: z.number().int().positive(), metricId: z.string().min(1).max(100), + minimumQuality: z.number().min(0).max(1), + alignment: z.object({ version: z.literal('reference-alignment-v1'), pointToleranceMs: z.number().int().min(0).max(300000), + clockOffsetMs: z.number().int().min(-3600000).max(3600000), offsetSource: z.enum(['fixed', 'training']) }).strict(), + accuracyPolicy: packageAccuracyPolicySchema.optional(), preflightPlan: packagePreflightPlanSchema.optional(), + uncertainty: nightBootstrapSchema, resourcePolicy: packageResourcePolicySchema, +}).strict() diff --git a/scripts/algorithm-experiments/managed.test.ts b/scripts/algorithm-experiments/managed.test.ts new file mode 100644 index 000000000..a0256a3dc --- /dev/null +++ b/scripts/algorithm-experiments/managed.test.ts @@ -0,0 +1,299 @@ +import { defaultWorkspacePolicy } from './workspace' +// @vitest-environment node +import Database from 'better-sqlite3' +import { createHash } from 'node:crypto' +import { mkdtemp, readFile, rm, writeFile, stat } from 'node:fs/promises' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import * as reportFiles from './files' +import { beforeAll, describe, expect, it, vi } from 'vitest' +import { configHash, dataHash } from '../../src/biometrics/algorithms/canonical' +import { previewReferenceImport } from '../../src/biometrics/references/normalize' +import { packageFixture } from '../../src/biometrics/packages/tests/fixture' +import { inspectPythonEnvironment } from './environment' +import { packageInputHash } from './input' +import { PackageExperimentLedger } from './ledger' +import { runManagedFiles } from './managed-files' +import { runManagedTrial } from './managed' +import { readManagedEvidence } from './managed-evidence' +import { runPackagePreflight } from './preflight' +import { verifyPreflightEvidence } from './preflight-evidence' +import { assessManagedQualification } from './qualification' +import { fixtureResourceTestPolicy } from './resource-test-policy' +const image = process.env.BIOMETRIC_EXPERIMENT_TEST_IMAGE +let environmentBytes: Buffer +beforeAll(async () => { + if (image) environmentBytes = Buffer.from(JSON.stringify(await inspectPythonEnvironment(image))) +}) +const start = 1791673200000 +function fixture(ledger: PackageExperimentLedger, withPreflight = false) { + const { manifest, files } = packageFixture() + const code = `import json, sys +init = json.loads(sys.stdin.readline()) +for line in sys.stdin: + msg = json.loads(line) + frame = msg['frame'] + print(json.dumps({'schema': 'biometric-metric-frame-v1', 'runHash': init['runHash'], 'frameHash': msg['frameHash'], 'sequence': frame['sequence'], 'state': {}, 'readings': [{'timestampMs': frame['timestampMs'], 'side': 'left', 'metric': 'heart_rate', 'value': 60 + init['request']['config']['offset'], 'quality': 1, 'flags': []}]}), flush=True) +` + files.set('engine.py', Buffer.from(code)) + if (environmentBytes) files.set(manifest.dependencyLock, environmentBytes) + const declared = { ...manifest, metrics: manifest.metrics.map(metric => ({ ...metric, reference: { metric: 'heart_rate', windowMs: null } })), + resources: { cpuMs: 5000, wallMs: 5000, memoryBytes: 64 * 1024 * 1024, processes: 1, outputBytes: 16384 }, + artifacts: [...files].map(([path, bytes]) => ({ path, bytes: bytes.length, sha256: createHash('sha256').update(bytes).digest('hex') })) } + const frames = [0, 1].map(sequence => ({ schema: 'biometric-signal-frame-v1', sequence, timestampMs: start + sequence * 1000, + samples: { left: [1, 2, 3], right: null }, context: { occupied: true, pumpActive: false } })) + const referenceImport = { side: 'left' as const, subjectId: 'fixture', source: 'synthetic', rows: [0, 1].map(index => ({ recordKey: String(index), metric: 'heart_rate', value: 60, unit: 'bpm', timestampMs: start + index * 1000 })) } + const preflightSuite = { schema: 'biometric-package-fixtures-v1', resourcePolicy: fixtureResourceTestPolicy, + fixtures: [{ id: 'clean', category: 'clean_signal', frames, + request: { schema: 'biometric-package-run-v1', packageHash: configHash(declared), inputHash: packageInputHash(frames), + hardwareId: 'pod4', sampleRateHz: 500, side: 'left', config: { offset: 0 }, seed: 1 }, + expectations: [{ kind: 'numeric', metric: 'heart_rate', startMs: start, endMs: start + 1006, + minimum: 59, maximum: 61, minimumReadings: 2, minimumQuality: 0.8 }] }] } + const preflightPlan = { schema: 'biometric-package-preflight-plan-v1', suiteHash: dataHash(preflightSuite), repetitions: 2, + configHash: configHash({ config: { offset: 0 }, seed: 1 }), hardwareId: 'pod4', sampleRateHz: 500, side: 'left' } + const protocol = { schema: 'biometric-managed-protocol-v1', hardwareId: 'pod4', sampleRateHz: 500, metricId: 'heart_rate', minimumQuality: 0.8, ...(withPreflight ? { preflightPlan } : {}), + alignment: { version: 'reference-alignment-v1', pointToleranceMs: 0, clockOffsetMs: 0, offsetSource: 'fixed' }, + uncertainty: { version: 'paired-night-percentile-v1', seed: 'fixture', resamples: 200, minimumNights: 3 }, resourcePolicy: fixtureResourceTestPolicy } + const study = ledger.createStudy({ schema: 'biometric-package-study-v1', subjectId: 'fixture', side: 'left', protocolHash: configHash(protocol), + nights: [{ id: 'night', startMs: start, endMs: start + 120000, inputHash: packageInputHash(frames), referenceHash: previewReferenceImport(referenceImport).summary.contentHash, split: 'holdout' }] }) + const variants = (['default', 'current', 'candidate'] as const).map((label, index) => ({ label, manifest: declared, files, + config: { offset: 4 - index * 2 }, seed: 1, + policy: { image: image ?? 'sha256:' + 'f'.repeat(64), maximumWallMs: 10000, maximumMemoryBytes: 128 * 1024 * 1024, maximumProcesses: 4, maximumOutputBytes: 32768, maximumInputBytes: 32768 } })) + const request = { studyId: study.id, split: 'holdout', variants: variants.map(variant => ({ label: variant.label, packageHash: configHash(declared), + configHash: configHash({ config: variant.config, seed: variant.seed }), image: variant.policy.image })) } + return { trialId: ledger.reserveTrial(request, 'first'), request, protocol, variants, referenceImport, frames, ...(withPreflight ? { preflightSuite } : {}) } +} +describe.skipIf(!image)('managed package trials through real Docker', () => { + it('executes combined fixture and determinism preflight for the exact candidate configuration', async () => { + const db = new Database(':memory:') + try { + const input = fixture(new PackageExperimentLedger(db)), candidate = input.variants[2] + const suite = { schema: 'biometric-package-fixtures-v1', resourcePolicy: fixtureResourceTestPolicy, + fixtures: [{ id: 'clean', category: 'clean_signal', frames: input.frames, + request: { schema: 'biometric-package-run-v1', packageHash: configHash(candidate.manifest), inputHash: packageInputHash(input.frames), + hardwareId: 'pod4', sampleRateHz: 500, side: 'left', config: candidate.config, seed: candidate.seed }, + expectations: [{ kind: 'numeric', metric: 'heart_rate', startMs: start, endMs: start + 1006, + minimum: 59, maximum: 61, minimumReadings: 2, minimumQuality: 0.8 }] }] } + const plan = { schema: 'biometric-package-preflight-plan-v1', suiteHash: dataHash(suite), repetitions: 2, + configHash: configHash({ config: candidate.config, seed: candidate.seed }), hardwareId: 'pod4', sampleRateHz: 500, side: 'left' } + const result = await runPackagePreflight(candidate, plan, suite) + expect(result).toMatchObject({ status: 'incomplete', packageHash: configHash(candidate.manifest), planHash: configHash(plan), + fixtureReport: { missingCategories: expect.arrayContaining(['empty_bed', 'motion']) }, + determinism: [{ fixtureId: 'clean', error: null, report: { deterministic: true, runs: expect.any(Array) } }] }) + expect(verifyPreflightEvidence(result, { plan, packageHash: configHash(candidate.manifest), image: candidate.policy.image }).status).toBe('incomplete') + const changed = structuredClone(result) + changed.status = 'external_checks_passed' + expect(() => verifyPreflightEvidence(changed, { plan, packageHash: result.packageHash, image: result.image })).toThrow('integrity mismatch') + const changedRuns = structuredClone(result) + const repeated = changedRuns.determinism[0].report + if (!repeated) throw new Error('Missing determinism fixture') + repeated.runs[1].outputHash = 'f'.repeat(64) + expect(() => verifyPreflightEvidence(changedRuns, { plan, packageHash: result.packageHash, image: result.image })).toThrow('integrity mismatch') + expect(result.determinism[0].report?.runs).toHaveLength(2) + expect(result.determinism[0].resources).toHaveLength(2) + const failingSuite = structuredClone(suite) + failingSuite.fixtures[0].expectations[0].minimum = 58 + failingSuite.fixtures[0].expectations[0].maximum = 59 + expect(await runPackagePreflight(candidate, { ...plan, suiteHash: dataHash(failingSuite) }, failingSuite)).toMatchObject({ + status: 'failed', fixtureReport: { status: 'failed' }, + }) + for (const change of [{ configHash: 'f'.repeat(64) }, { side: 'right' }, { hardwareId: 'other' }, { sampleRateHz: 100 }, { suiteHash: 'f'.repeat(64) }]) { + await expect(runPackagePreflight(candidate, { ...plan, ...change }, suite)).rejects.toThrow(/mismatch/) + } + const controller = new AbortController() + controller.abort() + await expect(runPackagePreflight({ ...candidate, signal: controller.signal }, plan, suite)).rejects.toThrow() + } + finally { db.close() } + }, 30000) + + it('records exposure before loading, compares all variants, publishes verified artifacts and reuses completion', async () => { + const db = new Database(':memory:'), root = await mkdtemp(join(tmpdir(), 'managed-trial-test-')) + try { + const ledger = new PackageExperimentLedger(db) + const input = fixture(ledger, true) + let loads = 0 + const options = { ...input, ledger, reportDirectory: root, loadNight: async () => { + loads++ + expect(ledger.trial(input.trialId).state).toBe('running') + expect((db.prepare('SELECT COUNT(*) AS count FROM package_exposures').get() as { count: number }).count).toBe(1) + return { referenceImport: input.referenceImport, frames: () => input.frames } + } } + const result = await runManagedTrial(options) + expect(result).toMatchObject({ state: 'succeeded', evidence: { uniqueHoldoutAttempt: true } }) + const report = JSON.parse(await readFile(join(root, `${result.reportHash}.json`), 'utf8')) + expect(dataHash(report)).toBe(result.reportHash) + expect(report.preflight.status).toBe('incomplete') + const preflightPath = join(root, `${report.preflight.reportHash}.json`) + const preflightBytes = await readFile(preflightPath) + await rm(preflightPath) + await expect(runManagedTrial(options)).rejects.toThrow() + expect(loads).toBe(1) + await writeFile(preflightPath, preflightBytes) + expect(readManagedEvidence(ledger, input.trialId, root).preflight?.status).toBe('incomplete') + expect(report.uncertainty.map((row: { report: { estimate: number } }) => row.report.estimate)).toEqual([-4, -2]) + const night = JSON.parse(await readFile(join(root, `${report.nights[0].reportHash}.json`), 'utf8')) + expect(dataHash(night)).toBe(report.nights[0].reportHash) + expect(night.outputs).toHaveLength(3) + expect(await runManagedTrial(options)).toEqual({ state: 'already_completed', reportHash: result.reportHash }) + expect(loads).toBe(1) + const qualification = assessManagedQualification(ledger, input.trialId, root) + expect(assessManagedQualification(ledger, input.trialId, root).assessmentHash).toBe(qualification.assessmentHash) + expect(qualification.qualification.installationAuthorized).toBe(false) + expect(readManagedEvidence(ledger, input.trialId, root)).toMatchObject({ + reportHash: result.reportHash, currentEvidence: { uniqueHoldoutAttempt: true }, + }) + // A valid top-level hash does not excuse a missing or corrupted child. + const childPath = join(root, `${report.nights[0].reportHash}.json`) + await rm(childPath) + await expect(runManagedTrial(options)).rejects.toThrow() + await writeFile(childPath, JSON.stringify({ ...night, outputs: [] })) + await expect(runManagedTrial(options)).rejects.toThrow('integrity mismatch') + await writeFile(childPath, JSON.stringify(night)) + // Even a rehashed report graph cannot substitute a derived score. The + // verifier recomputes from the retained outputs and frozen protocol. + const forgedChild = structuredClone(night) + forgedChild.comparison.comparisons[0].differences.commonMae = -999 + const forgedChildHash = dataHash(forgedChild) + await writeFile(join(root, `${forgedChildHash}.json`), JSON.stringify(forgedChild)) + const forged = structuredClone(report) + forged.nights[0].reportHash = forgedChildHash + forged.nights[0].comparison = forgedChild.comparison + const forgedHash = dataHash(forged) + await writeFile(join(root, `${forgedHash}.json`), JSON.stringify(forged)) + db.prepare('UPDATE package_trials SET report_hash = ? WHERE id = ?').run(forgedHash, input.trialId) + expect(() => readManagedEvidence(ledger, input.trialId, root)).toThrow('night comparison integrity') + // Retained request/uncertainty substitutions are rejected independently. + for (const replacement of [{ ...report, uncertainty: [] }, { ...report, request: { ...report.request, split: 'tuning' } }]) { + const replacementHash = dataHash(replacement) + await writeFile(join(root, `${replacementHash}.json`), JSON.stringify(replacement)) + db.prepare('UPDATE package_trials SET report_hash = ? WHERE id = ?').run(replacementHash, input.trialId) + expect(() => readManagedEvidence(ledger, input.trialId, root)).toThrow('integrity mismatch') + } + db.prepare('UPDATE package_trials SET report_hash = ? WHERE id = ?').run(result.reportHash, input.trialId) + await writeFile(join(root, `${result.reportHash}.json`), JSON.stringify({ ...report, trialId: 'changed' })) + await expect(runManagedTrial(options)).rejects.toThrow('integrity mismatch') + expect(loads).toBe(1) + await writeFile(join(root, `${result.reportHash}.json`), JSON.stringify(report)) + const second = ledger.reserveTrial(input.request, 'second') + ledger.claim(second, 30000) + expect(ledger.currentEvidence(input.trialId).uniqueHoldoutAttempt).toBe(false) + expect(readManagedEvidence(ledger, input.trialId, root).currentEvidence.uniqueHoldoutAttempt).toBe(false) + const afterExposure = assessManagedQualification(ledger, input.trialId, root) + expect(afterExposure.assessmentHash).not.toBe(qualification.assessmentHash) + expect(afterExposure.qualification.accuracy.missing).toContain('no_unique_current_holdout') + expect(loads).toBe(1) + } + finally { + db.close() + await rm(root, { recursive: true, force: true }) + } + }, 30000) + it('runs a managed descriptor through persistent storage and reuses it without reopening night files', async () => { + const db = new Database(':memory:'), root = await mkdtemp(join(tmpdir(), 'managed-files-test-')) + try { + const ledger = new PackageExperimentLedger(db), input = fixture(ledger, true) + const variant = input.variants[0] + const proposal = { schema: 'biometric-package-proposal-v1', manifest: variant.manifest, + files: [...variant.files].map(([path, bytes]) => ({ path, base64: bytes.toString('base64') })) } + const descriptor = { schema: 'biometric-managed-descriptor-v1', study: ledger.study(input.request.studyId), protocol: input.protocol, + split: 'holdout', idempotencyKey: 'managed-files', preflightSuite: 'suite.json', + variants: input.variants.map(variant => ({ label: variant.label, proposal: 'proposal.json', config: variant.config, seed: variant.seed, policy: variant.policy })), + nights: [{ id: 'night', frames: 'frames.jsonl', referenceImport: 'references.json' }] } + await writeFile(join(root, 'proposal.json'), JSON.stringify(proposal)) + await writeFile(join(root, 'descriptor.json'), JSON.stringify(descriptor)) + await writeFile(join(root, 'suite.json'), JSON.stringify(input.preflightSuite)) + await writeFile(join(root, 'references.json'), JSON.stringify(input.referenceImport)) + await writeFile(join(root, 'frames.jsonl'), input.frames.map(frame => JSON.stringify(frame)).join('\n')) + const workspace = join(root, 'workspace') + const first = await runManagedFiles(workspace, join(root, 'descriptor.json')) + expect(first).toMatchObject({ state: 'succeeded', currentEvidence: { uniqueHoldoutAttempt: true }, + accuracy: { status: 'insufficient_evidence', missing: expect.arrayContaining(['no_frozen_accuracy_policy']) } }) + expect((await stat(join(workspace, 'experiments.sqlite'))).mode & 0o777).toBe(0o600) + expect(dataHash(JSON.parse(await readFile(first.reportFile, 'utf8')))).toBe(first.reportHash) + expect(first.preflight?.status).toBe('incomplete') + expect(first.qualification).toMatchObject({ status: 'insufficient_evidence', installationAuthorized: false }) + expect(first.assessmentHash).toMatch(/^[a-f0-9]{64}$/) + await rm(join(root, 'suite.json')) + await rm(join(root, 'frames.jsonl')) + await rm(join(root, 'references.json')) + const repeated = await runManagedFiles(workspace, join(root, 'descriptor.json')) + expect(repeated).toMatchObject({ state: 'already_completed', trialId: first.trialId, reportHash: first.reportHash, assessmentHash: first.assessmentHash }) + await expect(runManagedFiles(workspace, join(root, 'descriptor.json'), { ...defaultWorkspacePolicy, maximumReportFiles: 1 })).rejects.toThrow('file quota') + descriptor.variants[2].config.offset++ + await writeFile(join(root, 'descriptor.json'), JSON.stringify(descriptor)) + await expect(runManagedFiles(workspace, join(root, 'descriptor.json'))).rejects.toThrow('idempotency conflict') + } + finally { + db.close() + await rm(root, { recursive: true, force: true }) + } + }, 30000) + it('refuses publication after its lease expires while loading retained input', async () => { + const db = new Database(':memory:'), root = await mkdtemp(join(tmpdir(), 'managed-trial-stale-')) + let now = 1000 + try { + const ledger = new PackageExperimentLedger(db, () => now), input = fixture(ledger) + await expect(runManagedTrial({ ...input, ledger, reportDirectory: root, loadNight: async () => { + now += 30000 + return { referenceImport: input.referenceImport, frames: () => input.frames } + } })).rejects.toThrow('Stale trial lease') + expect(ledger.trial(input.trialId).report_hash).toBeNull() + expect(ledger.claim(input.trialId, 30000).unexposedHoldoutAtClaim).toBe(false) + } + finally { + db.close() + await rm(root, { recursive: true, force: true }) + } + }) + it('fails substituted references after recording access and preserves failure evidence', async () => { + const db = new Database(':memory:'), root = await mkdtemp(join(tmpdir(), 'managed-trial-failure-')) + try { + const ledger = new PackageExperimentLedger(db), input = fixture(ledger) + await expect(runManagedTrial({ ...input, ledger, reportDirectory: root, loadNight: async () => ({ + referenceImport: { ...input.referenceImport, source: 'changed' }, frames: () => input.frames, + }) })).rejects.toThrow('reference identity') + const trial = ledger.trial(input.trialId) + expect(trial.state).toBe('failed') + const report = JSON.parse(await readFile(join(root, `${trial.report_hash}.json`), 'utf8')) + expect(dataHash(report)).toBe(trial.report_hash) + expect(ledger.claim(input.trialId, 30000).unexposedHoldoutAtClaim).toBe(false) + } + finally { + db.close() + await rm(root, { recursive: true, force: true }) + } + }) +}) + +// These fail before any container or environment probe and run without Docker. +it.each([false, true])('finalizes failed trials when report publication fails (stale lease: %s)', async (stale) => { + const db = new Database(':memory:'), root = await mkdtemp(join(tmpdir(), 'managed-publication-failure-')) + let now = 1000 + const original = new Error('retained input unavailable') + const publication = new Error('report disk full') + const publish = vi.spyOn(reportFiles, 'publishJson').mockRejectedValue(publication) + try { + const ledger = new PackageExperimentLedger(db, () => now), input = fixture(ledger) + let replacement: ReturnType | undefined + const task = runManagedTrial({ ...input, ledger, reportDirectory: root, loadNight: async () => { + if (stale) { + now += 30000 + replacement = ledger.claim(input.trialId, 30000) + } + throw original + } }) + await expect(task).rejects.toMatchObject({ cause: original, errors: expect.arrayContaining([original, publication]), message: expect.stringContaining('failure evidence incomplete') }) + const trial = ledger.trial(input.trialId) + expect(trial).toMatchObject(stale + ? { state: 'running', attempt_id: replacement?.attemptId, report_hash: null } + : { state: 'failed', report_hash: null, lease_until: null }) + if (!stale) expect(ledger.claim(input.trialId, 30000).unexposedHoldoutAtClaim).toBe(false) + expect(publish).toHaveBeenCalledOnce() + } + finally { + publish.mockRestore() + db.close() + await rm(root, { recursive: true, force: true }) + } +}) diff --git a/scripts/algorithm-experiments/managed.ts b/scripts/algorithm-experiments/managed.ts new file mode 100644 index 000000000..a6fb9cdd4 --- /dev/null +++ b/scripts/algorithm-experiments/managed.ts @@ -0,0 +1,162 @@ +import { packageFixtureSuiteSchema } from './fixtures' +import { runPackagePreflight } from './preflight' +import { readManagedEvidence } from './managed-evidence' +import { managedProtocolSchema } from './managed-protocol' +export { managedProtocolSchema } from './managed-protocol' +import { createHash } from 'node:crypto' +import { mkdir } from 'node:fs/promises' +import { join } from 'node:path' +import { z } from 'zod' +import { configHash, dataHash } from '../../src/biometrics/algorithms/canonical' +import { createNightUncertainty } from '../../src/biometrics/backtest/uncertainty-core' +import { verifyAlgorithmPackage } from '../../src/biometrics/packages/manifest' +import { signalFrameSchema } from '../../src/biometrics/packages/protocol' +import { referenceImportSchema } from '../../src/biometrics/references/contracts' +import { previewReferenceImport } from '../../src/biometrics/references/normalize' +import { executePackageExperiment, type ExperimentPolicy } from './container' +import type { PackageExperimentLedger } from './ledger' +import { comparePackageOutputs } from './comparison' +import { assessPackageResources } from './resource-assessment' +import { publishJson } from './files' + +interface Variant { + label: 'default' | 'current' | 'candidate' + manifest: unknown + files: ReadonlyMap + config: Record + seed: number + policy: ExperimentPolicy +} +const uncertainty = createNightUncertainty(text => createHash('sha256').update(text).digest(), 'equal_night_mean_common_mae_difference') + +/** The loader is trusted host code, called only after exposure is committed. + * It must read retained inputs; no candidate module is loaded on the host. */ +export async function runManagedTrial(options: { + ledger: PackageExperimentLedger + trialId: string + protocol: unknown + preflightSuite?: unknown + variants: Variant[] + reportDirectory: string + publishReport?: typeof publishJson + loadNight: (id: string) => Promise<{ referenceImport: unknown, frames: () => Iterable }> +}) { + const publishReport = options.publishReport ?? publishJson + const trial = options.ledger.trial(options.trialId) + if (trial.state === 'succeeded') { + const evidence = readManagedEvidence(options.ledger, trial.id, options.reportDirectory) + return { state: 'already_completed' as const, reportHash: evidence.reportHash } + } + const study = options.ledger.study(trial.request.studyId) + const protocol = managedProtocolSchema.parse(structuredClone(options.protocol)) + if (configHash(protocol) !== study.protocolHash) throw new Error('Managed protocol identity mismatch') + if (options.variants.length !== 3 || new Set(options.variants.map(variant => variant.label)).size !== 3) throw new Error('Three distinct variants are required') + const variants = options.variants.map((variant) => { + const verified = verifyAlgorithmPackage(variant.manifest, variant.files) + const expected = trial.request.variants.find(item => item.label === variant.label) + const config = structuredClone(variant.config) + const seed = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER).parse(variant.seed) + if (!expected || verified.packageHash !== expected.packageHash || configHash({ config, seed }) !== expected.configHash || variant.policy.image !== expected.image) throw new Error('Managed variant identity mismatch') + return { ...variant, manifest: verified.manifest, packageHash: verified.packageHash, config, seed, + files: new Map([...variant.files].map(([path, bytes]) => [path, Buffer.from(bytes)])), policy: structuredClone(variant.policy) } + }) + const candidate = variants.find(variant => variant.label === 'candidate') + if (protocol.preflightPlan && (!candidate || protocol.preflightPlan.configHash !== configHash({ config: candidate.config, seed: candidate.seed }) + || protocol.preflightPlan.hardwareId !== protocol.hardwareId || protocol.preflightPlan.sampleRateHz !== protocol.sampleRateHz + || protocol.preflightPlan.side !== study.side)) throw new Error('Managed preflight candidate mismatch') + if (Boolean(protocol.preflightPlan) !== (options.preflightSuite !== undefined)) throw new Error('Managed preflight plan and suite must be supplied together') + const preflightSuite = options.preflightSuite === undefined ? undefined : packageFixtureSuiteSchema.parse(structuredClone(options.preflightSuite)) + if (protocol.preflightPlan && dataHash(preflightSuite) !== protocol.preflightPlan.suiteHash) throw new Error('Managed preflight suite mismatch') + await mkdir(options.reportDirectory, { recursive: true, mode: 0o700 }) + const lease = options.ledger.claim(options.trialId, 30000) + const controller = new AbortController() + const heartbeat = setInterval(() => { + try { + options.ledger.renew(lease, 30000) + } + catch { controller.abort() } + }, 5000) + const nights: { + nightId: string + reportHash: string + comparison: ReturnType + resources: { label: Variant['label'], assessment: ReturnType }[] + }[] = [] + let preflight: { reportHash: string, status: 'failed' | 'incomplete' | 'external_checks_passed' } | undefined + try { + if (protocol.preflightPlan && candidate) { + const report = await runPackagePreflight({ ...candidate, signal: controller.signal }, protocol.preflightPlan, preflightSuite) + const reportHash = dataHash(report) + options.ledger.renew(lease, 30000) + await publishReport(join(options.reportDirectory, `${reportHash}.json`), report) + preflight = { reportHash, status: report.status } + } + for (const night of study.nights.filter(night => night.split === trial.request.split)) { + controller.signal.throwIfAborted() + options.ledger.renew(lease, 30000) + const source = await options.loadNight(night.id) + const referenceImport = referenceImportSchema.parse(structuredClone(source.referenceImport)) + if (referenceImport.side !== study.side || referenceImport.subjectId !== study.subjectId + || previewReferenceImport(referenceImport).summary.contentHash !== night.referenceHash) throw new Error('Managed reference identity mismatch') + const outputs = [] + for (const variant of variants) { + controller.signal.throwIfAborted() + options.ledger.renew(lease, 30000) + const request = { schema: 'biometric-package-run-v1', packageHash: variant.packageHash, inputHash: night.inputHash, + hardwareId: protocol.hardwareId, sampleRateHz: protocol.sampleRateHz, side: study.side, seed: variant.seed, config: variant.config } + const frames = function* () { + for (const input of source.frames()) { + const frame = signalFrameSchema.parse(input) + const length = frame.samples[study.side]?.length ?? 0 + if (frame.timestampMs < night.startMs || frame.timestampMs + length * 1000 / protocol.sampleRateHz > night.endMs) throw new Error('Frame outside frozen night') + yield frame + } + } + const result = await executePackageExperiment({ ...variant, request, frames: frames(), signal: controller.signal }) + outputs.push({ label: variant.label, manifest: variant.manifest, request, result }) + } + const comparison = comparePackageOutputs({ schema: 'biometric-package-comparison-v1', metricId: protocol.metricId, + startMs: night.startMs, endMs: night.endMs, minimumQuality: protocol.minimumQuality, referenceImport, alignment: protocol.alignment, outputs }) + const report = { schema: 'biometric-managed-night-v1', trialId: lease.trialId, attemptId: lease.attemptId, night, referenceImport, outputs, comparison, + resources: outputs.map(output => ({ label: output.label, assessment: assessPackageResources(output.result.execution, protocol.resourcePolicy) })) } + const reportHash = dataHash(report) + options.ledger.renew(lease, 30000) + await publishReport(join(options.reportDirectory, `${reportHash}.json`), report) + nights.push({ nightId: night.id, reportHash, comparison, resources: report.resources }) + } + const uncertaintyReports = (['default', 'current'] as const).map(baseline => ({ baseline, + report: uncertainty(nights.map(night => ({ cohortId: night.nightId, + difference: night.comparison.comparisons.find(comparison => comparison.candidate === 'candidate' && comparison.baseline === baseline)?.differences.commonMae ?? null })), protocol.uncertainty) })) + const report = { schema: 'biometric-managed-trial-v1', trialId: lease.trialId, attemptId: lease.attemptId, + study, protocol, request: trial.request, ...(preflight ? { preflight } : {}), nights, uncertainty: uncertaintyReports, exposureAtClaim: lease, + interpretation: 'requires_current_ledger_exposure_check_and_full_qualification_review' } + const reportHash = dataHash(report) + options.ledger.renew(lease, 30000) + await publishReport(join(options.reportDirectory, `${reportHash}.json`), report) + options.ledger.finish(lease, 'succeeded', reportHash) + return { state: 'succeeded' as const, reportHash, evidence: options.ledger.currentEvidence(lease.trialId) } + } + catch (error) { + controller.abort() + const report = { schema: 'biometric-managed-trial-failure-v1', trialId: lease.trialId, attemptId: lease.attemptId, ...(preflight ? { preflight } : {}), nights, + error: error instanceof Error ? error.message.slice(0, 1000) : 'Managed trial failed' } + const reportHash = dataHash(report) + let published = false + const secondaryErrors: unknown[] = [] + try { + await publishReport(join(options.reportDirectory, `${reportHash}.json`), report) + published = true + } + catch (publicationError) { secondaryErrors.push(publicationError) } + try { + // Publication failure must not leave a writable ledger's attempt running. + // A missing artifact has no report hash. The lease still fences replacement. + options.ledger.finish(lease, 'failed', published ? reportHash : null) + } + catch (ledgerError) { secondaryErrors.push(ledgerError) } + if (secondaryErrors.length) throw new AggregateError([error, ...secondaryErrors], + `${error instanceof Error ? error.message : 'Managed trial failed'}; failure evidence incomplete`, { cause: error }) + throw error + } + finally { clearInterval(heartbeat) } +} diff --git a/scripts/algorithm-experiments/policy.ts b/scripts/algorithm-experiments/policy.ts new file mode 100644 index 000000000..fcb286271 --- /dev/null +++ b/scripts/algorithm-experiments/policy.ts @@ -0,0 +1,5 @@ +import { z } from 'zod' +const positive = z.number().int().positive().max(Number.MAX_SAFE_INTEGER) +export const experimentPolicySchema = z.object({ image: z.string().regex(/^sha256:[a-f0-9]{64}$/), maximumWallMs: positive.max(3600000), + maximumMemoryBytes: positive, maximumProcesses: positive, maximumOutputBytes: positive, maximumInputBytes: positive, +}).strict() diff --git a/scripts/algorithm-experiments/preflight-evidence.ts b/scripts/algorithm-experiments/preflight-evidence.ts new file mode 100644 index 000000000..e6a0e12cf --- /dev/null +++ b/scripts/algorithm-experiments/preflight-evidence.ts @@ -0,0 +1,103 @@ +import { z } from 'zod' +import { configHash, dataHash } from '../../src/biometrics/algorithms/canonical' +import { algorithmPackageManifestSchema } from '../../src/biometrics/packages/manifest' +import { PackageFrameValidator } from '../../src/biometrics/packages/protocol' +import { packageComparisonInputSchema } from './comparison' +import { assessFixtureReadings, fixtureCategories, packageFixtureSuiteSchema } from './fixtures' +import { packageInputHash } from './input' +import { packagePreflightPlanSchema } from './preflight' +import { assessPackageResources } from './resource-assessment' + +const hash = z.string().regex(/^[a-f0-9]{64}$/) +const execution = z.object({ image: z.string(), limits: z.unknown(), measurements: z.unknown(), wallMs: z.unknown() }).passthrough() +const result = packageComparisonInputSchema.shape.outputs.element.shape.result.extend({ execution }).passthrough() +const reportSchema = z.object({ schema: z.literal('biometric-package-preflight-report-v1'), packageHash: hash, manifest: algorithmPackageManifestSchema, + image: z.string(), plan: packagePreflightPlanSchema, planHash: hash, + fixtureReport: z.object({ schema: z.literal('biometric-package-fixture-report-v1'), packageHash: hash, suiteHash: hash, + suite: packageFixtureSuiteSchema, status: z.enum(['failed', 'incomplete', 'fixture_checks_passed']), missingCategories: z.array(z.string()), + results: z.array(z.object({ fixtureId: z.string(), category: z.string(), fixtureHash: hash, assessment: z.unknown(), + resourceAssessment: z.unknown(), execution: result.nullable(), error: z.string().nullable() }).strict()).min(1).max(64), + promotion: z.literal('requires_determinism_resource_holdout_compatibility_and_release_review'), + }).strict(), + determinism: z.array(z.object({ fixtureId: z.string(), resources: z.array(z.unknown()), error: z.string().nullable(), + report: z.object({ schema: z.literal('biometric-package-determinism-v1'), deterministic: z.boolean(), + runs: z.array(z.object({ packageHash: hash, runHash: hash, input: result.shape.frozenInput, outputHash: hash, + responseCount: z.number().int().positive(), execution }).strict()).min(2).max(5), + qualification: z.literal('requires_resource_noise_holdout_compatibility_checks_and_review'), + }).strict().nullable(), + }).strict()).min(1).max(64), + status: z.enum(['failed', 'incomplete', 'external_checks_passed']), + interpretation: z.literal('requires_heldout_accuracy_target_pod_compatibility_and_reviewed_release'), +}).strict() +function same(actual: unknown, expected: unknown) { + if (dataHash(actual) !== dataHash(expected)) throw new Error('Retained preflight integrity mismatch') +} +/** Recompute derived preflight status and verify fixture/run identities. Host + * receipts remain trusted execution records, not candidate attestations. */ +export function verifyPreflightEvidence(raw: unknown, expected: { plan: unknown, packageHash: string, image: string }) { + const report = reportSchema.parse(raw), suite = report.fixtureReport.suite + same(report.plan, expected.plan) + if (report.packageHash !== expected.packageHash || configHash(report.manifest) !== expected.packageHash || report.image !== expected.image + || report.planHash !== configHash(report.plan) || dataHash(suite) !== report.plan.suiteHash + || report.fixtureReport.suiteHash !== report.plan.suiteHash || report.fixtureReport.packageHash !== expected.packageHash) throw new Error('Retained preflight identity mismatch') + if (report.fixtureReport.results.length !== suite.fixtures.length || report.determinism.length !== suite.fixtures.length + || new Set(report.fixtureReport.results.map(row => row.fixtureId)).size !== suite.fixtures.length + || new Set(report.determinism.map(row => row.fixtureId)).size !== suite.fixtures.length) throw new Error('Retained preflight fixture set mismatch') + const checkExecution = (value: z.infer) => { + if (value.image !== expected.image) throw new Error('Retained preflight image mismatch') + same(value.limits, report.manifest.resources) + return assessPackageResources(value, suite.resourcePolicy) + } + const fixtures = suite.fixtures.map((fixture) => { + const row = report.fixtureReport.results.find(row => row.fixtureId === fixture.id) + const repeated = report.determinism.find(row => row.fixtureId === fixture.id) + if (!row || !repeated || row.fixtureHash !== dataHash(fixture) || row.category !== fixture.category + || fixture.request.packageHash !== expected.packageHash || fixture.request.inputHash !== packageInputHash(fixture.frames) + || configHash({ config: fixture.request.config, seed: fixture.request.seed }) !== report.plan.configHash + || fixture.request.hardwareId !== report.plan.hardwareId || fixture.request.sampleRateHz !== report.plan.sampleRateHz + || fixture.request.side !== report.plan.side) throw new Error('Retained preflight fixture identity mismatch') + const runHash = configHash(fixture.request) + if (row.execution) { + if (row.error !== null || row.execution.runHash !== runHash || row.execution.packageHash !== expected.packageHash + || row.execution.frozenInput.hash !== fixture.request.inputHash || row.execution.frozenInput.frameCount !== fixture.frames.length + || row.execution.results.length !== fixture.frames.length) throw new Error('Retained preflight run identity mismatch') + const validator = new PackageFrameValidator(report.manifest, fixture.request) + fixture.frames.forEach((frame, index) => validator.validateStep(frame, row.execution?.results[index])) + same(row.assessment, assessFixtureReadings(fixture.expectations, row.execution.results.flatMap(frame => frame.readings))) + same(row.resourceAssessment, checkExecution(row.execution.execution)) + } + else { + if (row.error === null) throw new Error('Missing preflight execution failure') + same(row.assessment, { passed: false, failures: ['execution_failed'], usableReadings: [] }) + same(row.resourceAssessment, null) + } + if (repeated.report) { + if (repeated.error !== null || repeated.report.runs.length !== report.plan.repetitions) throw new Error('Retained repetition count mismatch') + for (const run of repeated.report.runs) { + if (run.runHash !== runHash || run.packageHash !== expected.packageHash || run.input.hash !== fixture.request.inputHash + || run.input.frameCount !== fixture.frames.length || run.responseCount !== fixture.frames.length) throw new Error('Retained determinism identity mismatch') + } + same(repeated.resources, repeated.report.runs.map(run => checkExecution(run.execution))) + same(repeated.report.deterministic, repeated.report.runs.every(run => run.outputHash === repeated.report?.runs[0].outputHash)) + } + else { + if (repeated.error === null) throw new Error('Missing determinism execution failure') + same(repeated.resources, []) + } + return { fixturePassed: row.execution !== null && assessFixtureReadings(fixture.expectations, row.execution.results.flatMap(frame => frame.readings)).passed, + resource: row.execution ? checkExecution(row.execution.execution).status : null, + deterministic: repeated.report?.deterministic ?? false, + repeatResources: repeated.report?.runs.map(run => checkExecution(run.execution).status) ?? [] } + }) + const missingCategories = fixtureCategories.filter(category => !suite.fixtures.some(fixture => fixture.category === category)) + same(report.fixtureReport.missingCategories, missingCategories) + const fixtureStatus = fixtures.some(row => !row.fixturePassed || row.resource === 'failed') + ? 'failed' + : missingCategories.length || fixtures.some(row => row.resource !== 'within_observed_limits') ? 'incomplete' : 'fixture_checks_passed' + same(report.fixtureReport.status, fixtureStatus) + const status = fixtureStatus === 'failed' || fixtures.some(row => !row.deterministic || row.repeatResources.includes('failed')) + ? 'failed' + : fixtureStatus === 'incomplete' || fixtures.some(row => row.repeatResources.some(status => status !== 'within_observed_limits')) ? 'incomplete' : 'external_checks_passed' + same(report.status, status) + return { status, planHash: report.planHash, packageHash: report.packageHash, image: report.image } +} diff --git a/scripts/algorithm-experiments/preflight.ts b/scripts/algorithm-experiments/preflight.ts new file mode 100644 index 000000000..6509b5bfb --- /dev/null +++ b/scripts/algorithm-experiments/preflight.ts @@ -0,0 +1,52 @@ +import { z } from 'zod' +import { configHash, dataHash } from '../../src/biometrics/algorithms/canonical' +import { verifyAlgorithmPackage } from '../../src/biometrics/packages/manifest' +import { checkPackageDeterminism } from './determinism' +import { packageFixtureSuiteSchema, runPackageFixtures } from './fixtures' +import { assessPackageResources } from './resource-assessment' +import type { executePackageExperiment } from './container' + +export const packagePreflightPlanSchema = z.object({ schema: z.literal('biometric-package-preflight-plan-v1'), + suiteHash: z.string().regex(/^[a-f0-9]{64}$/), repetitions: z.number().int().min(2).max(5), + configHash: z.string().regex(/^[a-f0-9]{64}$/), hardwareId: z.string().min(1).max(100), + sampleRateHz: z.number().int().positive(), side: z.enum(['left', 'right']), +}).strict() + +/** Execute all evidence in one host-controlled flow. Every fixture is replayed + * independently for determinism, with the same candidate configuration/seed. + * This does not establish pod compatibility, holdout accuracy or installation. */ +export async function runPackagePreflight(options: Pick[0], 'manifest' | 'files' | 'policy' | 'signal'>, + rawPlan: unknown, rawSuite: unknown) { + const plan = packagePreflightPlanSchema.parse(structuredClone(rawPlan)) + const suite = packageFixtureSuiteSchema.parse(structuredClone(rawSuite)) + if (dataHash(suite) !== plan.suiteHash) throw new Error('Preflight suite identity mismatch') + const files = new Map([...options.files].map(([path, bytes]) => [path, Buffer.from(bytes)])) + const verified = verifyAlgorithmPackage(options.manifest, files) + const stable = { manifest: verified.manifest, files, policy: structuredClone(options.policy), signal: options.signal } + for (const fixture of suite.fixtures) { + if (fixture.request.packageHash !== verified.packageHash + || configHash({ config: fixture.request.config, seed: fixture.request.seed }) !== plan.configHash + || fixture.request.hardwareId !== plan.hardwareId || fixture.request.sampleRateHz !== plan.sampleRateHz + || fixture.request.side !== plan.side) throw new Error('Preflight fixture configuration mismatch') + } + options.signal?.throwIfAborted() + const fixtureReport = await runPackageFixtures(stable, suite) + const determinism = [] + for (const fixture of suite.fixtures) { + options.signal?.throwIfAborted() + try { + const report = await checkPackageDeterminism({ ...stable, request: fixture.request, frames: () => fixture.frames }, plan.repetitions) + determinism.push({ fixtureId: fixture.id, report, resources: report.runs.map(run => assessPackageResources(run.execution, suite.resourcePolicy)), error: null }) + } + catch (error) { + options.signal?.throwIfAborted() + determinism.push({ fixtureId: fixture.id, report: null, resources: [], error: error instanceof Error ? error.message.slice(0, 1000) : 'Determinism execution failed' }) + } + } + const failed = fixtureReport.status === 'failed' || determinism.some(row => !row.report?.deterministic || row.resources.some(resource => resource.status === 'failed')) + const incomplete = fixtureReport.status === 'incomplete' || determinism.some(row => row.resources.some(resource => resource.status !== 'within_observed_limits')) + return { schema: 'biometric-package-preflight-report-v1' as const, packageHash: verified.packageHash, manifest: verified.manifest, image: stable.policy.image, + plan, planHash: configHash(plan), fixtureReport, determinism, + status: failed ? 'failed' as const : incomplete ? 'incomplete' as const : 'external_checks_passed' as const, + interpretation: 'requires_heldout_accuracy_target_pod_compatibility_and_reviewed_release' as const } +} diff --git a/scripts/algorithm-experiments/qualification.test.ts b/scripts/algorithm-experiments/qualification.test.ts new file mode 100644 index 000000000..4d758c8d3 --- /dev/null +++ b/scripts/algorithm-experiments/qualification.test.ts @@ -0,0 +1,71 @@ +// @vitest-environment node +import { describe, expect, it } from 'vitest' +import { configHash } from '../../src/biometrics/algorithms/canonical' +import { combinePackageQualification } from './qualification' +import { fixtureResourceTestPolicy } from './resource-test-policy' +import { managedProtocolSchema } from './managed-protocol' + +type Evidence = Parameters[0] +type Accuracy = Parameters[1] +function fixture() { + const protocol = managedProtocolSchema.parse({ schema: 'biometric-managed-protocol-v1', hardwareId: 'pod4', sampleRateHz: 500, + metricId: 'heart_rate', minimumQuality: 0.8, resourcePolicy: fixtureResourceTestPolicy, + alignment: { version: 'reference-alignment-v1', pointToleranceMs: 0, clockOffsetMs: 0, offsetSource: 'fixed' }, + uncertainty: { version: 'paired-night-percentile-v1', seed: 'test', resamples: 200, minimumNights: 3 } }) + const accuracy: Accuracy = { schema: 'biometric-package-accuracy-assessment-v1', status: 'criteria_met', policyHash: 'a'.repeat(64), failures: [], missing: [], + interpretation: 'reference_agreement_criteria_only_requires_full_package_qualification_and_review' } + // The combiner consumes only the already-verified resource rows for each + // night; comparisons themselves are evaluated by the accuracy assessor. + const resources = (['default', 'current', 'candidate'] as const).map(label => ({ label, + assessment: { status: 'within_observed_limits' as const, policyHash: configHash(protocol.resourcePolicy), failures: [], missing: [] } })) + return { protocol, accuracy, resources } +} +function input(): { evidence: Evidence, accuracy: Accuracy } { + const { protocol, accuracy, resources } = fixture() + // Keep this input's type limited to exactly what combination reads. + const evidence: Evidence = { protocol, preflight: { status: 'external_checks_passed', planHash: 'a'.repeat(64), packageHash: 'b'.repeat(64), image: 'sha256:' + 'c'.repeat(64) }, + nights: [{ nightId: 'night', resources }] } + return { evidence, accuracy } +} +describe('combined external package qualification', () => { + it('combines passing dimensions but never grants installation', () => { + const { evidence, accuracy } = input() + expect(combinePackageQualification(evidence, accuracy)).toMatchObject({ status: 'external_criteria_met', failures: [], missing: [], + installationAuthorized: false, remainingReleaseRequirements: expect.arrayContaining(['target_pod_compatibility_and_resource_qualification', 'reviewed_shipped_registry_release']) }) + }) + it('blocks missing or incomplete preflight even with passing accuracy', () => { + const { evidence, accuracy } = input() + evidence.preflight = null + expect(combinePackageQualification(evidence, accuracy).missing).toContain('no_bound_preflight') + evidence.preflight = { status: 'incomplete', planHash: 'a', packageHash: 'b', image: 'c' } + expect(combinePackageQualification(evidence, accuracy).status).toBe('insufficient_evidence') + }) + it('keeps failures dominant while retaining missing-evidence reasons', () => { + const { evidence, accuracy } = input() + accuracy.status = 'insufficient_evidence' + if (evidence.preflight) evidence.preflight.status = 'failed' + const result = combinePackageQualification(evidence, accuracy) + expect(result).toMatchObject({ status: 'failed', failures: ['preflight_failed'], missing: ['accuracy_evidence_incomplete'] }) + accuracy.status = 'criteria_not_met' + expect(combinePackageQualification(evidence, accuracy).failures).toContain('accuracy_criteria_not_met') + }) + it('requires exact complete resource evidence for every baseline and candidate', () => { + const { evidence, accuracy } = input() + evidence.nights[0].resources.pop() + expect(combinePackageQualification(evidence, accuracy).missing).toContain('night:candidate:resource_evidence_missing_or_ambiguous') + evidence.nights[0].resources.push(evidence.nights[0].resources[0]) + expect(combinePackageQualification(evidence, accuracy).missing).toContain('night:default:resource_evidence_missing_or_ambiguous') + evidence.nights = [] + expect(combinePackageQualification(evidence, accuracy).missing).toContain('no_night_resource_evidence') + }) + it('rejects budget violations, inadequate sampling and substituted policies', () => { + const { evidence, accuracy } = input() + const candidate = evidence.nights[0].resources[2].assessment + candidate.status = 'failed' + candidate.policyHash = 'f'.repeat(64) + const result = combinePackageQualification(evidence, accuracy) + expect(result.failures).toEqual(expect.arrayContaining(['night:candidate:resource_policy_mismatch', 'night:candidate:resource_budget_failed'])) + candidate.status = 'insufficient_evidence' + expect(combinePackageQualification(evidence, accuracy).missing).toContain('night:candidate:resource_evidence_incomplete') + }) +}) diff --git a/scripts/algorithm-experiments/qualification.ts b/scripts/algorithm-experiments/qualification.ts new file mode 100644 index 000000000..4fa69d8f6 --- /dev/null +++ b/scripts/algorithm-experiments/qualification.ts @@ -0,0 +1,52 @@ +import { configHash, dataHash } from '../../src/biometrics/algorithms/canonical' +import { assessPackageAccuracy } from './accuracy' +import { readManagedEvidence } from './managed-evidence' + +type Evidence = ReturnType +type Accuracy = ReturnType + +/** Combine verified external checks. This is deliberately not a registry entry + * or activation authorization: target-pod and release review are still needed. */ +export function combinePackageQualification(evidence: Pick & { nights: Pick[] }, accuracy: Accuracy) { + const failures: string[] = [], missing: string[] = [] + if (accuracy.status === 'criteria_not_met') failures.push('accuracy_criteria_not_met') + if (accuracy.status === 'insufficient_evidence' || accuracy.missing.length) missing.push('accuracy_evidence_incomplete') + if (!evidence.preflight) missing.push('no_bound_preflight') + else if (evidence.preflight.status === 'failed') failures.push('preflight_failed') + else if (evidence.preflight.status !== 'external_checks_passed') missing.push('preflight_incomplete') + const resourcePolicyHash = configHash(evidence.protocol.resourcePolicy) + if (!evidence.nights.length) missing.push('no_night_resource_evidence') + const resources = evidence.nights.map((night) => { + for (const label of ['default', 'current', 'candidate'] as const) { + const selected = night.resources.filter(row => row.label === label) + if (selected.length !== 1) { + missing.push(`${night.nightId}:${label}:resource_evidence_missing_or_ambiguous`) + continue + } + const assessment = selected[0].assessment + if (assessment.policyHash !== resourcePolicyHash) failures.push(`${night.nightId}:${label}:resource_policy_mismatch`) + if (assessment.status === 'failed') failures.push(`${night.nightId}:${label}:resource_budget_failed`) + if (assessment.status === 'insufficient_evidence' || ('missing' in assessment && assessment.missing?.length)) missing.push(`${night.nightId}:${label}:resource_evidence_incomplete`) + } + return { nightId: night.nightId, assessments: night.resources } + }) + return { schema: 'biometric-package-qualification-assessment-v1' as const, + status: failures.length ? 'failed' as const : missing.length ? 'insufficient_evidence' as const : 'external_criteria_met' as const, + failures, missing, accuracy, preflight: evidence.preflight, resourcePolicyHash, resources, + installationAuthorized: false as const, + remainingReleaseRequirements: ['representative_fixture_review', 'target_pod_compatibility_and_resource_qualification', + 'prospective_health_and_reference_comparison', 'reviewed_shipped_registry_release'] as const, + interpretation: 'external_evidence_snapshot_not_installation_or_activation_authorization' as const } +} + +/** The public orchestration entry point accepts ledger identities, never a + * client-authored success report. Re-read for every review: later overlapping + * holdout access changes the evidence and therefore the assessment hash. */ +export function assessManagedQualification(...args: Parameters) { + const evidence = readManagedEvidence(...args) + const accuracy = assessPackageAccuracy(evidence) + const qualification = combinePackageQualification(evidence, accuracy) + const snapshot = { trialId: evidence.trialId, reportHash: evidence.reportHash, protocolHash: configHash(evidence.protocol), + request: evidence.request, currentEvidence: evidence.currentEvidence, qualification } + return { ...snapshot, assessmentHash: dataHash(snapshot) } +} diff --git a/scripts/algorithm-experiments/recovery.test.ts b/scripts/algorithm-experiments/recovery.test.ts new file mode 100644 index 000000000..fecb9f178 --- /dev/null +++ b/scripts/algorithm-experiments/recovery.test.ts @@ -0,0 +1,70 @@ +// @vitest-environment node +import { describe, expect, it } from 'vitest' +import { expiredOwnedContainer, recoveryLabels } from './recovery' + +const now = 1000000 +const id = 'a'.repeat(64) +const valid = (kind: 'experiment' | 'environment' = 'experiment') => ({ Id: id, + Name: `/sleepypod-${kind}-12345678-1234-1234-1234-123456789abc`, + Config: { Labels: recoveryLabels(kind, 1000, now) } }) + +describe('expired experiment recovery ownership', () => { + it('preserves live controllers through their deadline and cleanup grace', () => { + expect(expiredOwnedContainer(valid(), now + 120999)).toBeNull() + expect(expiredOwnedContainer(valid(), now + 121000)).toBe(id) + expect(expiredOwnedContainer(valid('environment'), now + 121000)).toBe(id) + }) + it('requires exact version, host owner, kind and immutable full ID', () => { + const container = valid() + for (const [field, value] of [['version', '2'], ['owner', 'foreign'], ['kind', 'environment'], ['expires', 'NaN'], ['expires', '0'], ['expires', '9999999999999999']]) { + expect(expiredOwnedContainer({ ...container, Config: { Labels: { ...container.Config.Labels, + [`org.sleepypod.experiment.${field}`]: value } } }, now + 121000)).toBeNull() + } + for (const value of [null, {}, { ...container, Id: 'short' }, { ...container, Name: '/unrelated' }, + { ...container, Config: {} }, { ...container, Name: '/sleepypod-experiment-not-a-uuid' }]) { + expect(expiredOwnedContainer(value, now + 121000)).toBeNull() + } + }) + it('rejects invalid clock and budget inputs', () => { + for (const value of [-1, 0, NaN, Infinity, 3600001]) expect(() => recoveryLabels('experiment', value)).toThrow() + expect(() => recoveryLabels('experiment', 1000, -1)).toThrow() + expect(expiredOwnedContainer(valid(), NaN)).toBeNull() + }) +}) + +// Actual daemon test: an abandoned process remains alive until recovery, while +// an unexpired controller and an unlabelled lookalike survive the same sweep. +it.skipIf(!process.env.BIOMETRIC_EXPERIMENT_TEST_IMAGE)('removes an overdue running container and preserves other containers', async () => { + const { execFile } = await import('node:child_process') + const { promisify } = await import('node:util') + const { randomUUID } = await import('node:crypto') + const { recoverExpiredContainers } = await import('./recovery') + const exec = promisify(execFile) + const containers: string[] = [] + const clock = Date.now() + const image = process.env.BIOMETRIC_EXPERIMENT_TEST_IMAGE + if (!image) throw new Error('Test image required') + try { + for (const labels of [recoveryLabels('experiment', 1000, clock - 122000), recoveryLabels('experiment', 1000, clock), {}]) { + const result = await exec('docker', ['run', '-d', '--name', `sleepypod-experiment-${randomUUID()}`, + ...Object.entries(labels).flatMap(([key, value]) => ['--label', `${key}=${value}`]), + '--network', 'none', '--read-only', '--cap-drop', 'ALL', '--security-opt', 'no-new-privileges=true', + '--user', '65534:65534', '--pids-limit', '16', '--memory', '134217728', '--cpus', '1', + '--entrypoint', 'python', image, '-I', '-B', '-c', 'import time; time.sleep(120)'], + { timeout: 10000 }) + containers.push(result.stdout.trim()) + } + const report = await recoverExpiredContainers(clock) + expect(report.removed).toContain(containers[0]) + expect(report.removed).not.toContain(containers[1]) + expect(report.removed).not.toContain(containers[2]) + await expect(exec('docker', ['inspect', containers[0]])).rejects.toThrow() + for (const container of containers.slice(1)) { + expect((await exec('docker', ['inspect', '--format', '{{.State.Running}}', container])).stdout.trim()).toBe('true') + } + expect((await recoverExpiredContainers(clock)).removed).not.toContain(containers[0]) + } + finally { + for (const container of containers) await exec('docker', ['rm', '-f', container]).catch(() => undefined) + } +}, 60000) diff --git a/scripts/algorithm-experiments/recovery.ts b/scripts/algorithm-experiments/recovery.ts new file mode 100644 index 000000000..f1758ef33 --- /dev/null +++ b/scripts/algorithm-experiments/recovery.ts @@ -0,0 +1,68 @@ +import { execFile } from 'node:child_process' +import { promisify } from 'node:util' + +const exec = promisify(execFile) +const prefix = 'org.sleepypod.experiment' +const owner = `${process.getuid?.() ?? 'unknown'}` +const uuid = '[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}' +const namePattern = new RegExp(`^/sleepypod-(experiment|environment)-${uuid}$`) + +/** Immutable Docker labels survive a terminated controller. Leave two minutes + * beyond the hard deadline for startup, measurement and ordinary cleanup. */ +export function recoveryLabels(kind: 'experiment' | 'environment', maximumWallMs: number, now = Date.now()) { + if (!Number.isSafeInteger(maximumWallMs) || maximumWallMs <= 0 || maximumWallMs > 3600000 + || !Number.isSafeInteger(now) || now < 0 + || !Number.isSafeInteger(now + maximumWallMs + 120000)) throw new Error('Invalid recovery deadline') + return { [`${prefix}.version`]: '1', [`${prefix}.owner`]: owner, + [`${prefix}.kind`]: kind, [`${prefix}.expires`]: String(now + maximumWallMs + 120000) } +} +export function recoveryLabelArguments(kind: 'experiment' | 'environment', maximumWallMs: number) { + return Object.entries(recoveryLabels(kind, maximumWallMs)).flatMap(([key, value]) => ['--label', `${key}=${value}`]) +} + +/** Fail closed on malformed metadata; names alone never establish ownership. */ +export function expiredOwnedContainer(raw: unknown, now: number): string | null { + if (!Number.isSafeInteger(now) || now < 0 || !raw || typeof raw !== 'object') return null + const value = raw as { Id?: unknown, Name?: unknown, Config?: { Labels?: Record } } + if (typeof value.Id !== 'string' || !/^[a-f0-9]{64}$/.test(value.Id) + || typeof value.Name !== 'string' || !namePattern.test(value.Name)) return null + const labels = value.Config?.Labels + if (!labels || labels[`${prefix}.version`] !== '1' || labels[`${prefix}.owner`] !== owner) return null + const kind = value.Name.startsWith('/sleepypod-experiment-') ? 'experiment' : 'environment' + if (labels[`${prefix}.kind`] !== kind) return null + const expires = labels[`${prefix}.expires`] + if (typeof expires !== 'string' || !/^[1-9][0-9]{0,15}$/.test(expires) + || !Number.isSafeInteger(Number(expires)) || Number(expires) > now) return null + return value.Id +} + +/** Operates only on the selected Docker daemon and the invoking host UID. + * Remove by inspected immutable ID, never by a reusable container name. */ +export async function recoverExpiredContainers(now = Date.now()) { + if (!Number.isSafeInteger(now) || now < 0) throw new Error('Invalid recovery time') + const { stdout } = await exec('docker', ['ps', '-aq', '--no-trunc', '--filter', `label=${prefix}.version=1`, + '--filter', `label=${prefix}.owner=${owner}`], { timeout: 10000, maxBuffer: 1024 * 1024 }) + const ids = stdout.trim() ? stdout.trim().split(/\s+/) : [] + if (ids.length > 4096 || ids.some(id => !/^[a-f0-9]{64}$/.test(id))) throw new Error('Invalid recovery inventory') + const removed: string[] = [] + for (const id of ids) { + let inspection + try { + inspection = await exec('docker', ['inspect', '--type', 'container', id], { timeout: 10000, maxBuffer: 1024 * 1024 }) + } + catch (error) { + if (error instanceof Error && /No such (object|container):/.test(error.message)) continue + throw error + } + const values: unknown = JSON.parse(inspection.stdout) + if (!Array.isArray(values) || values.length !== 1 || expiredOwnedContainer(values[0], now) !== id) continue + try { + await exec('docker', ['rm', '-f', id], { timeout: 10000, maxBuffer: 65536 }) + removed.push(id) + } + catch (error) { + if (!(error instanceof Error) || !error.message.includes('No such container:')) throw error + } + } + return { schema: 'biometric-container-recovery-v1' as const, checkedAt: now, inspected: ids.length, removed } +} diff --git a/scripts/algorithm-experiments/resource-assessment.test.ts b/scripts/algorithm-experiments/resource-assessment.test.ts new file mode 100644 index 000000000..4d29364e6 --- /dev/null +++ b/scripts/algorithm-experiments/resource-assessment.test.ts @@ -0,0 +1,39 @@ +import { fixtureResourceTestPolicy } from './resource-test-policy' +import { describe, expect, it } from 'vitest' +import { assessPackageResources } from './resource-assessment' +import { ResourceObservations } from './resources' +function fixture() { + const observations = new ResourceObservations(0) + observations.record({ sourceTimestampMs: 1000, cpuMs: 10, memoryBytes: 1000000, processes: 1 }, 100) + return { measurements: observations.snapshot(200), limits: { cpuMs: 1000, wallMs: 1000, memoryBytes: 2000000, processes: 1, outputBytes: 10000 }, wallMs: 200 } +} +describe('package fixture resource assessment', () => { + it('binds explicit policy and accepts complete measurements within enforced limits', () => { + const result = assessPackageResources(fixture(), fixtureResourceTestPolicy) + expect(result.status).toBe('within_observed_limits') + expect(assessPackageResources(fixture(), { ...fixtureResourceTestPolicy, maximumCpuMs: 6000 }).policyHash).not.toBe(result.policyHash) + }) + it('does not turn missing, stale or inconsistent evidence into passing evidence', () => { + const input = fixture() + for (const measurements of [null, { ...input.measurements, attempts: 2, allAttemptsMeasured: false }, + { ...input.measurements, samples: 2 }, { ...input.measurements, source: 'candidate_claim' }]) { + expect(assessPackageResources({ ...input, measurements }, fixtureResourceTestPolicy).status).toBe('insufficient_evidence') + } + expect(assessPackageResources(input, { ...fixtureResourceTestPolicy, minimumSamples: 2 }).status).toBe('insufficient_evidence') + expect(assessPackageResources(input, { ...fixtureResourceTestPolicy, maximumObservationGapMs: 50 }).status).toBe('insufficient_evidence') + }) + it('fails measured overruns and limits broader than the reviewed policy', () => { + const input = fixture() + for (const changed of [{ ...input, wallMs: 6000 }, { ...input, measurements: { ...input.measurements, maximumObservedCpuMs: 6000 } }, + { ...input, measurements: { ...input.measurements, peakObservedMemoryBytes: 100000000 } }, + { ...input, measurements: { ...input.measurements, maximumObservedProcesses: 2 } }, + { ...input, limits: { ...input.limits, memoryBytes: 100000000 } }]) { + expect(assessPackageResources(changed, fixtureResourceTestPolicy).status).toBe('failed') + } + }) + it('retains a known budget failure even if observation coverage is incomplete', () => { + const input = fixture() + const result = assessPackageResources({ ...input, wallMs: 6000, measurements: { ...input.measurements, attempts: 2, allAttemptsMeasured: false } }, fixtureResourceTestPolicy) + expect(result).toMatchObject({ status: 'failed', failures: ['wall_budget_exceeded'], missing: ['unmeasured_attempts'] }) + }) +}) diff --git a/scripts/algorithm-experiments/resource-assessment.ts b/scripts/algorithm-experiments/resource-assessment.ts new file mode 100644 index 000000000..49d2c67e9 --- /dev/null +++ b/scripts/algorithm-experiments/resource-assessment.ts @@ -0,0 +1,48 @@ +import { z } from 'zod' +import { configHash } from '../../src/biometrics/algorithms/canonical' + +const positive = z.number().finite().positive() +const count = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) +const resourceLimitsSchema = z.object({ cpuMs: positive, wallMs: positive, memoryBytes: positive.int(), processes: positive.int(), outputBytes: positive.int() }).strict() +export const packageResourcePolicySchema = z.object({ + schema: z.literal('biometric-package-resource-policy-v1'), + maximumWallMs: positive, maximumCpuMs: positive, maximumMemoryBytes: positive.int(), maximumProcesses: positive.int(), + minimumSamples: positive.int(), maximumObservationGapMs: positive, + evidenceMode: z.literal('sampled_counters_and_enforced_limits'), +}).strict() +const measurementSchema = z.object({ + schema: z.literal('biometric-package-resources-v1'), source: z.literal('host_docker_engine_stats'), + attempts: count, samples: count, allAttemptsMeasured: z.boolean(), + maximumObservedCpuMs: z.number().finite().nonnegative().nullable(), peakObservedMemoryBytes: positive.nullable(), maximumObservedProcesses: positive.int().nullable(), + maximumObservationGapMs: z.number().finite().nonnegative(), elapsedMs: z.number().finite().nonnegative(), + exactLifetimeCpu: z.literal(false), exactPeakMemory: z.literal(false), memoryIncludesCache: z.literal(true), +}).strict().refine(value => value.samples <= value.attempts + && value.allAttemptsMeasured === (value.attempts > 0 && value.samples === value.attempts) + && value.maximumObservationGapMs <= value.elapsedMs + && (value.samples === 0 + ? value.maximumObservedCpuMs === null && value.peakObservedMemoryBytes === null && value.maximumObservedProcesses === null + : value.maximumObservedCpuMs !== null && value.peakObservedMemoryBytes !== null && value.maximumObservedProcesses !== null), 'Inconsistent resource observations') + +/** Assesses actual host evidence against explicit reviewed fixture criteria. + * No default thresholds or implicit target-pod fitness claim. */ +export function assessPackageResources(input: { measurements: unknown, limits: unknown, wallMs: unknown }, policyInput: unknown) { + const policy = packageResourcePolicySchema.parse(policyInput) + const policyHash = configHash(policy) + const measurements = measurementSchema.safeParse(input.measurements) + const limits = resourceLimitsSchema.safeParse(input.limits) + const wall = z.number().finite().nonnegative().safeParse(input.wallMs) + if (!measurements.success || !limits.success || !wall.success) return { status: 'insufficient_evidence' as const, policyHash, failures: ['invalid_or_missing_resource_evidence'] } + const evidence = measurements.data + const failures: string[] = [], missing: string[] = [] + if (!evidence.allAttemptsMeasured) missing.push('unmeasured_attempts') + if (evidence.samples < policy.minimumSamples) missing.push('insufficient_samples') + if (evidence.maximumObservationGapMs > policy.maximumObservationGapMs) missing.push('observation_gap') + if (wall.data > policy.maximumWallMs) failures.push('wall_budget_exceeded') + if (evidence.maximumObservedCpuMs !== null && evidence.maximumObservedCpuMs > policy.maximumCpuMs) failures.push('cpu_budget_exceeded') + if (evidence.peakObservedMemoryBytes !== null && evidence.peakObservedMemoryBytes > policy.maximumMemoryBytes) failures.push('memory_budget_exceeded') + if (evidence.maximumObservedProcesses !== null && evidence.maximumObservedProcesses > policy.maximumProcesses) failures.push('process_budget_exceeded') + if (limits.data.cpuMs > policy.maximumCpuMs || limits.data.wallMs > policy.maximumWallMs + || limits.data.memoryBytes > policy.maximumMemoryBytes || limits.data.processes > policy.maximumProcesses) failures.push('enforced_limits_exceed_policy') + return { status: failures.length ? 'failed' as const : missing.length ? 'insufficient_evidence' as const : 'within_observed_limits' as const, + policyHash, failures, missing } +} diff --git a/scripts/algorithm-experiments/resource-test-policy.ts b/scripts/algorithm-experiments/resource-test-policy.ts new file mode 100644 index 000000000..de77cec40 --- /dev/null +++ b/scripts/algorithm-experiments/resource-test-policy.ts @@ -0,0 +1,3 @@ +// Synthetic test criteria only; not target-pod qualification. +export const fixtureResourceTestPolicy = { schema: 'biometric-package-resource-policy-v1', maximumWallMs: 5000, maximumCpuMs: 5000, + maximumMemoryBytes: 64 * 1024 * 1024, maximumProcesses: 1, minimumSamples: 1, maximumObservationGapMs: 5000, evidenceMode: 'sampled_counters_and_enforced_limits' } diff --git a/scripts/algorithm-experiments/resources.test.ts b/scripts/algorithm-experiments/resources.test.ts new file mode 100644 index 000000000..1266363e3 --- /dev/null +++ b/scripts/algorithm-experiments/resources.test.ts @@ -0,0 +1,46 @@ +// @vitest-environment node +import { describe, expect, it } from 'vitest' +import { ResourceObservations, startResourceMonitor } from './resources' +const sample = { sourceTimestampMs: 1000, cpuMs: 10, memoryBytes: 1000000, processes: 1 } +describe('host experiment resource evidence', () => { + it('retains maxima and includes initial, interior and final observation gaps', () => { + const readings = new ResourceObservations(0) + readings.record(sample, 100) + readings.record({ ...sample, sourceTimestampMs: 2000, cpuMs: 20, memoryBytes: 900000, processes: 2 }, 400) + expect(readings.snapshot(1000)).toMatchObject({ attempts: 2, samples: 2, allAttemptsMeasured: true, + maximumObservedCpuMs: 20, peakObservedMemoryBytes: 1000000, maximumObservedProcesses: 2, + maximumObservationGapMs: 600, exactLifetimeCpu: false, exactPeakMemory: false }) + }) + it('marks failed, stale, regressing and invalid observations incomplete', () => { + const readings = new ResourceObservations(0) + readings.record(sample, 100) + readings.record(null, 200) + readings.record(sample, 300) + readings.record({ ...sample, sourceTimestampMs: 2000, cpuMs: 1 }, 400) + readings.record({ ...sample, sourceTimestampMs: 2000, memoryBytes: NaN }, 500) + expect(readings.snapshot(1000)).toMatchObject({ attempts: 5, samples: 1, allAttemptsMeasured: false, maximumObservationGapMs: 900 }) + expect(new ResourceObservations(0).snapshot(1000)).toMatchObject({ samples: 0, allAttemptsMeasured: false, + maximumObservedCpuMs: null, peakObservedMemoryBytes: null }) + }) + it('coalesces concurrent reads and waits for a pending observation at stop', async () => { + let resolve: (value: typeof sample) => void = () => undefined + let calls = 0 + const monitor = startResourceMonitor(() => { + calls++ + return new Promise((done) => { + resolve = done + }) + }, 0) + const first = monitor.sample(), second = monitor.sample() + expect(first).toBe(second) + const stopping = monitor.stop() + resolve(sample) + expect(await stopping).toMatchObject({ attempts: 1, samples: 1 }) + expect(calls).toBe(1) + }) + it('reports an unavailable transport rather than fabricated zero usage', async () => { + const monitor = startResourceMonitor(null, 0) + await monitor.sample() + expect(await monitor.stop()).toMatchObject({ attempts: 1, samples: 0, allAttemptsMeasured: false, maximumObservedCpuMs: null }) + }) +}) diff --git a/scripts/algorithm-experiments/resources.ts b/scripts/algorithm-experiments/resources.ts new file mode 100644 index 000000000..9413fc19f --- /dev/null +++ b/scripts/algorithm-experiments/resources.ts @@ -0,0 +1,121 @@ +import { execFile } from 'node:child_process' +import { get } from 'node:http' +import { promisify } from 'node:util' +import { performance } from 'node:perf_hooks' +import { z } from 'zod' + +const exec = promisify(execFile) +const counter = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) +const statsSchema = z.object({ read: z.string(), id: z.string().regex(/^[a-f0-9]{64}$/), + cpu_stats: z.object({ cpu_usage: z.object({ total_usage: counter }) }), + memory_stats: z.object({ usage: counter.positive() }), pids_stats: z.object({ current: counter.positive() }), +}) + +function dockerJson(socketPath: string, path: string): Promise { + return new Promise((resolve, reject) => { + const request = get({ socketPath, path }, (response) => { + if (response.statusCode !== 200) { + response.resume() + reject(new Error('Docker measurement unavailable')) + return + } + const chunks: Buffer[] = [] + let length = 0 + response.on('data', (chunk: Buffer) => { + length += chunk.length + if (length > 128 * 1024) request.destroy(new Error('Docker measurement byte limit')) + else chunks.push(chunk) + }) + response.on('error', reject) + response.on('end', () => { + try { + resolve(JSON.parse(Buffer.concat(chunks).toString('utf8'))) + } + catch (error) { reject(error) } + }) + }) + const timeout = setTimeout(() => request.destroy(new Error('Docker measurement timeout')), 1500) + request.on('error', reject) + request.on('close', () => clearTimeout(timeout)) + }) +} + +/** Independent host evidence. Never accepts counters emitted by candidate code. + * Unsupported remote Docker transports remain explicitly unmeasured. */ +export async function dockerResourceReader(containerId: string) { + if (!/^[a-f0-9]{64}$/.test(containerId)) throw new Error('Invalid measured container identity') + const host = !process.env.DOCKER_CONTEXT && process.env.DOCKER_HOST + ? process.env.DOCKER_HOST + : JSON.parse((await exec('docker', ['context', 'inspect', '--format', '{{json .Endpoints.docker.Host}}'], { timeout: 5000, maxBuffer: 16384 })).stdout) as string + if (!host.startsWith('unix:///')) throw new Error('Resource measurement requires a local Unix Docker endpoint') + const socket = host.slice('unix://'.length) + const version = z.object({ ApiVersion: z.string().regex(/^1\.\d+$/) }).parse(await dockerJson(socket, '/version')).ApiVersion + if (Number(version.split('.')[1]) < 41) throw new Error('Docker API does not support one-shot resource measurements') + return async () => { + const value = statsSchema.parse(await dockerJson(socket, `/v${version}/containers/${containerId}/stats?stream=false&one-shot=true`)) + if (value.id !== containerId) throw new Error('Measured container identity mismatch') + const sourceTimestampMs = Date.parse(value.read) + if (!Number.isFinite(sourceTimestampMs) || sourceTimestampMs <= 0) throw new Error('Invalid resource observation timestamp') + return { sourceTimestampMs, cpuMs: value.cpu_stats.cpu_usage.total_usage / 1e6, memoryBytes: value.memory_stats.usage, processes: value.pids_stats.current } + } +} + +interface Observation { sourceTimestampMs: number, cpuMs: number, memoryBytes: number, processes: number } +export class ResourceObservations { + private attempts = 0 + private samples = 0 + private peakMemoryBytes: number | null = null + private maximumCpuMs: number | null = null + private maximumProcesses: number | null = null + private sourceTimestampMs = 0 + private lastAt: number + private maximumGapMs = 0 + constructor(private readonly startedAt: number) { this.lastAt = startedAt } + record(value: Observation | null, at: number) { + this.attempts++ + if (!value || ![value.cpuMs, value.memoryBytes, value.processes].every(number => Number.isFinite(number) && number >= 0)) return + if (!Number.isFinite(value.sourceTimestampMs) || value.sourceTimestampMs <= this.sourceTimestampMs || !Number.isFinite(at) || at < this.lastAt) return + if (this.maximumCpuMs !== null && value.cpuMs < this.maximumCpuMs) return + this.sourceTimestampMs = value.sourceTimestampMs + this.samples++ + this.maximumGapMs = Math.max(this.maximumGapMs, at - this.lastAt) + this.lastAt = at + this.maximumCpuMs = Math.max(this.maximumCpuMs ?? 0, value.cpuMs) + this.peakMemoryBytes = Math.max(this.peakMemoryBytes ?? 0, value.memoryBytes) + this.maximumProcesses = Math.max(this.maximumProcesses ?? 0, value.processes) + } + + snapshot(endedAt: number) { + return { schema: 'biometric-package-resources-v1' as const, source: 'host_docker_engine_stats' as const, + attempts: this.attempts, samples: this.samples, allAttemptsMeasured: this.attempts > 0 && this.samples === this.attempts, + maximumObservedCpuMs: this.maximumCpuMs, peakObservedMemoryBytes: this.peakMemoryBytes, maximumObservedProcesses: this.maximumProcesses, + maximumObservationGapMs: Math.max(this.maximumGapMs, endedAt - this.lastAt), elapsedMs: endedAt - this.startedAt, + exactLifetimeCpu: false, exactPeakMemory: false, memoryIncludesCache: true } + } +} + +export function startResourceMonitor(read: (() => Promise) | null, startedAt: number) { + const observations = new ResourceObservations(startedAt) + let pending: Promise | null = null + let stopped = false + const sample = () => { + if (stopped) return Promise.resolve() + if (pending) return pending + pending = (async () => { + try { + observations.record(read ? await read() : null, performance.now()) + } + catch { observations.record(null, performance.now()) } + })().finally(() => { pending = null }) + return pending + } + const timer = setInterval(() => { + if (!stopped) void sample() + }, 250) + return { sample, stop: async () => { + stopped = true + clearInterval(timer) + if (pending) await pending + return observations.snapshot(performance.now()) + } } +} diff --git a/scripts/algorithm-experiments/staging-recovery.test.ts b/scripts/algorithm-experiments/staging-recovery.test.ts new file mode 100644 index 000000000..a2a6ef113 --- /dev/null +++ b/scripts/algorithm-experiments/staging-recovery.test.ts @@ -0,0 +1,66 @@ +// @vitest-environment node +import { mkdtemp, readFile, writeFile, rm, stat, symlink } from 'node:fs/promises' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import { recordExperimentStaging, recoverExperimentStaging } from './staging-recovery' +const docker = vi.hoisted(() => ({ mode: 'absent', calls: 0 })) +vi.mock('node:child_process', () => ({ execFile: (_file: string, _args: string[], _options: unknown, callback: (error: Error | null, stdout: string, stderr: string) => void) => { + docker.calls++ + callback(docker.mode === 'absent' ? new Error('No such container: test') : docker.mode === 'offline' ? new Error('Docker unavailable') : null, '[]', '') +} })) +let base: string +const deadPid = 2147483647 +beforeEach(async () => { + base = await mkdtemp(join(tmpdir(), 'staging-recovery-test-')) + docker.mode = 'absent' + docker.calls = 0 + vi.spyOn(process, 'kill').mockImplementation((pid) => { + if (pid === deadPid) throw Object.assign(new Error('gone'), { code: 'ESRCH' }) + return true + }) +}) +afterEach(async () => { + vi.restoreAllMocks() + await rm(base, { recursive: true, force: true }) +}) +async function fixture(changes: Record = {}) { + const root = await mkdtemp(join(base, 'sleepypod-package-')) + await recordExperimentStaging(root, 'sleepypod-experiment-12345678-1234-1234-1234-123456789abc', 1000, 1000) + const path = join(root, '.sleepypod-staging.json') + const receipt = JSON.parse(await readFile(path, 'utf8')) + await writeFile(path, JSON.stringify({ ...receipt, controllerPid: deadPid, ...changes })) + await writeFile(join(root, 'input.jsonl'), 'private input', { mode: 0o600 }) + return root +} +it('removes only an expired, owned staging directory after its controller and container disappear', async () => { + const root = await fixture() + expect((await recoverExperimentStaging(121999, base)).removed).toEqual([]) + expect(docker.calls).toBe(0) + expect((await recoverExperimentStaging(122000, base)).removed).toHaveLength(1) + await expect(stat(root)).rejects.toMatchObject({ code: 'ENOENT' }) + expect((await recoverExperimentStaging(122000, base)).removed).toEqual([]) +}) +it.each([{ controllerPid: process.pid }, { inode: 1 }, { owner: 999999 }, { schema: 'unknown' }])('preserves unproven ownership or a live controller: %j', async (changes) => { + const root = await fixture(changes) + expect((await recoverExperimentStaging(122000, base)).removed).toEqual([]) + expect((await stat(root)).isDirectory()).toBe(true) + expect(docker.calls).toBe(0) +}) +it('retains files when Docker still has the container or is unavailable', async () => { + const root = await fixture() + docker.mode = 'present' + expect((await recoverExperimentStaging(122000, base)).removed).toEqual([]) + docker.mode = 'offline' + await expect(recoverExperimentStaging(122000, base)).rejects.toThrow('Docker unavailable') + expect((await stat(root)).isDirectory()).toBe(true) +}) +it('does not follow symlinked ownership markers or infer legacy directory ownership', async () => { + const root = await fixture() + await rm(join(root, '.sleepypod-staging.json')) + const legacy = await mkdtemp(join(base, 'sleepypod-package-')) + await symlink('/etc/passwd', join(root, '.sleepypod-staging.json')) + expect((await recoverExperimentStaging(122000, base)).removed).toEqual([]) + expect((await stat(legacy)).isDirectory()).toBe(true) + expect(docker.calls).toBe(0) +}) diff --git a/scripts/algorithm-experiments/staging-recovery.ts b/scripts/algorithm-experiments/staging-recovery.ts new file mode 100644 index 000000000..bd16d7e99 --- /dev/null +++ b/scripts/algorithm-experiments/staging-recovery.ts @@ -0,0 +1,106 @@ +import { execFile } from 'node:child_process' +import { constants } from 'node:fs' +import { lstat, open, opendir, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { basename, join } from 'node:path' +import { promisify } from 'node:util' +import { z } from 'zod' + +const exec = promisify(execFile) +const marker = '.sleepypod-staging.json' +const directoryName = /^sleepypod-package-[a-zA-Z0-9]{6}$/ +const integer = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) +const receiptSchema = z.object({ schema: z.literal('biometric-private-staging-v1'), + directory: z.string().regex(directoryName), device: integer, inode: integer, owner: integer, + controllerPid: integer.positive().max(2147483647), expiresAtMs: integer, + containerName: z.string().regex(/^sleepypod-experiment-[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/), +}).strict() + +/** Written before input/package bytes. Legacy unmarked directories are never + * inferred to be owned from a matching name alone. */ +export async function recordExperimentStaging(root: string, containerName: string, maximumWallMs: number, now = Date.now()) { + z.number().int().min(1).max(3600000).parse(maximumWallMs) + integer.parse(now) + const info = await lstat(root) + if (!info.isDirectory() || info.isSymbolicLink() || info.mode & 0o077 || info.uid !== process.getuid?.()) throw new Error('Staging directory must be private and owned') + const receipt = receiptSchema.parse({ schema: 'biometric-private-staging-v1', directory: basename(root), device: info.dev, inode: info.ino, + owner: info.uid, controllerPid: process.pid, expiresAtMs: now + maximumWallMs + 120000, containerName }) + const file = await open(join(root, marker), constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600) + try { + await file.writeFile(JSON.stringify(receipt)) + await file.sync() + } + finally { await file.close() } +} +function processAbsent(pid: number) { + try { + process.kill(pid, 0) + } + catch (error) { return (error as NodeJS.ErrnoException).code === 'ESRCH' } + return false +} +async function containerAbsent(name: string) { + try { + await exec('docker', ['inspect', '--type', 'container', name], { timeout: 10000, maxBuffer: 1024 * 1024 }) + return false + } + catch (error) { + if (error instanceof Error && /No such (object|container):/.test(error.message)) return true + throw error // An unavailable daemon is not proof of absence. + } +} +/** Trusted host entry point; base is injectable for isolated tests. Container + * recovery must run first. Only an expired receipt, dead controller and absent + * named container permit deletion. PID reuse conservatively retains a directory. */ +export async function recoverExperimentStaging(now = Date.now(), base = tmpdir()) { + integer.parse(now) + const removed: string[] = [] + let scanned = 0, candidates = 0 + const entries = await opendir(base) + for await (const entry of entries) { + if (++scanned > 16384) throw new Error('Staging recovery inventory exceeds limit') + if (!directoryName.test(entry.name)) continue + if (++candidates > 128) throw new Error('Staging recovery candidate limit exceeded') + const root = join(base, entry.name) + let before, receipt, markerIdentity + try { + before = await lstat(root) + if (!before.isDirectory() || before.isSymbolicLink() || before.mode & 0o077 || before.uid !== process.getuid?.()) continue + const file = await open(join(root, marker), constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK) + try { + markerIdentity = await file.stat() + if (!markerIdentity.isFile() || markerIdentity.nlink !== 1 || markerIdentity.mode & 0o077 || markerIdentity.uid !== before.uid || markerIdentity.size > 4096) continue + const bytes = Buffer.alloc(4097) + const { bytesRead } = await file.read(bytes, 0, bytes.length, 0) + if (bytesRead !== markerIdentity.size) continue + receipt = receiptSchema.parse(JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes.subarray(0, bytesRead)))) + } + finally { await file.close() } + } + catch { continue } // Unknown or damaged ownership never authorizes deletion. + if (receipt.directory !== entry.name || receipt.device !== before.dev || receipt.inode !== before.ino || receipt.owner !== before.uid + || receipt.expiresAtMs > now || !processAbsent(receipt.controllerPid)) continue + if (!await containerAbsent(receipt.containerName)) continue + let current, currentMarker + try { + current = await lstat(root) + currentMarker = await lstat(join(root, marker)) + } + catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') continue + throw error + } + if (current.dev !== before.dev || current.ino !== before.ino || current.mode !== before.mode || current.uid !== before.uid + || currentMarker.dev !== markerIdentity.dev || currentMarker.ino !== markerIdentity.ino || currentMarker.size !== markerIdentity.size + || currentMarker.mtimeMs !== markerIdentity.mtimeMs || !processAbsent(receipt.controllerPid)) continue + try { + await rm(root, { recursive: true }) + } + catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') continue + throw error + } + removed.push(entry.name) + } + return { schema: 'biometric-staging-recovery-v1' as const, checkedAt: now, scanned, candidates, removed } +} diff --git a/scripts/algorithm-experiments/workspace.test.ts b/scripts/algorithm-experiments/workspace.test.ts new file mode 100644 index 000000000..75e5f7aa9 --- /dev/null +++ b/scripts/algorithm-experiments/workspace.test.ts @@ -0,0 +1,104 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import * as filesystem from 'node:fs/promises' +import { spawn } from 'node:child_process' +import { once } from 'node:events' +import { mkdtemp, readdir, rm, symlink, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import { withExperimentWorkspace } from './workspace' + +vi.mock('node:fs/promises', async (original) => { + const actual = await original() + return { ...actual, statfs: vi.fn(actual.statfs) } +}) + +let root: string +const policy = { maximumReportBytes: 80, maximumReportFiles: 2, maximumSingleReportBytes: 60, + maximumDatabaseBytes: 65536, minimumFreeBytes: 0 } +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'experiment-workspace-test-')) +}) +afterEach(async () => { + vi.restoreAllMocks() + await rm(root, { recursive: true, force: true }) +}) +it('rejects concurrent workspace owners and releases on ordinary failure', async () => { + await withExperimentWorkspace(root, policy, async () => { + await expect(withExperimentWorkspace(root, policy, async () => null)).rejects.toThrow('locked') + }) + await expect(withExperimentWorkspace(root, policy, async () => { + throw new Error('failed') + })).rejects.toThrow('failed') + expect(await withExperimentWorkspace(root, policy, async () => 'available')).toBe('available') +}) +it('bounds report bytes/counts before publication without deleting prior evidence', async () => { + await withExperimentWorkspace(root, policy, async (workspace) => { + await workspace.publishReport(join(workspace.reports, 'first.json'), { value: 'a'.repeat(30) }) + await expect(workspace.publishReport(join(workspace.reports, 'second.json'), { value: 'b'.repeat(30) })).rejects.toThrow('byte quota') + await expect(workspace.publishReport(join(workspace.reports, 'oversize.json'), { value: 'c'.repeat(60) })).rejects.toThrow('single-report') + await workspace.publishReport(join(workspace.reports, 'small.json'), {}) + await expect(workspace.publishReport(join(workspace.reports, 'third.json'), {})).rejects.toThrow('file quota') + expect((await readdir(workspace.reports)).sort()).toEqual(['first.json', 'small.json']) + await expect(workspace.publishReport(join(root, 'outside.json'), {})).rejects.toThrow('outside') + }) +}) +it('refuses unsafe existing report entries and a symlinked lock', async () => { + await withExperimentWorkspace(root, policy, async (workspace) => { + await symlink('/tmp', join(workspace.reports, 'link')) + await expect(workspace.checkCapacity()).rejects.toThrow('Unsafe') + }) + await rm(join(root, '.writer.sqlite')) + await symlink('/tmp', join(root, '.writer.sqlite')) + await expect(withExperimentWorkspace(root, policy, async () => null)).rejects.toThrow('Unsafe') +}) +it('enforces the ledger page ceiling and refuses a reduced limit below existing size', async () => { + const db = new Database(join(root, 'experiments.sqlite')) + try { + await withExperimentWorkspace(root, policy, async (workspace) => { + workspace.configureDatabase(db) + db.exec('CREATE TABLE payload(bytes BLOB)') + expect(() => db.prepare('INSERT INTO payload VALUES (?)').run(Buffer.alloc(100000))).toThrow(/full/) + expect(db.prepare('SELECT COUNT(*) AS count FROM payload').get()).toEqual({ count: 0 }) + }) + db.pragma('max_page_count = 1000') + db.prepare('INSERT INTO payload VALUES (?)').run(Buffer.alloc(100000)) + await expect(withExperimentWorkspace(root, policy, async workspace => workspace.configureDatabase(db))).rejects.toThrow('database quota') + } + finally { db.close() } +}) +it('fails admission before starting work when the free-space reserve is unavailable', async () => { + vi.mocked(filesystem.statfs).mockResolvedValueOnce({ type: 0n, bsize: 4096n, blocks: 1n, bfree: 0n, bavail: 0n, files: 0n, ffree: 0n }) + let entered = false + await expect(withExperimentWorkspace(root, { ...policy, minimumFreeBytes: 1024 ** 4 }, async () => { + entered = true + })).rejects.toThrow('free-space') + expect(entered).toBe(false) +}) +it('releases the OS writer lock after the owning controller is killed', async () => { + await writeFile(join(root, '.writer.sqlite'), '', { mode: 0o600 }) + const child = spawn(process.execPath, ['-e', `const Database=require('better-sqlite3');const db=new Database(process.argv[1]);db.exec('BEGIN IMMEDIATE');process.send('locked');setInterval(()=>{},1000)`, join(root, '.writer.sqlite')], { stdio: ['ignore', 'ignore', 'pipe', 'ipc'] }) + const closed = once(child, 'close') + try { + await once(child, 'message') + await expect(withExperimentWorkspace(root, policy, async () => null)).rejects.toThrow('locked') + child.kill('SIGKILL') + await closed + expect(await withExperimentWorkspace(root, policy, async () => 'recovered')).toBe('recovered') + } + finally { + child.kill('SIGKILL') + await closed + } +}) + +it('serializes concurrent publications under the same quota snapshot', async () => { + await withExperimentWorkspace(root, policy, async (workspace) => { + const first = workspace.publishReport(join(workspace.reports, 'first.json'), { value: 'a'.repeat(30) }) + const second = workspace.publishReport(join(workspace.reports, 'second.json'), { value: 'b'.repeat(30) }) + const results = await Promise.allSettled([first, second]) + expect(results.map(result => result.status)).toEqual(['fulfilled', 'rejected']) + expect(await readdir(workspace.reports)).toEqual(['first.json']) + }) +}) diff --git a/scripts/algorithm-experiments/workspace.ts b/scripts/algorithm-experiments/workspace.ts new file mode 100644 index 000000000..2597e865b --- /dev/null +++ b/scripts/algorithm-experiments/workspace.ts @@ -0,0 +1,100 @@ +import Database from 'better-sqlite3' +import { constants } from 'node:fs' +import { lstat, mkdir, open, opendir, statfs } from 'node:fs/promises' +import { dirname, join, resolve } from 'node:path' +import { z } from 'zod' +import { publishJson } from './files' + +const integer = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) +export const workspacePolicySchema = z.object({ + maximumReportBytes: integer.positive().max(64 * 1024 ** 3), + maximumReportFiles: integer.positive().max(100000), + maximumSingleReportBytes: integer.positive().max(32 * 1024 ** 2), + maximumDatabaseBytes: integer.min(65536).max(1024 ** 3), + minimumFreeBytes: integer.max(1024 ** 4), +}).strict() +/** Off-pod workstation limits, not pod production budgets. */ +export const defaultWorkspacePolicy = Object.freeze({ maximumReportBytes: 1024 ** 3, maximumReportFiles: 2048, + maximumSingleReportBytes: 32 * 1024 ** 2, maximumDatabaseBytes: 64 * 1024 ** 2, minimumFreeBytes: 1024 ** 3 }) + +async function privateDirectory(path: string) { + await mkdir(path, { recursive: true, mode: 0o700 }) + const info = await lstat(path) + if (!info.isDirectory() || info.isSymbolicLink() || (info.mode & 0o077) || (process.getuid && info.uid !== process.getuid())) throw new Error('Experiment workspace must be private and owned') +} +async function privateFile(path: string, missing = false) { + try { + const info = await lstat(path) + if (!info.isFile() || info.isSymbolicLink() || info.nlink !== 1 || (info.mode & 0o077) + || (process.getuid && info.uid !== process.getuid())) throw new Error('Unsafe experiment workspace file') + return info + } + catch (error) { + if (missing && (error as NodeJS.ErrnoException).code === 'ENOENT') return null + throw error + } +} + +/** A separate SQLite write lock survives async work and is released by the OS + * on controller death. Never remove its inode or steal ownership by timeout. */ +export async function withExperimentWorkspace(directory: string, rawPolicy: unknown, + action: (workspace: { reports: string, configureDatabase: (db: Database.Database) => void, publishReport: typeof publishJson, checkCapacity: () => Promise }) => Promise) { + const policy = workspacePolicySchema.parse(rawPolicy), root = resolve(directory) + await privateDirectory(root) + const lockPath = join(root, '.writer.sqlite') + try { + const file = await open(lockPath, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY | constants.O_NOFOLLOW, 0o600) + await file.close() + } + catch (error) { if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error } + for (const suffix of ['', '-wal', '-shm', '-journal']) { + const info = await privateFile(lockPath + suffix, suffix !== '') + if (info && info.size > 65536) throw new Error('Workspace lock exceeds its size bound') + } + const lock = new Database(lockPath, { timeout: 0 }) + let publication: Promise = Promise.resolve() + try { + lock.exec('BEGIN IMMEDIATE') + if ((lock.prepare('SELECT COUNT(*) AS count FROM sqlite_master').get() as { count: number }).count !== 0) throw new Error('Unexpected workspace lock database') + const reports = join(root, 'reports') + await privateDirectory(reports) + const capacity = async (additionalBytes = 0, additionalFiles = 0) => { + let count = additionalFiles, bytes = additionalBytes + const entries = await opendir(reports) + for await (const entry of entries) { + if (++count > policy.maximumReportFiles) throw new Error('Experiment report file quota exceeded') + const info = await privateFile(join(reports, entry.name)) + bytes += info?.size ?? 0 + if (bytes > policy.maximumReportBytes) throw new Error('Experiment report byte quota exceeded') + } + if (count > policy.maximumReportFiles || bytes > policy.maximumReportBytes) throw new Error('Experiment report quota exceeded') + const disk = await statfs(root, { bigint: true }) + // Reserve for a complete rollback journal and growth of the bounded ledger. + const required = BigInt(policy.minimumFreeBytes) + BigInt(2 * policy.maximumDatabaseBytes) + BigInt(additionalBytes) + if (disk.bavail < 0n || disk.bsize <= 0n || disk.bavail * disk.bsize < required) throw new Error('Experiment workspace free-space reserve unavailable') + } + await capacity() + return await action({ reports, checkCapacity: () => capacity(), configureDatabase: (db) => { + const pageSize = Number(db.pragma('page_size', { simple: true })) + if (!Number.isSafeInteger(pageSize) || pageSize < 512) throw new Error('Invalid experiment database page size') + const pages = Math.floor(policy.maximumDatabaseBytes / pageSize) + if (Number(db.pragma(`max_page_count = ${pages}`, { simple: true })) > pages) throw new Error('Experiment database quota exceeded') + if (db.pragma('journal_mode = DELETE', { simple: true }) !== 'delete') throw new Error('Bounded experiment rollback journal required') + }, publishReport: async (path, value) => { + if (dirname(resolve(path)) !== reports) throw new Error('Report path outside managed workspace') + const snapshot = structuredClone(value) + const bytes = Buffer.byteLength(JSON.stringify(snapshot, null, 2) + '\n') + if (bytes > policy.maximumSingleReportBytes) throw new Error('Experiment single-report quota exceeded') + const result = publication.then(async () => { + await capacity(bytes, 1) + await publishJson(path, snapshot) + }) + publication = result.catch(() => {}) + return result + } }) + } + finally { + await publication + lock.close() + } +} diff --git a/scripts/benchmarks/biometrics-live-shadow-runner.ts b/scripts/benchmarks/biometrics-live-shadow-runner.ts new file mode 100644 index 000000000..39d64eaca --- /dev/null +++ b/scripts/benchmarks/biometrics-live-shadow-runner.ts @@ -0,0 +1,133 @@ +/** Operational measurement only; never installs a qualification or changes profiles. + * Run on the pod: STAGE APP_ROOT PYTHON MODULES DATA_DIR [SECONDS=180] + * Live frames stay on the pod. Only summary metadata may be exported afterward. + */ +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { createHash } from 'node:crypto' +import { mkdtemp, readFile, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { setTimeout as delay } from 'node:timers/promises' +import WebSocket from 'ws' +import { defaultConfig, manifestHash } from '../../src/biometrics/algorithms/registry' +import { configHash, dataHash } from '../../src/biometrics/algorithms/canonical' +import { ShadowStore } from '../../src/biometrics/shadow/store' +import { runShadowAttempt } from '../../src/biometrics/shadow/attempt' +import { plannedShadowSourceFactory } from '../../src/biometrics/shadow/planned-sources' +import { startShadowHealthMonitor } from '../../src/biometrics/shadow/health-monitor' +import { collectAdmissionSample, evaluateAdmission, type AdmissionSample } from '../../src/biometrics/backtest/admission' +import { probeShadowBuild } from '../../src/biometrics/shadow/build' +import { publishProspectiveFrame, failProspectiveFrames } from '../../src/streaming/prospectiveFrameTap' + +async function main() { + const [stage, app, pythonPath, modulesPath, data, rawSeconds = '180'] = process.argv.slice(2) + const seconds = Number(rawSeconds) + if (!stage || !app || !pythonPath || !modulesPath || !data || !Number.isInteger(seconds) || seconds < 150 || seconds > 600) throw new Error('Invalid measurement arguments') + const directory = await mkdtemp(join(stage, 'live-shadow-')) + const configurationDatabase = join(data, 'sleepypod.db'), biometricsDatabase = join(data, 'biometrics.db') + const configuration = new Database(configurationDatabase, { readonly: true, fileMustExist: true, timeout: 100 }) + const sink = new Database(join(directory, 'isolated.db')) + sink.pragma('journal_mode = WAL') + migrate(drizzle(sink), { migrationsFolder: join(app, 'src/db/biometrics-migrations') }) + const controller = new AbortController(), socket = new WebSocket('ws://127.0.0.1:3001', { maxPayload: 128 * 1024 }) + let piezoAt: number | undefined, received = 0, closed = false + socket.on('open', () => socket.send(JSON.stringify({ type: 'subscribe', sensors: ['piezo-dual', 'frzHealth'] }))) + socket.on('message', (bytes) => { + try { + const frame = JSON.parse(bytes.toString()) + if (!['piezo-dual', 'frzHealth'].includes(frame.type)) return + const now = Date.now() + if (frame.type === 'piezo-dual') piezoAt = now + received++ + publishProspectiveFrame(frame, false, now) + } + catch { + controller.abort() + failProspectiveFrames() + } + }) + socket.on('error', () => { + controller.abort() + failProspectiveFrames() + }) + socket.on('close', () => { + if (!closed) { + controller.abort() + failProspectiveFrames() + } + }) + let monitor: Awaited> | undefined + let runId: string | undefined, result: unknown = null, failure: string | null = null + let lastHealth: AdmissionSample | null = null, refusal: unknown = null + const store = new ShadowStore(sink, { maximumRuns: 1, maximumPoints: 2000, maximumBytes: 8 * 1024 * 1024 }) + try { + const deadline = Date.now() + 30000 + while (!piezoAt && Date.now() < deadline && !controller.signal.aborted) await delay(100) + if (!piezoAt || controller.signal.aborted) throw new Error('No live piezo stream') + const assertDefaults = () => { + if (!configuration.readonly || (configuration.prepare('SELECT count(*) AS count FROM biometric_profile_bindings').get() as { count: number }).count !== 0) throw new Error('Measurement requires unchanged unpersonalized defaults') + } + assertDefaults() + const policy = { qualificationId: 'live-shadow-operational-measurement-only', minimumAvailableMemoryBytes: 768 * 1024 * 1024, + minimumFreeDiskBytes: 2 * 1024 * 1024 * 1024, maximumLoadPerCpu: 1.1, maximumEventLoopDelayMs: 100, + maximumPiezoAgeMs: 5000, maximumTemperatureC: 55, maximumSampleAgeMs: 2000 } + monitor = await startShadowHealthMonitor(async () => { + lastHealth = await collectAdmissionSample(directory, () => piezoAt) + return lastHealth + }, policy, controller.signal) + const hash = configHash(defaultConfig), python = await probeShadowBuild({ pythonPath, modulesPath }) + const runnerHash = createHash('sha256').update(await readFile(process.argv[1])).digest('hex') + const startMs = Date.now() + 2000, endMs = startMs + seconds * 1000 + const plan = { schema: 'biometric-shadow-plan-v1', side: 'left', subjectId: 'local-operational-measurement', startMs, endMs, + candidateConfigHash: hash, activeConfigHash: hash, qualificationHash: dataHash(policy), engineBuildHash: dataHash({ runnerHash, python: python.hash }), + maximumPoints: 2000, maximumBytes: 8 * 1024 * 1024, metrics: ['heart_rate'], + joining: { maximumPendingPoints: 200, maximumLatenessMs: 60000, drainGraceMs: 65000 }, + monitoring: { expectedIntervalMs: 60000, maximumGapMs: 120000, heartbeatStaleMs: 2000, minimumCandidateCoverage: 0.01, + maximumCoverageLoss: 1, minimumQuality: 0, alignment: { version: 'reference-alignment-v1', pointToleranceMs: 0, clockOffsetMs: 0, offsetSource: 'fixed' } }, + execution: { version: 'shadow-execution-policy-v1', configHashes: { left: hash, right: hash }, maximumRecords: 10000, maximumOutputs: 2000, + maximumCpuMs: 120000, maximumInputBytes: 64 * 1024 * 1024, maximumOutputBytes: 8 * 1024 * 1024, + budgets: { cpuSeconds: 120, addressSpaceBytes: 768 * 1024 * 1024, maximumRssBytes: 384 * 1024 * 1024, + wallMs: (seconds + 90) * 1000, acknowledgmentMs: 30000, idleMs: 10000, rssIntervalMs: 100, rssTimeoutMs: 1000 }, + leaseMs: 5000, heartbeatMs: 500, sources: { pollMs: 500, maximumActiveRows: 100, databaseBusyTimeoutMs: 100, + frames: { maximumFrames: 30, maximumBytes: 4 * 1024 * 1024, maximumEventAgeMs: 10000, maximumFutureSkewMs: 1000 }, + context: { presenceStaleMs: 300000, pumpStaleMs: 5000, maximumPresenceBytes: 65536 } } } } + const created = store.create(plan, 'measurement'), openSources = plannedShadowSourceFactory(plan, + { configurationDatabase, biometricsDatabase, presenceFile: join(data, 'sleep-detector-state.json') }) + runId = created.id + await delay(Math.max(0, startMs - Date.now())) + const attempt = await runShadowAttempt(store, runId, { location: { pythonPath, modulesPath }, + configs: { left: defaultConfig, right: defaultConfig }, openSources, admit: () => { + assertDefaults() + if (!monitor?.read()) { + refusal = { checkedAtMs: Date.now(), sample: lastHealth, decision: evaluateAdmission(lastHealth, policy) } + throw new Error('Measurement health admission refused') + } + } }, controller.signal) + result = { runnerHash, pythonBuildHash: python.hash, manifestHash, receipt: attempt.receipt } + } + catch (error) { + failure = error instanceof Error ? error.message : String(error) + process.exitCode = 1 + } + finally { + closed = true + controller.abort() + monitor?.close() + socket.terminate() + const run = runId ? store.get(runId) : null + const canonicalRows = (sink.prepare('SELECT count(*) AS count FROM vitals').get() as { count: number }).count + await writeFile(join(stage, 'live-shadow-summary.json'), JSON.stringify({ + version: 'live-shadow-operational-measurement-v1', scope: 'loopback live input and read-only active observations with isolated sink; not production registry admission or accuracy qualification', + seconds, receivedFrames: received, state: run?.state ?? null, reason: run?.reason ?? null, + pointCount: run?.point_count ?? 0, byteCount: run?.byte_count ?? 0, canonicalRowsInIsolatedSink: canonicalRows, + result, failure, refusal, workerReceipt: run?.workerReceipt ?? null, + }, null, 2)) + configuration.close() + sink.close() + } +} +void main().catch((error) => { + console.error(error) + process.exitCode = 1 +}) diff --git a/scripts/benchmarks/biometrics-probe.py b/scripts/benchmarks/biometrics-probe.py new file mode 100644 index 000000000..8d3036515 --- /dev/null +++ b/scripts/benchmarks/biometrics-probe.py @@ -0,0 +1,88 @@ +"""Independent, read-only loopback telemetry probe around a benchmark command. + +Usage: python3 biometrics-probe.py OUTPUT.json COMMAND [ARG ...] +Never retains response bodies or sensor samples. +""" +import json +import os +import signal +import pathlib +import subprocess +import sys +import threading +import time +import urllib.request + +samples = [] +phase = "baseline" +stop = threading.Event() +benchmark_pid = None + + +def process_tree_memory(root_pid): + if root_pid is None: + return 0, 0 + processes = {} + for path in pathlib.Path('/proc').glob('[0-9]*/status'): + try: + fields = dict(line.split(':', 1) for line in path.read_text().splitlines() if ':' in line) + processes[int(path.parent.name)] = (int(fields['PPid']), int(fields.get('VmRSS', '0 kB').split()[0])) + except (OSError, ValueError, KeyError): + continue + selected = {root_pid} + while True: + children = {pid for pid, (parent, _) in processes.items() if parent in selected} + if children <= selected: + break + selected |= children + return sum(processes.get(pid, (0, 0))[1] for pid in selected), len(selected & processes.keys()) + + +def sample(): + while not stop.is_set(): + row = {"phase": phase, "at": time.monotonic()} + cpu = list(map(int, pathlib.Path("/proc/stat").read_text().splitlines()[0].split()[1:])) + row["cpuTotal"] = sum(cpu[:8]) + row["cpuIdle"] = cpu[3] + cpu[4] + row['benchmarkTreeRssKiB'], row['benchmarkProcessCount'] = process_tree_memory(benchmark_pid) + memory = pathlib.Path('/proc/meminfo').read_text().splitlines() + row['memAvailableKiB'] = int(next(line.split()[1] for line in memory if line.startswith('MemAvailable:'))) + row['temperatureC'] = int(pathlib.Path('/sys/class/thermal/thermal_zone0/temp').read_text()) / 1000 + for endpoint in ["health/performance", "system/sensor-source", "device/status"]: + began = time.monotonic() + try: + with urllib.request.urlopen("http://127.0.0.1:3000/api/" + endpoint, timeout=2) as response: + response.read() + status = response.status + row[endpoint] = {"ms": (time.monotonic() - began) * 1000, "status": status} + except Exception as error: + row[endpoint] = {"ms": (time.monotonic() - began) * 1000, "error": type(error).__name__} + samples.append(row) + stop.wait(0.25) + + +if len(sys.argv) < 3: + raise SystemExit("OUTPUT.json COMMAND [ARG ...] required") +timeout_seconds = int(os.environ.get('BIOMETRICS_BENCHMARK_TIMEOUT_SECONDS', '600')) +if not 1 <= timeout_seconds <= 10800: + raise SystemExit('Benchmark timeout must be between 1 and 10800 seconds') +thread = threading.Thread(target=sample) +thread.start() +try: + time.sleep(4) + phase = "benchmark" + result = subprocess.Popen(sys.argv[2:], start_new_session=True) + benchmark_pid = result.pid + try: + result.wait(timeout=timeout_seconds) + except subprocess.TimeoutExpired: + os.killpg(result.pid, signal.SIGKILL) + result.wait() + benchmark_pid = None + phase = "after" + time.sleep(3) +finally: + stop.set() + thread.join() + pathlib.Path(sys.argv[1]).write_text(json.dumps({"version": "read-only-biometric-probe-v2", "timeoutSeconds": timeout_seconds, "samples": samples}, indent=2)) +raise SystemExit(result.returncode) diff --git a/scripts/benchmarks/biometrics-runner.ts b/scripts/benchmarks/biometrics-runner.ts new file mode 100644 index 000000000..db13a78a3 --- /dev/null +++ b/scripts/benchmarks/biometrics-runner.ts @@ -0,0 +1,158 @@ +/** Synthetic-only qualification harness. Bundle with esbuild (better-sqlite3 external). + * Pass NEW_DIRECTORY MIGRATIONS WORKER [mixed|dense] [COORDINATOR]. + * Omitting COORDINATOR measures the old, synchronous calling-process path. + * Reads only loopback sensor/API telemetry; never opens the application databases. + */ +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs' +import { resolve, join } from 'node:path' +import { monitorEventLoopDelay, performance } from 'node:perf_hooks' +import { createHash } from 'node:crypto' +import WebSocket from 'ws' +import * as schema from '../../src/db/biometrics-schema' +import { DatasetStore } from '../../src/biometrics/backtest/datasets' +import { BacktestJobStore } from '../../src/biometrics/backtest/jobs' +import { runCoordinatorTask } from '../../src/biometrics/backtest/coordinator' +import { executeNextBacktest } from '../../src/biometrics/backtest/runner' +import { workerBuildHash } from '../../src/biometrics/backtest/child' +import { defaultConfig, manifestHash } from '../../src/biometrics/algorithms/registry' +import { configHash } from '../../src/biometrics/algorithms/canonical' +import { ReferenceStore } from '../../src/biometrics/references/store' +import { previewReferenceImport } from '../../src/biometrics/references/normalize' + +async function main() { + const [directoryArg, migrationsArg, workerArg, fixture = 'mixed', coordinatorArg] = process.argv.slice(2) + if (!directoryArg || !migrationsArg || !workerArg) throw new Error('NEW_DIRECTORY MIGRATIONS WORKER required') + if (!['mixed', 'dense'].includes(fixture)) throw new Error('Fixture must be mixed or dense') + const dense = fixture === 'dense' + const directory = resolve(directoryArg) + // Exclusive creation prevents accidentally benchmarking against an existing DB. + mkdirSync(directory, { mode: 0o700 }) + const sqlite = new Database(join(directory, 'synthetic.db')) + sqlite.pragma('journal_mode = WAL') + const db = drizzle(sqlite, { schema }) + migrate(db, { migrationsFolder: resolve(migrationsArg) }) + const workerPath = resolve(workerArg), evaluatorBuildHash = await workerBuildHash(workerPath) + const sessionIds: number[] = [] + const references: Array> = [] + const day = 86400000, start = Date.UTC(2026, 0, 1, 20) + sqlite.transaction(() => { + for (let night = 0; night < 20; night++) { + const from = start + night * day, id = night + 1 + sessionIds.push(id) + db.insert(schema.sleepRecords).values({ id, side: 'left', enteredBedAt: new Date(from), leftBedAt: new Date(from + 12 * 3600000), sleepDurationSeconds: 43200, timesExitedBed: 0 }).run() + for (let row = 0; row < (dense ? 950 : 720); row++) { + db.insert(schema.vitals).values({ side: 'left', timestamp: new Date(from + row * (dense ? 45000 : 60000)), heartRate: row % 47 === 0 ? null : 55 + 12 * Math.sin(row / 30), hrv: 30 + row % 40, breathingRate: 12 + row % 7 }).run() + if (!dense && row % 4 === 0) db.insert(schema.movement).values({ side: 'left', timestamp: new Date(from + row * (dense ? 45000 : 60000)), totalMovement: row % 32 === 0 ? 250 : row % 60 }).run() + } + for (let row = 0; row < (dense ? 20 : 40); row++) { + const timestampMs = from + row * (dense ? 2160000 : 1080000) + references.push({ recordKey: `${night}:hr:${row}`, metric: 'heart_rate', value: 58 + row % 12, unit: 'bpm', timestampMs }) + references.push({ recordKey: `${night}:stage:${row}`, metric: 'sleep_stages', value: ['wake', 'light', 'deep', 'rem'][row % 4], unit: 'stage', startMs: timestampMs, endMs: timestampMs + (dense ? 2160000 : 1080000) }) + } + } + })() + const imported = { side: 'left' as const, subjectId: 'synthetic-qualification', source: 'synthetic-v1', rows: references } + const preview = previewReferenceImport(imported) + const reference = new ReferenceStore(db).commit({ import: imported, previewHash: preview.summary.contentHash }) + const coordinator = coordinatorArg ? { databasePath: join(directory, 'synthetic.db'), coordinatorPath: resolve(coordinatorArg), workerPath, modulePath: process.env.NODE_PATH } : null + const snapshotHistogram = monitorEventLoopDelay({ resolution: 20 }) + snapshotHistogram.enable() + await new Promise(resolve => setTimeout(resolve, 50)) + const snapshotStart = performance.now() + const datasetInput = { side: 'left' as const, subjectId: imported.subjectId, sessionIds, referenceImportIds: [reference.id], timezone: 'UTC', idempotencyKey: 'synthetic-v1' } + const dataset = coordinator ? await runCoordinatorTask(coordinator, { type: 'createDataset', input: datasetInput }) : new DatasetStore(db).create(datasetInput) + const snapshotMs = performance.now() - snapshotStart + await new Promise(resolve => setTimeout(resolve, 50)) + const snapshotEventLoopMaxMs = snapshotHistogram.max / 1e6 + snapshotHistogram.disable() + const originalHash = () => createHash('sha256').update(JSON.stringify(sqlite.prepare('SELECT * FROM vitals ORDER BY id').all())).digest('hex') + const beforeHash = originalHash() + const jobs = new BacktestJobStore(db) + const protocol = { version: 'descriptive-evaluation-v1' as const, mode: 'stage_replay' as const, assumedStageCalibrationQuality: 1, pointToleranceMs: 30000, clockOffsetMs: 0 } + const experiment = coordinator ? await runCoordinatorTask(coordinator, { type: 'createExperiment', input: { datasetId: dataset.id, protocol, idempotencyKey: 'synthetic-experiment' } }) : jobs.createExperiment(dataset.id, protocol) + const configs = (['default', 'current', 'candidate'] as const).map((label, index) => { + const resolved = structuredClone(defaultConfig) + // Synthetic comparison roles: this benchmark tests resources, not promotion. + resolved.sleep_stages.ruleset.rules[1].when[0].value = 0.90 + index * 0.02 + return { label, revisionId: null, manifestHash, resolvedHash: configHash(resolved), resolved } + }) + let phase = 'baseline', stopped = false, piezoAt: number | null = null, piezoSourceAt: number | null = null + const samples: unknown[] = [], runs: unknown[] = [] + const socket = new WebSocket('ws://127.0.0.1:3001') + socket.on('open', () => socket.send(JSON.stringify({ type: 'subscribe', sensors: ['piezo-dual'] }))) + socket.on('error', () => {}) + socket.on('message', (bytes) => { + try { + const frame = JSON.parse(bytes.toString()) + if (frame.type === 'piezo-dual') { + piezoAt = Date.now() + piezoSourceAt = typeof frame.ts === 'number' ? frame.ts * 1000 : null + } + } + catch { /* No frame contents are retained. */ } + }) + const histogram = monitorEventLoopDelay({ resolution: 20 }) + histogram.enable() + const sampler = (async () => { + while (!stopped) { + const sample: Record = { phase, atMs: Date.now(), parentRss: process.memoryUsage().rss, piezoAgeMs: piezoAt === null ? null : Date.now() - piezoAt, piezoSourceAgeMs: piezoSourceAt === null ? null : Date.now() - piezoSourceAt } + sample.memAvailableKiB = Number(/^MemAvailable:\s+(\d+)/m.exec(readFileSync('/proc/meminfo', 'utf8'))?.[1]) + sample.temperatureC = Number(readFileSync('/sys/class/thermal/thermal_zone0/temp', 'utf8')) / 1000 + for (const path of ['health/performance', 'device/status']) { + const began = performance.now() + try { + const response = await fetch(`http://127.0.0.1:3000/api/${path}`, { signal: AbortSignal.timeout(2000) }) + await response.arrayBuffer() + sample[path] = { ms: performance.now() - began, status: response.status } + } + catch { sample[path] = { failed: true, ms: performance.now() - began } } + } + samples.push(sample) + await new Promise(resolve => setTimeout(resolve, 250)) + } + })() + try { + await new Promise(resolve => setTimeout(resolve, 4000)) + for (let attempt = 0; attempt < 3; attempt++) { + const job = jobs.enqueue({ evaluatorBuildHash, experimentId: experiment.id, configs, split: 'tuning', idempotencyKey: `run-${attempt}` }) + phase = `replay-${attempt}` + histogram.reset() + const began = performance.now() + const outcome = coordinator + ? await runCoordinatorTask(coordinator, { type: 'execute', owner: 'synthetic-qualification' }, { admit: async () => true }) + : await executeNextBacktest(db, { owner: 'synthetic-qualification', workerPath, admit: async () => true, timeoutMs: 120000 }) + const elapsedMs = performance.now() - began + // Let timers observe the final synchronous publication before reading delay. + await new Promise(resolve => setTimeout(resolve, 50)) + const readStarted = performance.now(), result = coordinator ? await runCoordinatorTask(coordinator, { type: 'readResult', input: { id: job.id } }) : jobs.result(job.id) + const resultReadMs = performance.now() - readStarted + const report = result?.report as { operational?: unknown } | undefined + const reportBytes = Buffer.byteLength(JSON.stringify(report ?? null)) + // Include IPC deserialization and report serialization in the calling-loop maximum. + await new Promise(resolve => setTimeout(resolve, 50)) + runs.push({ outcome, elapsedMs, resultReadMs, available: result?.available, reportBytes, operational: report?.operational, + parentEventLoopP95Ms: histogram.percentile(95) / 1e6, parentEventLoopMaxMs: histogram.max / 1e6, parentPeakRssKiB: process.resourceUsage().maxRSS }) + if (outcome.state !== 'succeeded') throw new Error(`Qualification run failed: ${jobs.get(job.id).error}`) + } + phase = 'after' + await new Promise(resolve => setTimeout(resolve, 3000)) + } + finally { + stopped = true + await sampler + socket.terminate() + histogram.disable() + const output = { version: 'synthetic-runner-qualification-v1', fixture, coordinator: coordinator !== null, evaluatorBuildHash, datasetHash: dataset.contentHash, snapshotMs, snapshotEventLoopMaxMs, + canonicalObservationsUnchanged: beforeHash === originalHash(), runs, samples } + writeFileSync(join(directory, 'measurements.json'), JSON.stringify(output, null, 2)) + sqlite.close() + console.log(JSON.stringify({ directory, evaluatorBuildHash, runs, canonicalObservationsUnchanged: output.canonicalObservationsUnchanged })) + } +} +void main().catch((error) => { + console.error(error) + process.exitCode = 1 +}) diff --git a/scripts/benchmarks/biometrics-shadow-runner.ts b/scripts/benchmarks/biometrics-shadow-runner.ts new file mode 100644 index 000000000..e966b85ce --- /dev/null +++ b/scripts/benchmarks/biometrics-shadow-runner.ts @@ -0,0 +1,72 @@ +/** Synthetic-only target-pod shadow supervisor measurement. Bundle with esbuild. + * Arguments: OUTPUT_JSON PYTHON_PATH MODULES_PATH [SECONDS=1800] + * Uses no live inputs, databases, admission registry or hardware interfaces. + * Run alone: the shipped child requires the shared kernel replay/shadow slot. + */ +import { randomUUID, createHash } from 'node:crypto' +import { readFile, writeFile } from 'node:fs/promises' +import { monitorEventLoopDelay } from 'node:perf_hooks' +import { defaultConfig, manifestHash } from '../../src/biometrics/algorithms/registry' +import { configHash } from '../../src/biometrics/algorithms/canonical' +import { runShadowChild } from '../../src/biometrics/shadow/worker-child' +import { probeShadowBuild } from '../../src/biometrics/shadow/build' +import type { ShadowPartialReceipt } from '../../src/biometrics/shadow/receipt' + +async function main() { + const [output, pythonPath, modulesPath, duration = '1800'] = process.argv.slice(2) + const seconds = Number(duration) + if (!output || !pythonPath || !modulesPath || !Number.isInteger(seconds) || seconds < 150 || seconds > 28800) { + throw new Error('Expected OUTPUT_JSON PYTHON_PATH MODULES_PATH and 150–28800 synthetic seconds') + } + const pythonBuild = await probeShadowBuild({ pythonPath, modulesPath }) + const runnerHash = createHash('sha256').update(await readFile(process.argv[1])).digest('hex') + const startMs = 1700000000000, hash = configHash(defaultConfig) + // Measurement stop conditions, not deployment qualification or recommended limits. + const budgets = { cpuSeconds: 1200, addressSpaceBytes: 768 * 1024 * 1024, maximumRssBytes: 384 * 1024 * 1024, + wallMs: 1800000, acknowledgmentMs: 30000, idleMs: 30000, rssIntervalMs: 100, rssTimeoutMs: 1000 } + const init = { schema: 'biometric-shadow-worker-v1', runId: randomUUID(), attemptId: randomUUID(), side: 'left', + startMs, endMs: startMs + seconds * 1000, configs: { left: defaultConfig, right: defaultConfig }, + configHashes: { left: hash, right: hash }, maximumRecords: seconds, maximumOutputs: seconds, + maximumCpuMs: 1200000, maximumInputBytes: seconds * 6500 + 128 * 1024, maximumOutputBytes: seconds * 4096 } + async function* frames() { + for (let second = 0; second < seconds; second++) { + const channel = Buffer.alloc(2000) + for (let sample = 0; sample < 500; sample++) { + const t = second + sample / 500 + channel.writeInt32LE(Math.trunc(1000000 * Math.sin(2 * Math.PI * 1.2 * t)), sample * 4) + } + yield { type: 'frame', sequence: second, observedAtMs: startMs + second * 1000, + record: { type: 'piezo-dual', left1: channel.toString('base64'), right1: Buffer.alloc(2000).toString('base64') }, + context: { activeSides: ['left'], leftOccupied: true, rightOccupied: false, pumpContextKnown: true } } + } + } + const histogram = monitorEventLoopDelay({ resolution: 20 }), digest = createHash('sha256') + let outputCount = 0, partial: ShadowPartialReceipt | null = null, receipt: unknown = null, failure: string | null = null + histogram.enable() + const began = performance.now() + try { + receipt = await runShadowChild({ pythonPath, modulesPath }, init, frames(), (row) => { + outputCount++ + digest.update(JSON.stringify(row)).update('\n') + }, { budgets, onResources: (value) => { partial = value } }) + } + catch (error) { + failure = error instanceof Error ? error.message : String(error) + process.exitCode = 1 + } + finally { + histogram.disable() + await writeFile(output, JSON.stringify({ version: 'synthetic-shadow-supervisor-measurement-v1', + scope: 'accelerated synthetic frames through shipped supervisor and kernel-limited child; no live capture, database sink, held-out accuracy or installation qualification', + runnerHash, pythonBuildHash: pythonBuild.hash, pythonBuildIdentity: pythonBuild.identity, + node: { version: process.version, platform: process.platform, arch: process.arch }, + manifestHash, configHash: hash, seconds, sampleRateHz: 500, channels: 2, wallMs: performance.now() - began, + outputCount, outputHash: digest.digest('hex'), receipt, partial, failure, + parentEventLoopP95Ms: histogram.percentile(95) / 1e6, parentEventLoopMaxMs: histogram.max / 1e6, + parentPeakRssKiB: process.resourceUsage().maxRSS }, null, 2)) + } +} +void main().catch((error) => { + console.error(error) + process.exitCode = 1 +}) diff --git a/scripts/benchmarks/biometrics-waveform-fixture.py b/scripts/benchmarks/biometrics-waveform-fixture.py new file mode 100644 index 000000000..25629f77c --- /dev/null +++ b/scripts/benchmarks/biometrics-waveform-fixture.py @@ -0,0 +1,46 @@ +"""Create bounded deterministic synthetic dual-channel archives, never live data.""" +import argparse +from dataclasses import asdict +import gzip +import json +from pathlib import Path +import time + +import cbor2 +import numpy as np +from common.biometrics.raw_sources import ArchiveSource + +parser = argparse.ArgumentParser() +parser.add_argument('new_directory') +parser.add_argument('--seconds', type=int, default=1800) +args = parser.parse_args() +if not 600 <= args.seconds <= 28800: + raise SystemExit('Fixture duration must be between 600 seconds and 8 hours') +root = Path(args.new_directory) +root.mkdir(mode=0o700) # Never overwrite an existing archive directory. +start = 1767297600000 +began = time.monotonic() +rng = np.random.default_rng(814) +inspections = [] +for segment, offset in enumerate(range(0, args.seconds, 7200)): + name = '%03d.RAW.gz' % segment + with (root / name).open('xb') as output: + with gzip.GzipFile(filename='', fileobj=output, mode='wb', compresslevel=1, mtime=0) as stream: + for second in range(offset, min(offset + 7200, args.seconds)): + timestamp = start / 1000 + second + records = [] + if second % 30 == 0: + records.append({'type': 'frzHealth', 'ts': timestamp, + 'left': {'pumpRpm': 0}, 'right': {'pumpRpm': 0}}) + t = (np.arange(500) + second * 500) / 500 + left = (np.sin(t * 2 * np.pi * 1.2) * 1000000 + rng.normal(0, 100000, 500)).astype(' resolve(value)) + const fixture = z.object({ version: z.literal('synthetic-waveform-fixture-v1'), startMs: z.number(), endMs: z.number(), seconds: z.number(), + generationSeconds: z.number(), inspections: z.array(rawInspectionSchema).min(1).max(4) }).strict().parse(JSON.parse(readFileSync(fixturePath, 'utf8'))) + mkdirSync(directory, { mode: 0o700 }) + setPriority(process.pid, 19) + const sqlite = new Database(join(directory, 'synthetic.db')) + sqlite.pragma('journal_mode = WAL') + const db = drizzle(sqlite, { schema }) + migrate(db, { migrationsFolder: migrations }) + const replayBudget = process.env.SLEEPYPOD_BIOMETRICS_REPLAY_BUDGET === undefined + ? undefined + : replayBudgetSchema.parse(JSON.parse(process.env.SLEEPYPOD_BIOMETRICS_REPLAY_BUDGET)) + const options = { replayBudget, databasePath: join(directory, 'synthetic.db'), workerPath, coordinatorPath, + modulePath: process.env.NODE_PATH, rawLocation: { pythonPath, modulesPath, archiveDirectory } } + const coordinator = new CoordinatorQueue(options) + let retainedDatasetId: string | undefined + const start = fixture.startMs + 300000 + db.insert(schema.sleepRecords).values({ id: 1, side: 'left', enteredBedAt: new Date(start), leftBedAt: new Date(fixture.endMs), + sleepDurationSeconds: (fixture.endMs - start) / 1000, timesExitedBed: 0, presentIntervals: [[start / 1000, fixture.endMs / 1000]] }).run() + db.insert(schema.vitals).values({ side: 'left', timestamp: new Date(start), heartRate: 65 }).run() + const originalRows = JSON.stringify(sqlite.prepare('SELECT * FROM vitals').all()) + const sources = fixture.inspections.map(inspection => new RawSourceCatalog(db).record(inspection)) + let phase = 'baseline', stopped = false, piezoAt: number | undefined + let completed = false, readFailures = 0 + const samples: unknown[] = [], operations: unknown[] = [] + let lastHttpHealthy = false + const socket = new WebSocket('ws://127.0.0.1:3001') + socket.on('open', () => socket.send(JSON.stringify({ type: 'subscribe', sensors: ['piezo-dual'] }))) + socket.on('error', () => {}) + socket.on('message', (bytes) => { + try { + if (JSON.parse(bytes.toString()).type === 'piezo-dual') piezoAt = Date.now() + } + catch { /* Never retain sensor payloads. */ } + }) + const histogram = monitorEventLoopDelay({ resolution: 20 }) + histogram.enable() + const sampler = (async () => { + while (!stopped) { + const sample = await collectAdmissionSample(directory, () => piezoAt) + const requests = [] + for (const endpoint of ['health/performance', 'device/status']) { + const began = performance.now() + try { + const response = await fetch(`http://127.0.0.1:3000/api/${endpoint}`, { signal: AbortSignal.timeout(1500) }) + await response.arrayBuffer() + requests.push({ endpoint, status: response.status, ms: performance.now() - began }) + } + catch { requests.push({ endpoint, status: 0, ms: performance.now() - began }) } + } + let concurrentReadMs: number | undefined + if (phase === 'waveform-replay' && retainedDatasetId && samples.length % 10 === 0) { + const began = performance.now() + try { + await coordinator.request({ type: 'datasetDescriptor', input: { id: retainedDatasetId } }) + } + catch { readFailures++ } + concurrentReadMs = performance.now() - began + } + lastHttpHealthy = readFailures === 0 && requests.every(row => row.status === 200 && row.ms < 1000) + samples.push({ phase, sample, requests, concurrentReadMs, parentRss: process.memoryUsage().rss, loadPerCpu: loadavg()[0] / availableParallelism() }) + await new Promise(resolve => setTimeout(resolve, 500)) + } + })() + // Conservative stop conditions for measurement, not a production qualification policy. + const admit = async () => { + const sample = await collectAdmissionSample(directory, () => piezoAt) + return sample !== null && lastHttpHealthy && sample.availableMemoryBytes >= 512 * 1024 * 1024 + && sample.freeDiskBytes >= 1024 * 1024 * 1024 && sample.temperatureC <= 65 + && Date.now() - sample.piezoReceivedAtMs <= 10000 && sample.eventLoopDelayMs <= 100 + } + async function measure(name: string, action: () => Promise) { + phase = name + histogram.reset() + const began = performance.now() + let succeeded = false + try { + const result = await action() + succeeded = true + return result + } + finally { + await new Promise(resolve => setTimeout(resolve, 50)) + operations.push({ phase, succeeded, elapsedMs: performance.now() - began, eventLoopP95Ms: histogram.percentile(95) / 1e6, + eventLoopMaxMs: histogram.max / 1e6, parentPeakRssKiB: process.resourceUsage().maxRSS }) + } + } + try { + await new Promise(resolve => setTimeout(resolve, 8000)) + if (!await admit()) throw new Error('Live health is unavailable or outside conservative benchmark limits') + const parent = await measure('snapshot', () => coordinator.request({ type: 'createDataset', input: { side: 'left', subjectId: 'synthetic', + sessionIds: [1], referenceImportIds: [], timezone: 'UTC', idempotencyKey: 'synthetic-parent' } })) + const published = await measure('retain', () => coordinator.request({ type: 'createWaveformDataset', input: { + parentDatasetId: parent.id, expectedContentHash: parent.contentHash, idempotencyKey: 'synthetic-waveform', + cohorts: datasetManifestSchema.parse(parent.manifest).splitProtocol.assignments.map(cohort => ({ cohortId: cohort.cohortId, sourceIds: sources })), + } }, { admit })) + if (published.status !== 'published') throw new Error('Waveform retention was deferred') + retainedDatasetId = published.dataset.id + const build = await waveformEvaluatorBuildIdentity(coordinatorPath, workerPath, options.rawLocation) + const jobs = new BacktestJobStore(db) + const experiment = jobs.createExperiment(published.dataset.id, { version: 'descriptive-evaluation-v1', mode: 'signal_replay', waveformStages: true, + assumedStageCalibrationQuality: 0, pointToleranceMs: 30000, clockOffsetMs: 0 }) + const configs = (['default', 'current', 'candidate'] as const).map((label, index) => { + const resolved = structuredClone(defaultConfig) + resolved.heart_rate.trackingToleranceBpm = 15 + index * 2 + return { label, revisionId: null, manifestHash, resolvedHash: configHash(resolved), resolved } + }) + const job = jobs.enqueue({ experimentId: experiment.id, evaluatorBuildHash: build.hash, idempotencyKey: 'synthetic-signal', split: 'tuning', configs, + waveform: { pythonBuildHash: build.python.hash, otherSide: { manifestHash, resolvedHash: configHash(defaultConfig), resolved: defaultConfig } } }) + await measure('waveform-replay', async () => { + const outcome = await coordinator.request({ type: 'execute', owner: 'synthetic-qualification' }, { admit }) + if (outcome.state !== 'succeeded') throw new Error(`Waveform job stopped: ${jobs.get(job.id).error}`) + }) + const summary = await measure('summary', () => coordinator.request({ type: 'readSummary', input: { id: job.id } })) + if (!summary?.available || !summary.contentHash) throw new Error('Committed report summary unavailable') + const report = z.object({ version: z.literal('waveform-backtest-report-v1'), datasetHash: z.string(), + waveform: z.object({ outputs: z.array(z.object({ configHash: z.string(), run: z.object({ build_hash: z.string(), + counts: z.object({ scored_outputs: z.number().int().positive() }) }) })).length(3) }), + }).parse(summary.report) + if (report.datasetHash !== published.dataset.contentHash + || new Set(report.waveform.outputs.map(output => output.configHash)).size !== 3 + || report.waveform.outputs.some(output => output.run.build_hash !== build.python.hash || !configs.some(config => config.resolvedHash === output.configHash))) { + throw new Error('Qualification report identity or configuration coverage mismatch') + } + operations.push({ summaryBytes: Buffer.byteLength(JSON.stringify(summary)), resultHash: summary?.contentHash, + compoundBuildHash: build.hash, pythonBuildHash: build.python.hash }) + if (readFailures) throw new Error('Concurrent dataset reads failed') + completed = true + phase = 'after' + await new Promise(resolve => setTimeout(resolve, 3000)) + } + finally { + stopped = true + await sampler + await coordinator.stop() + socket.terminate() + histogram.disable() + const output = { version: 'synthetic-waveform-qualification-v1', completed, readFailures, replayBudget: replayBudget ?? null, fixtureSeconds: fixture.seconds, compressedBytes: fixture.inspections.reduce((sum, row) => sum + row.artifact.compressed_bytes, 0), + canonicalObservationsUnchanged: originalRows === JSON.stringify(sqlite.prepare('SELECT * FROM vitals').all()), operations, samples } + writeFileSync(join(directory, 'measurements.json'), JSON.stringify(output, null, 2), { mode: 0o600 }) + sqlite.close() + console.log(JSON.stringify({ directory, ...output, samples: output.samples.length })) + } +} +void main().catch((error) => { + console.error(error) + process.exitCode = 1 +}) diff --git a/scripts/build/biometrics-worker.mjs b/scripts/build/biometrics-worker.mjs new file mode 100644 index 000000000..d82a79cae --- /dev/null +++ b/scripts/build/biometrics-worker.mjs @@ -0,0 +1,51 @@ +import { build } from 'esbuild' +import { mkdir, writeFile } from 'node:fs/promises' + +const output = process.argv[2] ?? '.next/standalone/biometrics-worker' +await mkdir(output, { recursive: true }) +const result = await build({ + entryPoints: ['src/biometrics/backtest/worker-entry.ts'], + outfile: `${output}/worker.cjs`, + bundle: true, + platform: 'node', + target: 'node22', + format: 'cjs', + minify: false, + metafile: true, + treeShaking: true, +}) +// Fail packaging if the pure worker ever pulls in live application entrypoints. +const forbidden = Object.keys(result.metafile.inputs).filter(path => + /src\/(hardware|automation|scheduler|homekit|streaming)\//.test(path) + || /src\/db\//.test(path) || /drizzle-orm|better-sqlite3/.test(path), +) +if (forbidden.length) throw new Error(`Replay worker has live dependencies: ${forbidden.join(', ')}`) +await writeFile(`${output}/dependencies.json`, JSON.stringify(Object.keys(result.metafile.inputs).sort(), null, 2)) + +// The coordinator owns SQLite and artifact I/O, but never live application services. +const coordinator = await build({ + entryPoints: ['src/biometrics/backtest/coordinator-entry.ts'], + outfile: `${output}/coordinator.cjs`, bundle: true, platform: 'node', target: 'node22', + format: 'cjs', metafile: true, external: ['better-sqlite3'], +}) +const live = Object.keys(coordinator.metafile.inputs).filter(path => + /src\/(hardware|automation|scheduler|homekit|streaming)\//.test(path) + || /src\/db\/(index|biometrics)\.ts$/.test(path), +) +if (live.length) throw new Error(`Coordinator has live entrypoints: ${live.join(', ')}`) +await writeFile(`${output}/coordinator-dependencies.json`, JSON.stringify(Object.keys(coordinator.metafile.inputs).sort(), null, 2)) + +// The prospective host observes a shared passive tap. Live receive timestamps +// are injected by the service; bundling hardware/streaming entrypoints is forbidden. +const shadow = await build({ + entryPoints: ['src/biometrics/shadow/host-entry.ts'], + outfile: `${output}/shadow-host.cjs`, bundle: true, platform: 'node', target: 'node22', + format: 'cjs', metafile: true, external: ['better-sqlite3'], +}) +const shadowLive = Object.keys(shadow.metafile.inputs).filter(path => + /src\/(hardware|automation|scheduler|homekit)\//.test(path) + || (/src\/streaming\//.test(path) && !path.endsWith('/prospectiveFrameTap.ts')) + || /src\/db\/(index|biometrics)\.ts$/.test(path), +) +if (shadowLive.length) throw new Error(`Shadow host has live entrypoints: ${shadowLive.join(', ')}`) +await writeFile(`${output}/shadow-host-dependencies.json`, JSON.stringify(Object.keys(shadow.metafile.inputs).sort(), null, 2)) diff --git a/scripts/build/demo-biometric-worker.mjs b/scripts/build/demo-biometric-worker.mjs new file mode 100644 index 000000000..c62778d95 --- /dev/null +++ b/scripts/build/demo-biometric-worker.mjs @@ -0,0 +1,8 @@ +import { build } from 'esbuild' +import { createHash } from 'node:crypto' +import { writeFile } from 'node:fs/promises' +const result = await build({ entryPoints: ['src/demo/biometrics/worker.ts'], bundle: true, write: false, platform: 'browser', format: 'esm', metafile: true }) +const forbidden = Object.keys(result.metafile.inputs).filter(path => /src\/(hardware|db|automation|scheduler|homekit|streaming)\/|better-sqlite3|drizzle-orm/.test(path)) +if (forbidden.length) throw new Error(`Demo worker has live dependencies: ${forbidden.join(', ')}`) +const buildHash = createHash('sha256').update(result.outputFiles[0].contents).digest('hex') +await writeFile('src/demo/biometrics/worker-build.json', JSON.stringify({ buildHash, identity: 'esbuild-browser-evaluator-bundle', simulated: true }, null, 2) + '\n') diff --git a/scripts/deploy b/scripts/deploy index 69b656a2f..a840dcbaf 100755 --- a/scripts/deploy +++ b/scripts/deploy @@ -119,7 +119,7 @@ if [ -s .git-info ]; then cp -f .git-info .next/standalone/.git-info; fi # traced into .next/standalone). Keep the standalone server: it embeds the # compiled config and avoids loading build-only plugins at runtime. COPYFILE_DISABLE=1 tar czf "$LOCAL_STAGE/sleepypod-core.tar.gz" \ - --exclude='node_modules' --exclude='.next/cache' \ + --exclude='node_modules' --exclude='.next/cache' --exclude='.venv' --exclude='__pycache__' --exclude='.pytest_cache' \ --exclude='.git' --exclude='.env*' --exclude='*.db' --exclude='*.db-*' \ --exclude='.DS_Store' --exclude='test-results' --exclude='coverage' \ --exclude='.serena' --exclude='.claude' --exclude='.codex' --exclude='.ygg' \ diff --git a/scripts/import-watch-health.py b/scripts/import-watch-health.py new file mode 100755 index 000000000..3e117d8cb --- /dev/null +++ b/scripts/import-watch-health.py @@ -0,0 +1,207 @@ +#!/usr/bin/env -S uv run --quiet --no-project +""" +Import Apple Watch sleep stages and heart rate from a Health export into the pod. + + scripts/import-watch-health.py ~/Desktop/apple_health_export --pod 192.168.1.88 --side left + +Reads export.xml (or the unzipped apple_health_export/ directory) with the same +parser as sleep-stage-dataset.py, keeps samples whose sourceName matches +--source (default: Watch), and posts them to the pod's REST API: + + POST /api/biometrics/reference-stages SleepAnalysis segments (wake/light/deep/rem) + POST /api/biometrics/reference-vitals HeartRate samples + +Heart rate is first-write-wins on side + time + source, so re-running on a +newer export only adds what is new. Stages are stricter: a batch is rejected +(HTTP 400) when a segment overlaps a stored one with different boundaries, +e.g. a night the iOS app already posted. Such a batch is skipped with a +warning, heart rate is still imported, and the run exits 1 at the end. +Nothing is written to the pod's own `vitals` table: Watch heart rate lives in +`reference_vitals`, and the Biometrics page draws it as a second line over the +pod's heart rate. + +Options: + --days N only samples from the last N days (default 30; 0 = everything) + --dry-run parse and count, post nothing + --stages-only / --hr-only +""" +import argparse +import importlib.util +import json +import os +import sys +import time +import zipfile +from contextlib import contextmanager +import urllib.error +import urllib.request +from datetime import datetime, timezone + +HERE = os.path.dirname(os.path.abspath(__file__)) +_spec = importlib.util.spec_from_file_location('sleep_stage_dataset', os.path.join(HERE, 'sleep-stage-dataset.py')) +_ds = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_ds) +parse_export = _ds.parse_export + +# The API caps both endpoints at 2000 items per call. +BATCH = 2000 +# importReferenceStages rejects a batch whose segments overlap each other. +# Health exports occasionally carry two sources' segments for the same minute; +# the dataset builder resolves that by letting the later start win, and so do we. + + +def warn(msg): + print(f'warning: {msg}', file=sys.stderr) + + +def dedupe_segments(segments): + """Resolve overlaps exactly as sleep-stage-dataset.stage_lookup does (the + segment starting latest wins) by splitting at every segment boundary and + merging runs of the same stage, so the posted labels match the export-based + dataset. Returns non-overlapping [(start_s, end_s, stage)] sorted by start.""" + if not segments: + return [] + stage_at = _ds.stage_lookup(segments) + bounds = sorted({int(t.timestamp()) for s, e, _ in segments for t in (s, e)}) + out = [] + for a, b in zip(bounds, bounds[1:]): + stage = stage_at(a) + if stage is None: + continue + if out and out[-1][1] == a and out[-1][2] == stage: + out[-1] = (out[-1][0], b, stage) + else: + out.append((a, b, stage)) + return out + + +def chunks(items, n=BATCH): + for i in range(0, len(items), n): + yield items[i:i + n] + + +class Rejected(Exception): + """The pod answered 4xx: the batch is bad, the pod is fine.""" + + def __init__(self, url, code, body): + super().__init__(f'{url} returned {code}: {body[:500]}') + self.code = code + + +def post_json(url, payload, timeout=30): + data = json.dumps(payload).encode('utf-8') + req = urllib.request.Request(url, data=data, headers={'Content-Type': 'application/json'}, method='POST') + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + return json.loads(resp.read().decode('utf-8') or '{}') + except urllib.error.HTTPError as e: + body = e.read().decode('utf-8', 'replace') + if 400 <= e.code < 500: + raise Rejected(url, e.code, body) + raise SystemExit(f'error: {url} returned {e.code}: {body[:500]}') + except urllib.error.URLError as e: + raise SystemExit(f'error: cannot reach {url}: {e.reason}') + + +def base_url(pod): + if pod.startswith('http://') or pod.startswith('https://'): + return pod.rstrip('/') + return f'http://{pod}:3000' if ':' not in pod else f'http://{pod}' + + +@contextmanager +def export_stream(export): + """Read only the Health XML member; never extract arbitrary ZIP paths.""" + if zipfile.is_zipfile(export): + with zipfile.ZipFile(export) as archive: + entries = [entry for entry in archive.infolist() + if entry.filename in ('export.xml', 'apple_health_export/export.xml')] + if len(entries) != 1: + raise ValueError('ZIP must contain exactly one apple_health_export/export.xml or export.xml') + if entries[0].file_size > 1024 * 1024 * 1024: + raise ValueError('Expanded export.xml exceeds 1 GiB; export a smaller date range') + with archive.open(entries[0]) as xml: + yield xml + else: + yield export + + +def run(export, pod, side, source_pattern='Watch', days=30, dry_run=False, + stages=True, hr=True, post=post_json, now=None, report=False): + now = time.time() if now is None else now + cutoff = now - days * 86400 if days > 0 else float('-inf') + with export_stream(export) as xml: + segments, hr_samples = parse_export(xml, source_pattern, cutoff=cutoff) + + segs = [s for s in dedupe_segments(segments) if s[1] >= cutoff] if stages else [] + samples = [(t, bpm) for t, bpm in hr_samples if t >= cutoff and 20 <= bpm <= 250] if hr else [] + + def span(times): + if not times: + return 'none' + lo, hi = min(times), max(times) + fmt = lambda t: datetime.fromtimestamp(t, timezone.utc).astimezone().strftime('%Y-%m-%d %H:%M') + return f'{fmt(lo)} → {fmt(hi)}' + + print(f'{len(segs)} stage segments ({span([s[0] for s in segs])}), ' + f'{len(samples)} heart-rate samples ({span([t for t, _ in samples])}) for side {side}', file=sys.stderr) + if dry_run: + return {'stages': 0, 'vitals': 0} + + url = base_url(pod) + written = {'stages': 0, 'vitals': 0} + rejected = 0 + for batch in chunks(segs): + try: + out = post(f'{url}/api/biometrics/reference-stages', { + 'side': side, 'source': 'apple_watch', + 'segments': [{'start': a, 'end': b, 'stage': st} for a, b, st in batch], + }) + except Rejected as e: + # A stored segment with different boundaries (iOS app, older + # import) blocks this batch; the rest of the run still goes in. + warn(f'skipped a batch of {len(batch)} stage segments ({span([s[0] for s in batch])}): {e}') + rejected += 1 + continue + written['stages'] += int(out.get('written', 0)) + for batch in chunks(samples): + out = post(f'{url}/api/biometrics/reference-vitals', { + 'side': side, 'source': 'apple_watch', + 'samples': [{'timestamp': t, 'heartRate': bpm} for t, bpm in batch], + }) + written['vitals'] += int(out.get('written', 0)) + print(f"wrote {written['stages']} new stage segments and {written['vitals']} new heart-rate samples to {url}", + file=sys.stderr) + if report: + return {**written, 'rejectedStageBatches': rejected, 'matchedStages': len(segs), 'matchedVitals': len(samples)} + if rejected: + raise SystemExit(f'error: {rejected} stage batch(es) rejected by the pod; heart rate was still imported') + return written + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.split('\n\n')[0]) + ap.add_argument('export', help='export.xml or the apple_health_export directory') + ap.add_argument('--pod', required=True, help='pod host/IP (port 3000 assumed) or a full http(s) URL') + ap.add_argument('--side', choices=('left', 'right'), required=True, help='the side the Watch wearer sleeps on') + ap.add_argument('--source', default='Watch', help='regex on export sourceName (default: Watch)') + ap.add_argument('--days', type=int, default=30, help='only the last N days (default 30; 0 = all)') + ap.add_argument('--json', action='store_true', help='return import counts and partial stage conflicts as JSON') + ap.add_argument('--dry-run', action='store_true', help='parse and count only') + group = ap.add_mutually_exclusive_group() + group.add_argument('--stages-only', action='store_true') + group.add_argument('--hr-only', action='store_true') + args = ap.parse_args(argv) + + export = os.path.join(args.export, 'export.xml') if os.path.isdir(args.export) else args.export + if not os.path.exists(export): + ap.error(f'{export} not found') + result = run(export, args.pod, args.side, args.source, args.days, args.dry_run, + stages=not args.hr_only, hr=not args.stages_only, report=args.json) + if args.json: + print(json.dumps(result)) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/repair-capped-sessions.py b/scripts/repair-capped-sessions.py new file mode 100755 index 000000000..5f6d0e7b2 --- /dev/null +++ b/scripts/repair-capped-sessions.py @@ -0,0 +1,245 @@ +#!/usr/bin/env python3 +""" +Repair sleep_records that the sleep-detector force-closed at its 16 h cap. + +A stuck presence signal held sessions open on an empty bed until +MAX_SESSION_S (57600 s), so their left_bed_at, duration and intervals run up +to 16 h past the real exit. This rewrites each capped record's end from the +vitals already in biometrics.db, deterministically: + + - Occupancy evidence is the side's vitals rows inside the record. Movement + rows are not evidence: the detector writes one every minute for as long + as a session is open, empty bed included, so they run to the cap. + - The piezo processor writes about one vitals row a minute while someone + is in bed, and also short low-signal bursts on an empty bed (pod 88: + a 22-minute run 35 minutes after the bed emptied). So the vitals are + split into runs at gaps over VITALS_GAP_S, and the occupant left at the + end of the last run lasting at least MIN_RUN_S. + - left_bed_at = that end + ABSENCE_TIMEOUT_S, never later than the + original. The 120 s is padding for the time between the last vitals row + and the exit; a natural close stores the first absent sample instead, + with no padding. Duration is recomputed, intervals are clipped to the + new end, and exits are recounted from the present intervals that remain, + each ending in an exit as on a natural close. + - Shorter runs after that end are dropped as empty-bed noise. That also + drops a genuine short return to bed (60 min, a 10-min gap, 20 min), so + such records are listed as short-final-run for review before --apply. + - Records with no qualifying run, or with interval JSON that isn't a list + of [start, stop] pairs, are left unchanged and listed. + +Dry run by default. --apply first writes a consistent copy of the database to +.bak. (.bak.. if that exists; SQLite backup API, +safe alongside the running services), then updates all repairable records in +one transaction. Re-running finds nothing: repaired records are no longer +exactly MAX_SESSION_S long. + +Usage: + scripts/repair-capped-sessions.py [--db PATH] # dry run + scripts/repair-capped-sessions.py [--db PATH] --apply +""" + +import argparse +import json +import os +import sqlite3 +import sys +import time +from datetime import datetime, timezone +from pathlib import Path + +# The detector's own values, from the module it imports them from. +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "modules" / "sleep-detector")) +from session_limits import ABSENCE_TIMEOUT_S, MAX_SESSION_S, MIN_SESSION_S # noqa: E402 + +# Vitals gaps longer than this split runs; runs shorter than MIN_RUN_S are +# empty-bed noise. +VITALS_GAP_S = 5 * 60 +MIN_RUN_S = 30 * 60 + +DEFAULT_DB = os.environ.get( + "BIOMETRICS_DATABASE_URL", "file:/persistent/sleepypod-data/biometrics.db" +).replace("file:", "") + + +def vitals_runs(vitals_ts): + """[start, end] of each vitals run, split at gaps over VITALS_GAP_S.""" + runs = [] + for ts in vitals_ts: + if runs and ts - runs[-1][1] <= VITALS_GAP_S: + runs[-1][1] = ts + else: + runs.append([ts, ts]) + return runs + + +def occupancy_end(vitals_ts): + """End of the last vitals run lasting at least MIN_RUN_S, or None.""" + ends = [end for start, end in vitals_runs(vitals_ts) if end - start >= MIN_RUN_S] + return ends[-1] if ends else None + + +def parse_intervals(raw): + """A record's [start, stop] intervals, or None if the JSON isn't that.""" + if not raw: + return [] + try: + intervals = json.loads(raw) + except ValueError: + return None + if not isinstance(intervals, list) or not all( + isinstance(iv, list) and len(iv) == 2 + and all(isinstance(x, (int, float)) for x in iv) for iv in intervals): + return None + return intervals + + +def clip_intervals(intervals, end): + """Intervals clipped to `end`; ones starting at or after it are dropped.""" + return [[start, min(stop, end)] for start, stop in intervals if start < end] + + +def plan(conn): + rows = conn.execute( + """SELECT id, side, entered_bed_at, left_bed_at, times_exited_bed, + present_intervals, not_present_intervals + FROM sleep_records WHERE sleep_duration_seconds = ? ORDER BY id""", + (MAX_SESSION_S,), + ).fetchall() + out = [] + for rid, side, entered, left, exits, present, absent in rows: + vitals = [r[0] for r in conn.execute( + "SELECT timestamp FROM vitals WHERE side = ? AND timestamp BETWEEN ? AND ? ORDER BY timestamp", + (side, entered, left), + )] + end = occupancy_end(vitals) + item = {"id": rid, "side": side, "entered": entered, "left": left, + "vitals": len(vitals), "last_vitals": vitals[-1] if vitals else None} + present_iv, absent_iv = parse_intervals(present), parse_intervals(absent) + if present_iv is None or absent_iv is None: + item["action"] = "malformed" + elif end is None: + item["action"] = "no-evidence" + else: + new_left = min(left, end + ABSENCE_TIMEOUT_S) + if new_left >= left: + item["action"] = "keep" + elif new_left - entered < MIN_SESSION_S: + item["action"] = "too-short" + item["new_left"] = new_left + else: + kept = clip_intervals(present_iv, new_left) + # Each kept present interval ends in an exit, the last at + # new_left. Older records without intervals count the final one. + new_exits = min(exits + 1, len(kept)) if kept else exits + 1 + item.update(action="repair", new_left=new_left, + new_duration=new_left - entered, + new_exits=new_exits, + present=json.dumps(kept), + absent=json.dumps(clip_intervals(absent_iv, new_left)), + dropped_runs=[r for r in vitals_runs(vitals) if r[0] > end]) + out.append(item) + return out + + +def fmt(ts): + return datetime.fromtimestamp(ts, tz=timezone.utc).strftime("%Y-%m-%d %H:%M") + + +def report(items, verbose): + by = {} + for it in items: + by.setdefault(it["action"], []).append(it) + repaired = by.get("repair", []) + removed_h = sum(it["left"] - it["new_left"] for it in repaired) / 3600 + if verbose: + print(f"{'id':>5} {'side':5} {'entered (UTC)':16} {'left (UTC)':16} {'new left':16} {'vitals':>6} action") + for it in items: + new = fmt(it["new_left"]) if "new_left" in it else "-" + print(f"{it['id']:>5} {it['side']:5} {fmt(it['entered'])} {fmt(it['left'])} {new:16} " + f"{it['vitals']:>6} {it['action']}") + print() + print(f"capped records (sleep_duration_seconds = {MAX_SESSION_S}): {len(items)}") + print(f" repair: {len(repaired)} ({removed_h:.1f} h of empty-bed padding removed)") + if repaired: + hours = sorted((it["new_duration"]) / 3600 for it in repaired) + print(f" new durations {hours[0]:.1f}-{hours[-1]:.1f} h, " + f"median {hours[len(hours) // 2]:.1f} h") + short = [it for it in repaired if it["dropped_runs"]] + if short: + print(f" short-final-run: {len(short)} of these drop later vitals runs under " + f"{MIN_RUN_S // 60} min as empty-bed noise; review before --apply:") + for it in short: + runs = ", ".join(f"{fmt(a)[11:]}-{fmt(b)[11:]} ({(b - a) / 60:.0f} min)" + for a, b in it["dropped_runs"]) + print(f" {it['id']:>5} {it['side']:5} new left {fmt(it['new_left'])} dropped {runs}") + print(f" keep: {len(by.get('keep', []))} (vitals run to the cap)") + print(f" too-short: {len(by.get('too-short', []))} (occupancy under {MIN_SESSION_S}s; left unchanged)") + print(f" no-evidence: {len(by.get('no-evidence', []))} (no vitals run of {MIN_RUN_S // 60}+ min; left unchanged)") + if by.get("malformed"): + print(f" malformed: {len(by['malformed'])} (interval JSON unreadable; left unchanged)") + + +def reserve_backup_path(db_path): + """Create and return a backup path no earlier run is using. Exclusive + create, so two runs in the same second can't overwrite each other's.""" + base = f"{db_path}.bak.{int(time.time())}" + for n in range(1000): + path = base if n == 0 else f"{base}.{n}" + try: + os.close(os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644)) + except FileExistsError: + continue + return path + raise RuntimeError(f"no free backup path at {base}.*") + + +def apply(db_path, conn, items): + repaired = [it for it in items if it["action"] == "repair"] + if not repaired: + print("nothing to apply") + return + backup_path = reserve_backup_path(db_path) + dst = sqlite3.connect(backup_path) + try: + conn.backup(dst) + finally: + dst.close() + print(f"backup written: {backup_path}") + with conn: + for it in repaired: + cur = conn.execute( + """UPDATE sleep_records + SET left_bed_at = ?, sleep_duration_seconds = ?, times_exited_bed = ?, + present_intervals = ?, not_present_intervals = ? + WHERE id = ? AND sleep_duration_seconds = ? AND left_bed_at = ?""", + (it["new_left"], it["new_duration"], it["new_exits"], + it["present"], it["absent"], it["id"], MAX_SESSION_S, it["left"]), + ) + if cur.rowcount != 1: + raise RuntimeError(f"record {it['id']} changed underneath the repair; rolled back") + print(f"applied: {len(repaired)} records updated") + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument("--db", default=DEFAULT_DB, help=f"biometrics.db path (default {DEFAULT_DB})") + ap.add_argument("--apply", action="store_true", help="write a .bak, then update the records") + ap.add_argument("-v", "--verbose", action="store_true", help="list every capped record") + args = ap.parse_args() + if not os.path.exists(args.db): + sys.exit(f"no database at {args.db}") + conn = sqlite3.connect(args.db, timeout=10.0) + conn.execute("PRAGMA busy_timeout=10000") + try: + items = plan(conn) + report(items, args.verbose) + if args.apply: + apply(args.db, conn, items) + else: + print("\ndry run: nothing written (pass --apply to repair)") + finally: + conn.close() + + +if __name__ == "__main__": + main() diff --git a/scripts/sleep-stage-dataset.py b/scripts/sleep-stage-dataset.py new file mode 100755 index 000000000..37321c606 --- /dev/null +++ b/scripts/sleep-stage-dataset.py @@ -0,0 +1,547 @@ +#!/usr/bin/env -S uv run --quiet --no-project +""" +Sleep-stage dataset builder: joins Apple Watch sleep stages with pod vitals. + +Emits one CSV row per pod vitals sample (about one per minute) for every night +that has Watch stage labels: + + night,side,ts,watch_stage,hr,hrv,br,movement,hr_quality,watch_hr + +- night local date the night started (sample time minus 12 h) +- ts pod vitals timestamp, unix seconds +- watch_stage wake|light|deep|rem of the Watch segment containing ts, empty if none +- hr,hrv,br raw pod vitals (not outlier-filtered) +- movement nearest movement.total_movement within 60 s, empty if none +- hr_quality vitals_quality.quality_score for that vitals row, empty if none +- watch_hr nearest Watch heart-rate sample within 60 s, empty if none + +A night's window is the Watch stage coverage padded by --pad-min on each side. +Nights whose pod-vs-Watch mean absolute HR difference exceeds --max-hr-mad are +skipped with a warning: that pattern means the Watch wearer slept on the other +side. + +--summary prints per-night agreement with the Watch for the classifier in +src/lib/sleep-stages.ts (ported below), as deployed (calibrationQuality 0, +movement-only) and with the iOS rule set (calibrationQuality 1), plus the +always-light baseline. + +Usage: + scripts/sleep-stage-dataset.py export.xml biometrics.db --side left --out dataset.csv + scripts/sleep-stage-dataset.py apple_health_export/ biometrics.db --summary + scripts/sleep-stage-dataset.py --from-db biometrics.db --side left --out dataset.csv + +--from-db reads Watch stages from the reference_stages table and Watch heart +rate from reference_vitals (both filled by scripts/import-watch-health.py or +the iOS app), instead of a Health export. On a DB without reference_vitals the +watch_hr column is empty and the side-mismatch guard cannot run. + +See docs/research/sleep-stage-calibration/README.md for pulling the pod DB. +Stdlib only; runs under python3 >= 3.9 directly or via the uv shebang. +""" + +import argparse +import bisect +import csv +import math +import os +import re +import sqlite3 +import sys +import xml.etree.ElementTree as ET +from collections import Counter +from datetime import datetime, timedelta +from pathlib import Path + +STAGES = ('wake', 'light', 'deep', 'rem') +CSV_FIELDS = ('night', 'side', 'ts', 'watch_stage', 'hr', 'hrv', 'br', 'movement', 'hr_quality', 'watch_hr') + +# HKCategoryValueSleepAnalysis* -> stage. InBed, Asleep and AsleepUnspecified +# carry no stage and are ignored. +HK_STAGE = { + 'HKCategoryValueSleepAnalysisAwake': 'wake', + 'HKCategoryValueSleepAnalysisAsleepCore': 'light', + 'HKCategoryValueSleepAnalysisAsleepDeep': 'deep', + 'HKCategoryValueSleepAnalysisAsleepREM': 'rem', +} +HK_SLEEP = 'HKCategoryTypeIdentifierSleepAnalysis' +HK_HR = 'HKQuantityTypeIdentifierHeartRate' + +JOIN_TOLERANCE_S = 60 +MIN_HR_MATCHES = 10 +NIGHT_ROLLOVER = timedelta(hours=12) + + +def warn(msg): + print(f'warning: {msg}', file=sys.stderr) + + +def night_of(dt): + """Local date the night started; dt must be timezone-aware local time.""" + return (dt - NIGHT_ROLLOVER).date().isoformat() + + +# ── Health export ────────────────────────────────────────────────────────── + +def parse_export(path, source_pattern='Watch', cutoff=None): + """Stream export.xml and return (stage_segments, hr_samples) for sources + matching source_pattern. + + stage_segments: [(start_dt, end_dt, stage)] in file order (aware datetimes) + hr_samples: [(unix_s, bpm)] sorted by time + """ + if isinstance(path, (str, os.PathLike)) and os.path.isdir(path): + path = os.path.join(path, 'export.xml') + source_re = re.compile(source_pattern) + segments, hr = [], [] + + def add(a): + kind = a.get('type') + if kind not in (HK_SLEEP, HK_HR) or not source_re.search(a.get('sourceName', '')): + return + try: + start = datetime.strptime(a['startDate'], '%Y-%m-%d %H:%M:%S %z') + end = datetime.strptime(a['endDate'], '%Y-%m-%d %H:%M:%S %z') + except (KeyError, ValueError): + return + if cutoff is not None and end.timestamp() < cutoff: + return + if kind == HK_SLEEP: + stage = HK_STAGE.get(a.get('value', '')) + if stage and end > start: + segments.append((start, end, stage)) + return + try: + bpm = float(a['value']) + except (KeyError, ValueError): + return + if math.isfinite(bpm): + hr.append((int(start.timestamp()), bpm)) + + # Incremental parse, so record layout (line breaks, attribute wrapping) + # does not matter. Clearing the root after each record keeps memory flat. + events = ET.iterparse(path, events=('start', 'end')) + _, root = next(events) + for event, elem in events: + if event == 'end' and elem.tag == 'Record': + add(elem.attrib) + root.clear() + hr.sort() + return segments, hr + + +def segments_from_db(conn, side, source='apple_watch'): + """Read Watch stage segments from reference_stages (sleepypod-core-116). + + Timestamps are unix seconds (Drizzle mode 'timestamp'); millisecond values + are tolerated. Returns segments as local aware datetimes, like parse_export. + """ + exists = conn.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='reference_stages'").fetchone() + if not exists: + raise SystemExit( + 'error: reference_stages table not found in the DB. It is created by the ' + 'biometrics.importReferenceStages migration; pass a Health export instead.') + + def to_dt(v): + v = float(v) + if v > 1e11: + v /= 1000.0 + return datetime.fromtimestamp(v).astimezone() + + rows = conn.execute( + 'SELECT start, "end", stage FROM reference_stages WHERE side = ? AND source = ? ORDER BY start', + (side, source)).fetchall() + return [(to_dt(s), to_dt(e), st) for s, e, st in rows if st in STAGES and float(e) > float(s)] + + +def hr_from_db(conn, side, source='apple_watch'): + """Read Watch heart-rate samples from reference_vitals (see + scripts/import-watch-health.py) as [(unix_s, bpm)] sorted by time, like + parse_export. An older DB without the table yields no samples, so the + watch_hr column stays empty and the side-mismatch guard cannot run.""" + exists = conn.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='reference_vitals'").fetchone() + if not exists: + warn('reference_vitals table not found; watch_hr will be empty and the side-mismatch guard is off') + return [] + rows = conn.execute( + 'SELECT timestamp, heart_rate FROM reference_vitals WHERE side = ? AND source = ? ORDER BY timestamp', + (side, source)).fetchall() + out = [] + for ts, bpm in rows: + ts, bpm = float(ts), float(bpm) + if ts > 1e11: + ts /= 1000.0 + if math.isfinite(bpm): + out.append((int(ts), bpm)) + # SQL ordered the raw values; normalizing mixed second/millisecond rows can + # reorder them, and nearest() bisects, so sort again. + out.sort() + return out + + +def stage_lookup(segments): + """Return stage_at(ts) -> stage of the segment whose [start, end) contains + unix second ts, else None. Where segments overlap, the one starting latest + wins, so export file order and DB order give the same labels.""" + segs = sorted((s.timestamp(), e.timestamp(), st) for s, e, st in segments) + starts = [s for s, _, _ in segs] + reach, m = [], float('-inf') + for _, e, _ in segs: + m = max(m, e) + reach.append(m) + + def stage_at(ts): + j = bisect.bisect_right(starts, ts) - 1 + # reach[j] is the latest end among segments 0..j; once it is <= ts no + # earlier segment can contain ts. + while j >= 0 and reach[j] > ts: + if segs[j][1] > ts: + return segs[j][2] + j -= 1 + return None + + return stage_at + + +def group_nights(segments, pad_min): + """{night: (window_start_s, window_end_s)} from Watch coverage + padding. + + Windows are disjoint: a window that touches the previous night's (sleep + spanning the noon rollover, or padding) is merged into that night, so no + vitals row is emitted under two nights.""" + spans = {} + for start, end, _ in segments: + n = night_of(start) + s, e = start.timestamp(), end.timestamp() + if n in spans: + spans[n] = (min(spans[n][0], s), max(spans[n][1], e)) + else: + spans[n] = (s, e) + pad = pad_min * 60 + out, prev = {}, None + for n, (s, e) in sorted(spans.items(), key=lambda kv: kv[1][0]): + s, e = int(s - pad), int(e + pad) + if prev is not None and s <= out[prev][1]: + out[prev] = (out[prev][0], max(out[prev][1], e)) + continue + out[n] = (s, e) + prev = n + return out + + +# ── Pod DB ───────────────────────────────────────────────────────────────── + +def load_pod(conn, side, start_s, end_s, join_pad_s=0): + """Return (vitals, movement) for one side and window, ascending. + + vitals: [(id, ts, hr, hrv, br, quality)] + movement: [(ts, total_movement)], over the window widened by join_pad_s + on each side so vitals at the edges can match their nearest row + """ + has_quality = conn.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='vitals_quality'").fetchone() + if has_quality: + q = ('SELECT v.id, v.timestamp, v.heart_rate, v.hrv, v.breathing_rate, ' + '(SELECT quality_score FROM vitals_quality WHERE vitals_id = v.id ORDER BY id DESC LIMIT 1) ' + 'FROM vitals v WHERE v.side = ? AND v.timestamp BETWEEN ? AND ? ORDER BY v.timestamp') + else: + q = ('SELECT id, timestamp, heart_rate, hrv, breathing_rate, NULL ' + 'FROM vitals WHERE side = ? AND timestamp BETWEEN ? AND ? ORDER BY timestamp') + vitals = conn.execute(q, (side, start_s, end_s)).fetchall() + movement = conn.execute( + 'SELECT timestamp, total_movement FROM movement WHERE side = ? AND timestamp BETWEEN ? AND ? ' + 'ORDER BY timestamp', (side, start_s - join_pad_s, end_s + join_pad_s)).fetchall() + return vitals, movement + + +def nearest(times, values, t, tol=JOIN_TOLERANCE_S): + """Value whose time is nearest t within tol (earlier wins ties), else None.""" + i = bisect.bisect_left(times, t) + best, best_d = None, None + for j in (i - 1, i): + if 0 <= j < len(times): + d = abs(times[j] - t) + if d <= tol and (best_d is None or d < best_d): + best, best_d = values[j], d + return best + + +def join_night(night, side, vitals, movement, stage_at, watch_hr): + """Build dataset rows (dicts keyed by CSV_FIELDS) for one night.""" + mov_t = [t for t, _ in movement] + mov_v = [v for _, v in movement] + hr_t = [t for t, _ in watch_hr] + hr_v = [v for _, v in watch_hr] + rows = [] + for _id, ts, hr, hrv, br, quality in vitals: + rows.append({ + 'night': night, + 'side': side, + 'ts': int(ts), + 'watch_stage': stage_at(ts), + 'hr': hr, + 'hrv': hrv, + 'br': br, + 'movement': nearest(mov_t, mov_v, ts), + 'hr_quality': quality, + 'watch_hr': nearest(hr_t, hr_v, ts), + }) + return rows + + +def hr_mad(rows): + """(mean |pod HR - Watch HR|, matched count) over rows with both values + finite.""" + diffs = [abs(r['hr'] - r['watch_hr']) for r in rows if r['hr'] is not None and r['watch_hr'] is not None] + diffs = [d for d in diffs if math.isfinite(d)] + return (sum(diffs) / len(diffs) if diffs else None), len(diffs) + + +# ── Classifier port of src/lib/sleep-stages.ts ───────────────────────────── +# Keep in lockstep with classifySleepStages / classifyEpoch / filterOutliers. + +def _js_round(x): + return math.floor(x + 0.5) + + +def filter_outliers(vitals): + """[(ts, hr, hrv, br)] -> same with outliers nulled (hard limits + ±2 HR median filter).""" + def ok_hr(h): + return h is not None and 45 <= h <= 130 + + out = [] + n = len(vitals) + for i, (ts, hr, hrv, br) in enumerate(vitals): + if hr is not None and (hr < 45 or hr > 130): + hr = None + if hrv is not None and (hrv < 1 or hrv > 300): + hrv = None + if br is not None and (br < 8 or br > 25): + br = None + if hr is not None: + window = [w[1] for w in vitals[max(0, i - 2):min(n - 1, i + 2) + 1] if ok_hr(w[1])] + if window: + median = sorted(window)[len(window) // 2] + mean = sum(window) / len(window) + std = math.sqrt(sum((h - mean) ** 2 for h in window) / len(window)) + if std > 0 and abs(hr - median) > 2 * std: + hr = None + out.append((ts, hr, hrv, br)) + return out + + +def classify_epoch(hr, hrv, mov, avg_hr, cq): + if cq < 0.3: + if mov is not None and mov > 200: + return 'wake' + return 'light' + if mov is not None and mov > 200: + return 'wake' + if hr is not None and hr > 0: + ratio = hr / avg_hr + if ratio < 0.92: + return 'deep' + if ratio >= 0.95: + if hrv is not None and hrv < 25 and mov is not None and mov < 30: + return 'rem' + if mov is not None and mov < 50 and hrv is not None and hrv < 40: + return 'rem' + if mov is not None and mov > 100: + return 'wake' + return 'light' + + +def classify_sleep_stages(vitals, movement, calibration_quality=0.0): + """vitals [(ts_s, hr, hrv, br)], movement [(ts_s, total)] -> [(ts_s, stage)]. + + Mirrors classifySleepStages: outlier filter, 5-min movement buckets + (last row wins), phase-1 rules, A-B-A smoothing, transition constraints. + """ + if not vitals: + return [] + v = filter_outliers(sorted(vitals, key=lambda r: r[0])) + + buckets = {} + for ts, total in movement: + buckets[_js_round(ts * 1000 / 300_000) * 300_000] = total + + valid = [hr for _, hr, _, _ in v if hr is not None and hr > 0] + avg_hr = sum(valid) / len(valid) if valid else 60 + + stages = [ + classify_epoch(hr, hrv, buckets.get(_js_round(ts * 1000 / 300_000) * 300_000), avg_hr, calibration_quality) + for ts, hr, hrv, _ in v + ] + for i in range(1, len(stages) - 1): + if stages[i - 1] == stages[i + 1] and stages[i] != stages[i - 1]: + stages[i] = stages[i - 1] + blocked = {('wake', 'deep'), ('deep', 'wake'), ('deep', 'rem'), ('rem', 'deep')} + for i in range(1, len(stages)): + if (stages[i - 1], stages[i]) in blocked: + stages[i] = 'light' + return [(ts, s) for (ts, *_), s in zip(v, stages)] + + +# ── Summary ──────────────────────────────────────────────────────────────── + +def agreement(rows, predicted): + """Percent of Watch-labeled rows where predicted[ts] equals watch_stage.""" + labeled = [r for r in rows if r['watch_stage']] + if not labeled: + return None + hit = sum(1 for r in labeled if predicted.get(r['ts']) == r['watch_stage']) + return 100.0 * hit / len(labeled) + + +def summarize_night(rows, movement): + vitals = [(r['ts'], r['hr'], r['hrv'], r['br']) for r in rows] + deployed = dict(classify_sleep_stages(vitals, movement, 0.0)) + ios = dict(classify_sleep_stages(vitals, movement, 1.0)) + light = {r['ts']: 'light' for r in rows} + return { + 'labeled': sum(1 for r in rows if r['watch_stage']), + 'deployed': agreement(rows, deployed), + 'ios': agreement(rows, ios), + 'light': agreement(rows, light), + 'watch_dist': Counter(r['watch_stage'] for r in rows if r['watch_stage']), + } + + +def print_summary(results, out=None): + out = out or sys.stdout + + def pct(x): + return ' n/a' if x is None else f'{x:5.0f}%' + + def mad(x): + return ' n/a' if x is None else f'{x:6.1f}' + + print('night side rows labeled HR-MAD deployed(cq=0) iOS(cq=1) always-light', file=out) + for r in results: + print(f"{r['night']} {r['side']:5} {r['rows']:5} {r['labeled']:7} {mad(r['mad'])} " + f"{pct(r['deployed']):>14} {pct(r['ios']):>9} {pct(r['light']):>12}", file=out) + print('\nWatch-labeled samples per night:', file=out) + for r in results: + dist = ', '.join(f'{s}={r["watch_dist"].get(s, 0)}' for s in STAGES) + print(f" {r['night']}: {dist}", file=out) + + +# ── Driver ───────────────────────────────────────────────────────────────── + +def build(conn, segments, watch_hr, side, pad_min=10, max_hr_mad=8.0, nights=None): + """Return [(night, rows, movement, mad, matched)] for nights passing the guard.""" + stage_at = stage_lookup(segments) + out = [] + for night, (start_s, end_s) in group_nights(segments, pad_min).items(): + if nights and night not in nights: + continue + vitals, near_mov = load_pod(conn, side, start_s, end_s, JOIN_TOLERANCE_S) + if not vitals: + warn(f'{night}: no {side} vitals in the Watch window; skipped') + continue + # Join candidates reach JOIN_TOLERANCE_S past the window; the + # classifier's movement input stays inside it. + hr_in = [(t, v) for t, v in watch_hr if start_s - JOIN_TOLERANCE_S <= t <= end_s + JOIN_TOLERANCE_S] + rows = join_night(night, side, vitals, near_mov, stage_at, hr_in) + movement = [(t, v) for t, v in near_mov if start_s <= t <= end_s] + mad, matched = hr_mad(rows) + if matched < MIN_HR_MATCHES: + warn(f'{night}: only {matched} pod/Watch HR matches; side-mismatch guard not applied') + elif mad > max_hr_mad: + warn(f'{night}: pod {side} vs Watch HR MAD {mad:.1f} bpm > {max_hr_mad:g} ' + f'({matched} matches); likely the other side, skipped') + continue + out.append((night, rows, movement, mad, matched)) + return out + + +def _fmt(v): + if v is None: + return '' + if isinstance(v, float): + if v != 0 and abs(v) < 0.0005: + # Three decimals would print it as 0. + return f'{v:.3g}' + return f'{v:.3f}'.rstrip('0').rstrip('.') + return v + + +def write_csv(nights, fh): + w = csv.writer(fh, lineterminator='\n') + w.writerow(CSV_FIELDS) + for _night, rows, *_ in nights: + for r in rows: + w.writerow([_fmt(r[k]) for k in CSV_FIELDS]) + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.split('\n\n')[0]) + ap.add_argument('inputs', nargs='+', metavar='PATH', + help=' , or just with --from-db') + ap.add_argument('--from-db', action='store_true', + help='read Watch stages and heart rate from the reference_stages / reference_vitals ' + 'tables instead of an export') + ap.add_argument('--side', choices=('left', 'right'), default='left') + ap.add_argument('--out', help='CSV path (default: stdout unless --summary)') + ap.add_argument('--summary', action='store_true', help='print per-night classifier agreement') + ap.add_argument('--nights', help='comma-separated night dates to keep, e.g. 2026-10-08,2026-10-09') + ap.add_argument('--source', default='Watch', help='regex on export sourceName (default: Watch)') + ap.add_argument('--pad-min', type=int, default=10, help='minutes added around Watch coverage (default 10)') + ap.add_argument('--max-hr-mad', type=float, default=8.0, help='side-mismatch guard threshold in bpm (default 8)') + args = ap.parse_args(argv) + + expected = 1 if args.from_db else 2 + if len(args.inputs) != expected: + ap.error('expected with --from-db' if args.from_db + else 'expected ') + db_path = args.inputs[-1] + if not os.path.exists(db_path): + ap.error(f'{db_path} not found') + inputs = [db_path] + if not args.from_db: + export = args.inputs[0] + export_xml = os.path.join(export, 'export.xml') if os.path.isdir(export) else export + if not os.path.exists(export_xml): + ap.error(f'{export_xml} not found') + inputs.append(export_xml) + if args.out and os.path.exists(args.out) and any(os.path.samefile(args.out, p) for p in inputs): + ap.error(f'--out {args.out} is an input file; refusing to overwrite it') + # as_uri() percent-encodes ?, # and %, which SQLite would otherwise read + # as URI syntax and open a different file. + conn = sqlite3.connect(Path(db_path).resolve().as_uri() + '?mode=ro', uri=True) + + if args.from_db: + segments, watch_hr = segments_from_db(conn, args.side), hr_from_db(conn, args.side) + else: + try: + segments, watch_hr = parse_export(export_xml, args.source) + except ET.ParseError as e: + raise SystemExit(f'error: {export_xml} is not valid XML: {e}') + if not segments: + raise SystemExit('error: no Watch sleep-stage samples found') + + wanted = {n.strip() for n in args.nights.split(',') if n.strip()} if args.nights else None + nights = build(conn, segments, watch_hr, args.side, args.pad_min, args.max_hr_mad, wanted) + + if args.out: + with open(args.out, 'w', newline='') as fh: + write_csv(nights, fh) + print(f'wrote {sum(len(n[1]) for n in nights)} rows for {len(nights)} nights to {args.out}', file=sys.stderr) + elif not args.summary: + write_csv(nights, sys.stdout) + + if args.summary: + results = [] + for night, rows, movement, mad, _matched in nights: + s = summarize_night(rows, movement) + results.append({'night': night, 'side': args.side, 'rows': len(rows), 'mad': mad, **s}) + if results: + print_summary(results) + else: + print('no nights passed the filters', file=sys.stderr) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/tests/eslintConfig.test.ts b/scripts/tests/eslintConfig.test.ts new file mode 100644 index 000000000..0ac21c220 --- /dev/null +++ b/scripts/tests/eslintConfig.test.ts @@ -0,0 +1,15 @@ +// @vitest-environment node +import { ESLint } from 'eslint' +import { expect, it } from 'vitest' + +it('recognizes runtime references in JS and TypeScript config files while rejecting unused variables', async () => { + const eslint = new ESLint({ cwd: process.cwd() }) + for (const filePath of ['lint-fixture.mjs', 'lint-fixture.mts']) { + const [used] = await eslint.lintText('const value = 1\nexport default value\n', { filePath }) + expect(used.messages).toEqual([]) + const [unused] = await eslint.lintText('const forgotten = 1\n', { filePath }) + expect(unused.messages).toEqual(expect.arrayContaining([ + expect.objectContaining({ ruleId: '@typescript-eslint/no-unused-vars', severity: 2 }), + ])) + } +}, 20000) diff --git a/scripts/tests/test_import_watch_health.py b/scripts/tests/test_import_watch_health.py new file mode 100644 index 000000000..0055c07ca --- /dev/null +++ b/scripts/tests/test_import_watch_health.py @@ -0,0 +1,215 @@ +"""Tests for scripts/import-watch-health.py with a synthetic Health export and a +recording stand-in for the pod's REST API. + +Run: python3 -m unittest scripts/tests/test_import_watch_health.py + (or: uv run --with pytest --no-project pytest scripts/tests/test_import_watch_health.py) +""" + +import contextlib +import importlib.util +import io +import os +import tempfile +import unittest +from datetime import datetime, timedelta, timezone + +HERE = os.path.dirname(os.path.abspath(__file__)) +_spec = importlib.util.spec_from_file_location( + 'import_watch_health', os.path.join(HERE, '..', 'import-watch-health.py')) +iw = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(iw) + +TZ = timezone(timedelta(hours=-7)) +T0 = datetime(2026, 10, 9, 2, 0, tzinfo=TZ) +WATCH = "Jon’s Apple Watch" + + +def hk_time(dt): + return dt.strftime('%Y-%m-%d %H:%M:%S %z') + + +def sleep_record(start, end, value, source=WATCH): + return (f' \n') + + +def hr_record(at, bpm, source=WATCH): + return (f' \n') + + +def write_export(path, segments, hr): + parts = ['\n\n'] + for a, b, v in segments: + parts.append(sleep_record(T0 + timedelta(minutes=a), T0 + timedelta(minutes=b), v)) + for m, bpm in hr: + parts.append(hr_record(T0 + timedelta(minutes=m), bpm)) + parts.append('\n') + with open(path, 'w', encoding='utf-8') as f: + f.write(''.join(parts)) + + +class Recorder: + def __init__(self): + self.calls = [] + + def __call__(self, url, payload): + self.calls.append((url, payload)) + key = 'segments' if 'segments' in payload else 'samples' + return {'written': len(payload[key])} + + +class RejectingRecorder(Recorder): + """Pod stand-in that refuses every stage batch like a stored-overlap 400.""" + + def __call__(self, url, payload): + if 'segments' in payload: + self.calls.append((url, payload)) + raise iw.Rejected(url, 400, 'Segments overlap stored reference stages') + return super().__call__(url, payload) + + +class ImportWatchHealthTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.export = os.path.join(self.tmp.name, 'export.xml') + self.now = (T0 + timedelta(days=1)).timestamp() + + def tearDown(self): + self.tmp.cleanup() + + def run_import(self, **kw): + post = Recorder() + err = io.StringIO() + with contextlib.redirect_stderr(err): + written = iw.run(self.export, '192.168.1.88', 'left', post=post, now=self.now, **kw) + return post, written, err.getvalue() + + def test_posts_stages_and_heart_rate_to_the_pod_endpoints(self): + write_export(self.export, [(0, 20, 'AsleepCore'), (20, 50, 'AsleepDeep'), (50, 60, 'Awake')], + [(1, 58), (6, 56.5), (11, 61)]) + post, written, err = self.run_import() + urls = [u for u, _ in post.calls] + self.assertEqual(urls, ['http://192.168.1.88:3000/api/biometrics/reference-stages', + 'http://192.168.1.88:3000/api/biometrics/reference-vitals']) + stages = post.calls[0][1] + self.assertEqual(stages['side'], 'left') + self.assertEqual(stages['source'], 'apple_watch') + self.assertEqual([s['stage'] for s in stages['segments']], ['light', 'deep', 'wake']) + self.assertEqual(stages['segments'][0]['start'], int(T0.timestamp())) + self.assertEqual(stages['segments'][0]['end'], int((T0 + timedelta(minutes=20)).timestamp())) + samples = post.calls[1][1]['samples'] + self.assertEqual(samples, [ + {'timestamp': int((T0 + timedelta(minutes=1)).timestamp()), 'heartRate': 58.0}, + {'timestamp': int((T0 + timedelta(minutes=6)).timestamp()), 'heartRate': 56.5}, + {'timestamp': int((T0 + timedelta(minutes=11)).timestamp()), 'heartRate': 61.0}, + ]) + self.assertEqual(written, {'stages': 3, 'vitals': 3}) + self.assertIn('wrote 3 new stage segments and 3 new heart-rate samples', err) + + def test_dry_run_posts_nothing_but_reports_counts(self): + write_export(self.export, [(0, 20, 'AsleepCore')], [(1, 58)]) + post, written, err = self.run_import(dry_run=True) + self.assertEqual(post.calls, []) + self.assertEqual(written, {'stages': 0, 'vitals': 0}) + self.assertIn('1 stage segments', err) + self.assertIn('1 heart-rate samples', err) + + def test_days_window_drops_old_samples(self): + write_export(self.export, [(0, 20, 'AsleepCore'), (-60 * 24 * 40, -60 * 24 * 40 + 30, 'AsleepDeep')], + [(1, 58), (-60 * 24 * 40, 70)]) + post, written, _ = self.run_import(days=30) + self.assertEqual(len(post.calls[0][1]['segments']), 1) + self.assertEqual(len(post.calls[1][1]['samples']), 1) + _, written_all, _ = self.run_import(days=0) + self.assertEqual(written_all, {'stages': 2, 'vitals': 2}) + + def test_batches_at_the_api_limit(self): + write_export(self.export, [], [(m, 60) for m in range(0, 4500)]) + post, written, _ = self.run_import() + self.assertEqual([len(p['samples']) for _, p in post.calls], [2000, 2000, 500]) + self.assertEqual(written['vitals'], 4500) + + def test_stages_only_and_hr_only(self): + write_export(self.export, [(0, 20, 'AsleepCore')], [(1, 58)]) + post, _, _ = self.run_import(hr=False) + self.assertEqual([u.rsplit('/', 1)[1] for u, _ in post.calls], ['reference-stages']) + post, _, _ = self.run_import(stages=False) + self.assertEqual([u.rsplit('/', 1)[1] for u, _ in post.calls], ['reference-vitals']) + + def test_overlapping_segments_are_trimmed_so_the_api_accepts_the_batch(self): + # Two sources' segments for the same minutes; the later start wins over + # the overlap, including a fully contained segment, matching stage_lookup. + write_export(self.export, [(0, 30, 'AsleepCore'), (20, 40, 'AsleepDeep'), (25, 28, 'AsleepREM')], []) + post, _, _ = self.run_import() + segs = post.calls[0][1]['segments'] + for a, b in zip(segs, segs[1:]): + self.assertLessEqual(a['end'], b['start']) + t = lambda m: int((T0 + timedelta(minutes=m)).timestamp()) + self.assertEqual([(s['start'], s['end'], s['stage']) for s in segs], [ + (t(0), t(20), 'light'), (t(20), t(25), 'deep'), (t(25), t(28), 'rem'), (t(28), t(40), 'deep'), + ]) + + def test_rejected_stage_batch_still_imports_heart_rate_and_exits_nonzero(self): + write_export(self.export, [(0, 20, 'AsleepCore')], [(1, 58), (6, 60)]) + post = RejectingRecorder() + err = io.StringIO() + with contextlib.redirect_stderr(err): + with self.assertRaises(SystemExit) as cm: + iw.run(self.export, '192.168.1.88', 'left', post=post, now=self.now) + self.assertEqual([u.rsplit('/', 1)[1] for u, _ in post.calls], ['reference-stages', 'reference-vitals']) + self.assertEqual(len(post.calls[1][1]['samples']), 2) + self.assertIn('skipped a batch of 1 stage segments', err.getvalue()) + self.assertIn('1 stage batch(es) rejected', str(cm.exception)) + + def test_drops_implausible_heart_rates(self): + write_export(self.export, [], [(1, 58), (2, 5), (3, 400)]) + post, _, _ = self.run_import() + self.assertEqual([s['heartRate'] for s in post.calls[0][1]['samples']], [58.0]) + + def test_ignores_other_sources(self): + write_export(self.export, [(0, 20, 'AsleepCore')], [(1, 58)]) + with open(self.export, 'a', encoding='utf-8') as f: + pass + post, _, _ = self.run_import(source_pattern='Nope') + self.assertEqual(post.calls, []) + + def test_base_url_forms(self): + self.assertEqual(iw.base_url('192.168.1.88'), 'http://192.168.1.88:3000') + self.assertEqual(iw.base_url('pod.local:8080'), 'http://pod.local:8080') + self.assertEqual(iw.base_url('https://pod.example/'), 'https://pod.example') + + def test_main_requires_an_existing_export(self): + with self.assertRaises(SystemExit): + with contextlib.redirect_stderr(io.StringIO()): + iw.main([os.path.join(self.tmp.name, 'missing.xml'), '--pod', 'x', '--side', 'left']) + + + +class WatchZipImportTest(unittest.TestCase): + def test_zip_import_and_partial_conflicts_are_reported(self): + import zipfile + with tempfile.TemporaryDirectory() as directory: + xml = os.path.join(directory, 'export.xml') + write_export(xml, [(0, 10, 'AsleepCore')], [(5, 62)]) + archive = os.path.join(directory, 'export.zip') + with zipfile.ZipFile(archive, 'w') as output: + output.write(xml, 'apple_health_export/export.xml') + result = iw.run(archive, 'pod', 'left', days=0, report=True, post=RejectingRecorder()) + self.assertEqual(result['vitals'], 1) + self.assertEqual(result['rejectedStageBatches'], 1) + self.assertEqual(result['matchedStages'], 1) + + def test_zip_does_not_extract_arbitrary_paths(self): + import zipfile + with tempfile.TemporaryDirectory() as directory: + archive = os.path.join(directory, 'export.zip') + with zipfile.ZipFile(archive, 'w') as output: + output.writestr('../export.xml', '') + with self.assertRaisesRegex(ValueError, 'exactly one'): + iw.run(archive, 'pod', 'left', report=True, post=Recorder()) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/tests/test_sleep_stage_dataset.py b/scripts/tests/test_sleep_stage_dataset.py new file mode 100644 index 000000000..1da4d1829 --- /dev/null +++ b/scripts/tests/test_sleep_stage_dataset.py @@ -0,0 +1,505 @@ +"""Tests for scripts/sleep-stage-dataset.py using small synthetic fixtures. + +Run: python3 -m unittest scripts/tests/test_sleep_stage_dataset.py + (or: uv run --with pytest --no-project pytest scripts/tests/test_sleep_stage_dataset.py) +""" + +import contextlib +import csv +import importlib.util +import io +import os +import sqlite3 +import tempfile +import time +import unittest +from datetime import datetime, timedelta, timezone + +HERE = os.path.dirname(os.path.abspath(__file__)) +_spec = importlib.util.spec_from_file_location( + 'sleep_stage_dataset', os.path.join(HERE, '..', 'sleep-stage-dataset.py')) +ds = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(ds) + +TZ = timezone(timedelta(hours=-7)) +# Night of 2026-10-08: Watch coverage 02:00-02:30 local on 10-09. +T0 = datetime(2026, 10, 9, 2, 0, tzinfo=TZ) +WATCH = "Jane’s Apple Watch" + + +def hk_time(dt): + return dt.strftime('%Y-%m-%d %H:%M:%S %z') + + +def sleep_record(start, end, value, source=WATCH): + return (f' \n' + f' \n \n') + + +def hr_record(at, bpm, source=WATCH): + return (f' \n') + + +# Segments: 02:00-02:10 light, 02:10-02:20 deep, 02:20-02:30 rem +SEGMENTS = [(0, 10, 'AsleepCore'), (10, 20, 'AsleepDeep'), (20, 30, 'AsleepREM')] + + +def write_export(path, hr_offset=0.0, with_hr=True): + parts = ['\n\n'] + for a, b, v in SEGMENTS: + parts.append(sleep_record(T0 + timedelta(minutes=a), T0 + timedelta(minutes=b), v)) + # Noise that must be ignored: other source, InBed, non-Watch HR. + parts.append(sleep_record(T0, T0 + timedelta(minutes=30), 'AsleepDeep', source='sleepypod')) + parts.append(sleep_record(T0, T0 + timedelta(minutes=30), 'InBed')) + parts.append(hr_record(T0, 150, source='sleepypod')) + if with_hr: + for m in range(30): + parts.append(hr_record(T0 + timedelta(minutes=m, seconds=20), pod_hr(m) + hr_offset)) + parts.append('\n') + with open(path, 'w', encoding='utf-8') as f: + f.write(''.join(parts)) + + +def pod_hr(minute): + return 60.0 + (minute % 3) + + +def make_db(path, side='left', minutes=range(-5, 35)): + conn = sqlite3.connect(path) + conn.executescript(''' + CREATE TABLE vitals (id INTEGER PRIMARY KEY AUTOINCREMENT, side TEXT, timestamp INTEGER, + heart_rate REAL, hrv REAL, breathing_rate REAL); + CREATE TABLE movement (id INTEGER PRIMARY KEY AUTOINCREMENT, side TEXT, timestamp INTEGER, + total_movement INTEGER); + CREATE TABLE vitals_quality (id INTEGER PRIMARY KEY AUTOINCREMENT, vitals_id INTEGER, + side TEXT, timestamp INTEGER, quality_score REAL); + ''') + base = int(T0.timestamp()) + for m in minutes: + ts = base + m * 60 + 5 + cur = conn.execute('INSERT INTO vitals (side, timestamp, heart_rate, hrv, breathing_rate) VALUES (?,?,?,?,?)', + (side, ts, pod_hr(m), 50.0 + m, 14.0)) + conn.execute('INSERT INTO vitals_quality (vitals_id, side, timestamp, quality_score) VALUES (?,?,?,?)', + (cur.lastrowid, side, ts, 0.5)) + conn.execute('INSERT INTO movement (side, timestamp, total_movement) VALUES (?,?,?)', + (side, base + m * 60 + 42, m * 10)) + conn.commit() + return conn + + +def add_reference_stages(conn, side='left'): + conn.execute('CREATE TABLE reference_stages (id INTEGER PRIMARY KEY AUTOINCREMENT, side TEXT, ' + 'source TEXT, start INTEGER, "end" INTEGER, stage TEXT, created_at INTEGER)') + stage = {'AsleepCore': 'light', 'AsleepDeep': 'deep', 'AsleepREM': 'rem'} + for a, b, v in SEGMENTS: + conn.execute('INSERT INTO reference_stages (side, source, start, "end", stage, created_at) VALUES (?,?,?,?,?,0)', + (side, 'apple_watch', int((T0 + timedelta(minutes=a)).timestamp()), + int((T0 + timedelta(minutes=b)).timestamp()), stage[v])) + conn.commit() + + +def add_reference_vitals(conn, side='left', minutes=range(0, 30), offset=0.0): + conn.execute('CREATE TABLE reference_vitals (id INTEGER PRIMARY KEY AUTOINCREMENT, side TEXT, ' + 'source TEXT, timestamp INTEGER, heart_rate REAL, created_at INTEGER)') + base = int(T0.timestamp()) + for m in minutes: + conn.execute('INSERT INTO reference_vitals (side, source, timestamp, heart_rate, created_at) VALUES (?,?,?,?,0)', + (side, 'apple_watch', base + m * 60 + 20, pod_hr(m) + offset)) + conn.commit() + + +class Fixture(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.export = os.path.join(self.tmp.name, 'export.xml') + self.db_path = os.path.join(self.tmp.name, 'biometrics.db') + + def tearDown(self): + self.tmp.cleanup() + + def run_main(self, *argv): + out, err = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + ds.main(list(argv)) + return out.getvalue(), err.getvalue() + + +class ParseExportTest(Fixture): + def test_keeps_only_watch_stage_samples_and_hr(self): + write_export(self.export) + segments, hr = ds.parse_export(self.export) + self.assertEqual([s for _, _, s in segments], ['light', 'deep', 'rem']) + self.assertEqual(len(hr), 30) + self.assertTrue(all(55 <= v <= 65 for _, v in hr)) + + def test_accepts_export_directory(self): + write_export(self.export) + segments, _ = ds.parse_export(self.tmp.name) + self.assertEqual(len(segments), 3) + + def test_source_regex_selects_other_source(self): + write_export(self.export) + segments, hr = ds.parse_export(self.export, source_pattern='^sleepypod$') + self.assertEqual([s for _, _, s in segments], ['deep']) + self.assertEqual(hr, [(int(T0.timestamp()), 150.0)]) + + def test_record_layout_does_not_matter(self): + # Attributes wrapped over lines, and two records on one line. + wrapped = sleep_record(T0, T0 + timedelta(minutes=10), 'AsleepCore').replace('" ', '"\n ') + one_line = (hr_record(T0 + timedelta(minutes=1), 61) + hr_record(T0 + timedelta(minutes=2), 62)).replace('\n', '') + with open(self.export, 'w', encoding='utf-8') as f: + f.write(f'\n\n{wrapped}{one_line}\n\n') + segments, hr = ds.parse_export(self.export) + self.assertEqual([s for _, _, s in segments], ['light']) + self.assertEqual([v for _, v in hr], [61.0, 62.0]) + + def test_nonfinite_watch_hr_is_dropped(self): + with open(self.export, 'w', encoding='utf-8') as f: + f.write('\n' + ''.join(hr_record(T0 + timedelta(minutes=m), v) + for m, v in enumerate(('nan', 'inf', '61'))) + '\n') + _, hr = ds.parse_export(self.export) + self.assertEqual([v for _, v in hr], [61.0]) + + +class JoinTest(Fixture): + def test_rows_per_minute_with_labels_movement_quality_and_watch_hr(self): + write_export(self.export) + make_db(self.db_path).close() + out, _ = self.run_main(self.export, self.db_path, '--side', 'left') + rows = list(csv.DictReader(io.StringIO(out))) + self.assertEqual(list(rows[0].keys()), list(ds.CSV_FIELDS)) + # Window = 02:00-02:30 padded 10 min = 01:50-02:40 → minutes -5..34 all present. + self.assertEqual(len(rows), 40) + self.assertEqual({r['night'] for r in rows}, {'2026-10-08'}) + by_min = {(int(r['ts']) - int(T0.timestamp())) // 60: r for r in rows} + self.assertEqual(by_min[-1]['watch_stage'], '') + self.assertEqual(by_min[0]['watch_stage'], 'light') + self.assertEqual(by_min[9]['watch_stage'], 'light') + self.assertEqual(by_min[10]['watch_stage'], 'deep') + self.assertEqual(by_min[25]['watch_stage'], 'rem') + self.assertEqual(by_min[30]['watch_stage'], '') + # Vitals at :05; previous minute's movement row at :42 (23 s away) beats this minute's (37 s). + self.assertEqual(by_min[3]['movement'], '20') + self.assertEqual(by_min[3]['hr_quality'], '0.5') + # Watch HR at :20 of each minute 0..29 is 15 s from the vitals sample. + self.assertEqual(by_min[4]['watch_hr'], by_min[4]['hr']) + self.assertEqual(by_min[-2]['watch_hr'], '') + self.assertEqual(by_min[32]['watch_hr'], '') + + def test_nearest_respects_tolerance_and_prefers_earlier_on_tie(self): + times, vals = [100, 200], ['a', 'b'] + self.assertEqual(ds.nearest(times, vals, 150), 'a') + self.assertEqual(ds.nearest(times, vals, 190), 'b') + self.assertIsNone(ds.nearest(times, vals, 300)) + self.assertIsNone(ds.nearest([], [], 5)) + + def test_only_requested_side_is_read(self): + write_export(self.export) + make_db(self.db_path, side='right').close() + out, err = self.run_main(self.export, self.db_path, '--side', 'left') + self.assertEqual(out.strip(), ','.join(ds.CSV_FIELDS)) + self.assertIn('no left vitals', err) + + def test_from_db_matches_export_rows(self): + # reference_stages has no tz; night labels use local time, so pin it to the export's zone. + old_tz = os.environ.get('TZ') + os.environ['TZ'] = 'America/Los_Angeles' + time.tzset() + self.addCleanup(self._restore_tz, old_tz) + write_export(self.export, with_hr=False) + conn = make_db(self.db_path) + add_reference_stages(conn) + conn.close() + from_export, _ = self.run_main(self.export, self.db_path) + from_db, _ = self.run_main('--from-db', self.db_path) + self.assertEqual(from_db, from_export) + self.assertEqual(len(from_db.strip().splitlines()), 41) + + def test_from_db_reads_watch_hr_from_reference_vitals(self): + old_tz = os.environ.get('TZ') + os.environ['TZ'] = 'America/Los_Angeles' + time.tzset() + self.addCleanup(self._restore_tz, old_tz) + write_export(self.export) + conn = make_db(self.db_path) + add_reference_stages(conn) + add_reference_vitals(conn) + conn.close() + from_export, _ = self.run_main(self.export, self.db_path) + from_db, err = self.run_main('--from-db', self.db_path) + self.assertEqual(from_db, from_export) + self.assertNotIn('reference_vitals', err) + rows = list(csv.DictReader(io.StringIO(from_db))) + self.assertTrue(any(r['watch_hr'] for r in rows)) + + def test_from_db_without_reference_vitals_warns_and_leaves_watch_hr_empty(self): + conn = make_db(self.db_path) + add_reference_stages(conn) + conn.close() + out, err = self.run_main('--from-db', self.db_path) + self.assertIn('reference_vitals table not found', err) + rows = list(csv.DictReader(io.StringIO(out))) + self.assertTrue(rows) + self.assertTrue(all(r['watch_hr'] == '' for r in rows)) + + def test_from_db_side_mismatch_guard_uses_reference_vitals(self): + conn = make_db(self.db_path) + add_reference_stages(conn) + add_reference_vitals(conn, offset=25.0) # Watch wearer slept on the other side + conn.close() + out, err = self.run_main('--from-db', self.db_path) + self.assertIn('skipp', err.lower()) + self.assertEqual(out.strip(), ','.join(ds.CSV_FIELDS)) + + def test_hr_from_db_tolerates_millisecond_timestamps(self): + conn = sqlite3.connect(':memory:') + conn.execute('CREATE TABLE reference_vitals (side TEXT, source TEXT, timestamp INTEGER, heart_rate REAL)') + conn.execute("INSERT INTO reference_vitals VALUES ('left', 'apple_watch', ?, 61.0)", (int(T0.timestamp()) * 1000,)) + conn.execute("INSERT INTO reference_vitals VALUES ('left', 'apple_watch', ?, 'nan')", (int(T0.timestamp()) + 60,)) + conn.execute("INSERT INTO reference_vitals VALUES ('right', 'apple_watch', ?, 70.0)", (int(T0.timestamp()),)) + self.assertEqual(ds.hr_from_db(conn, 'left'), [(int(T0.timestamp()), 61.0)]) + # A later sample stored in seconds sorts before an earlier one stored in + # milliseconds in SQL; the normalized list must still be time-ordered. + conn.execute("INSERT INTO reference_vitals VALUES ('left', 'apple_watch', ?, 63.0)", (int(T0.timestamp()) - 60,)) + self.assertEqual(ds.hr_from_db(conn, 'left'), [(int(T0.timestamp()) - 60, 63.0), (int(T0.timestamp()), 61.0)]) + + @staticmethod + def _restore_tz(old_tz): + if old_tz is None: + os.environ.pop('TZ', None) + else: + os.environ['TZ'] = old_tz + time.tzset() + + def test_out_matching_an_input_is_refused(self): + write_export(self.export) + make_db(self.db_path).close() + size = os.path.getsize(self.db_path) + for target in (self.db_path, self.export): + with self.assertRaises(SystemExit): + self.run_main(self.tmp.name, self.db_path, '--out', target) + self.assertEqual(os.path.getsize(self.db_path), size) + + def test_join_matches_rows_just_outside_the_window(self): + conn = sqlite3.connect(self.db_path) + conn.executescript(''' + CREATE TABLE vitals (id INTEGER PRIMARY KEY, side TEXT, timestamp INTEGER, + heart_rate REAL, hrv REAL, breathing_rate REAL); + CREATE TABLE movement (id INTEGER PRIMARY KEY, side TEXT, timestamp INTEGER, total_movement INTEGER); + ''') + base = int(T0.timestamp()) + conn.execute("INSERT INTO vitals (side, timestamp, heart_rate, hrv, breathing_rate) VALUES ('left',?,60,50,14)", + (base,)) + conn.execute("INSERT INTO movement (side, timestamp, total_movement) VALUES ('left',?,77)", (base - 10,)) + segs = [(T0, T0 + timedelta(minutes=30), 'light')] + [(_night, rows, movement, _mad, _matched)] = ds.build(conn, segs, [(base - 10, 60.0)], 'left', pad_min=0) + conn.close() + self.assertEqual((rows[0]['movement'], rows[0]['watch_hr']), (77, 60.0)) + # The classifier's movement stays inside the window. + self.assertEqual(movement, []) + + def test_db_path_with_uri_characters(self): + write_export(self.export) + db_dir = os.path.join(self.tmp.name, 'pod#copy?1%') + os.mkdir(db_dir) + db_path = os.path.join(db_dir, 'biometrics.db') + make_db(db_path).close() + out, _ = self.run_main(self.export, db_path) + self.assertEqual(len(out.strip().splitlines()), 41) + + def test_missing_export_is_a_usage_error(self): + make_db(self.db_path).close() + err = io.StringIO() + with contextlib.redirect_stderr(err), self.assertRaises(SystemExit) as cm: + ds.main([os.path.join(self.tmp.name, 'nope.xml'), self.db_path]) + self.assertEqual(cm.exception.code, 2) + self.assertIn('nope.xml not found', err.getvalue()) + + def test_from_db_without_table_exits_with_message(self): + make_db(self.db_path).close() + with self.assertRaises(SystemExit) as cm: + self.run_main('--from-db', self.db_path) + self.assertIn('reference_stages', str(cm.exception.code)) + + +class LabelAndWindowTest(unittest.TestCase): + def test_sample_before_mid_minute_transition_keeps_earlier_stage(self): + t = T0 + timedelta(minutes=1, seconds=30) + segs = [(T0, t, 'light'), (t, T0 + timedelta(minutes=5), 'deep')] + stage_at = ds.stage_lookup(segs) + base = int(T0.timestamp()) + self.assertEqual(stage_at(base + 70), 'light') + self.assertEqual(stage_at(base + 90), 'deep') + self.assertIsNone(stage_at(base + 300)) + self.assertIsNone(stage_at(base - 1)) + + def test_overlap_resolution_ignores_input_order(self): + segs = [(T0, T0 + timedelta(minutes=30), 'light'), + (T0 + timedelta(minutes=10), T0 + timedelta(minutes=12), 'rem')] + base = int(T0.timestamp()) + for order in (segs, segs[::-1]): + stage_at = ds.stage_lookup(order) + self.assertEqual([stage_at(base + m * 60) for m in (5, 11, 20)], ['light', 'rem', 'light']) + + def test_sleep_spanning_noon_belongs_to_one_night(self): + noon = datetime(2026, 10, 9, 12, 0, tzinfo=TZ) + segs = [(noon - timedelta(minutes=10), noon + timedelta(minutes=10), 'light'), + (noon + timedelta(minutes=10), noon + timedelta(minutes=30), 'deep')] + windows = ds.group_nights(segs, 10) + self.assertEqual(list(windows), ['2026-10-08']) + start, end = windows['2026-10-08'] + self.assertEqual((start, end), (int(noon.timestamp()) - 20 * 60, int(noon.timestamp()) + 40 * 60)) + + def test_dst_change_keeps_one_night_and_contiguous_labels(self): + # Segment times carry the offset in force when they were written, so a + # night across a DST change mixes two offsets. + pdt, pst = timezone(timedelta(hours=-7)), timezone(timedelta(hours=-8)) + fall = [(datetime(2026, 11, 1, 0, 30, tzinfo=pdt), datetime(2026, 11, 1, 1, 50, tzinfo=pdt), 'light'), + (datetime(2026, 11, 1, 1, 50, tzinfo=pdt), datetime(2026, 11, 1, 1, 20, tzinfo=pst), 'deep'), + (datetime(2026, 11, 1, 1, 20, tzinfo=pst), datetime(2026, 11, 1, 6, 0, tzinfo=pst), 'rem')] + spring = [(datetime(2027, 3, 14, 0, 30, tzinfo=pst), datetime(2027, 3, 14, 1, 50, tzinfo=pst), 'light'), + (datetime(2027, 3, 14, 3, 50, tzinfo=pdt), datetime(2027, 3, 14, 6, 0, tzinfo=pdt), 'deep')] + for segs, night in ((fall, '2026-10-31'), (spring, '2027-03-13')): + windows = ds.group_nights(segs, 0) + self.assertEqual(list(windows), [night]) + self.assertEqual(windows[night], (int(segs[0][0].timestamp()), int(segs[-1][1].timestamp()))) + # The repeated 01:00-02:00 hour labels by absolute time: 01:50 PDT is + # 01:20 + 30 min PST, so deep covers exactly 30 minutes. + stage_at = ds.stage_lookup(fall) + deep_start = int(fall[1][0].timestamp()) + self.assertEqual(int(fall[1][1].timestamp()) - deep_start, 30 * 60) + self.assertEqual([stage_at(deep_start - 1), stage_at(deep_start), stage_at(deep_start + 30 * 60)], + ['light', 'deep', 'rem']) + + +class SideMismatchGuardTest(Fixture): + def test_night_within_threshold_is_kept(self): + write_export(self.export, hr_offset=3.0) + make_db(self.db_path).close() + out, err = self.run_main(self.export, self.db_path) + self.assertEqual(len(out.strip().splitlines()), 41) + self.assertNotIn('MAD', err) + + def test_night_over_threshold_warns_and_is_skipped(self): + write_export(self.export, hr_offset=12.0) + make_db(self.db_path).close() + out, err = self.run_main(self.export, self.db_path) + self.assertEqual(out.strip(), ','.join(ds.CSV_FIELDS)) + self.assertIn('2026-10-08', err) + self.assertRegex(err, r'HR MAD 12\.\d bpm > 8') + + def test_threshold_is_configurable(self): + write_export(self.export, hr_offset=12.0) + make_db(self.db_path).close() + out, _ = self.run_main(self.export, self.db_path, '--max-hr-mad', '15') + self.assertEqual(len(out.strip().splitlines()), 41) + + def test_too_few_hr_matches_warns_but_keeps(self): + write_export(self.export, with_hr=False) + make_db(self.db_path).close() + out, err = self.run_main(self.export, self.db_path) + self.assertEqual(len(out.strip().splitlines()), 41) + self.assertIn('guard not applied', err) + + def test_hr_mad(self): + rows = [{'hr': 60.0, 'watch_hr': 70.0}, {'hr': 60.0, 'watch_hr': 58.0}, + {'hr': None, 'watch_hr': 90.0}, {'hr': 60.0, 'watch_hr': None}] + self.assertEqual(ds.hr_mad(rows), (6.0, 2)) + self.assertEqual(ds.hr_mad(rows + [{'hr': 60.0, 'watch_hr': float('nan')}]), (6.0, 2)) + + +def vit(minute, hr, hrv=None, br=None): + return (1_790_000_000 + minute * 60, hr, hrv, br) + + +def mov(minute, total): + return (1_790_000_000 + minute * 60, total) + + +class ClassifierPortTest(unittest.TestCase): + """Mirrors src/lib/tests/sleep-stages.test.ts for classifySleepStages.""" + + def stages(self, vitals, movement, cq=0.0): + return [s for _, s in ds.classify_sleep_stages(vitals, movement, cq)] + + def test_empty(self): + self.assertEqual(ds.classify_sleep_stages([], []), []) + + def test_high_movement_is_wake(self): + self.assertEqual(self.stages([vit(0, 70, 40, 15)], [mov(0, 500)]), ['wake']) + + def test_movement_only_mode_never_emits_deep_or_rem(self): + v = [vit(0, 80, 35, 15), vit(5, 65, 40, 15), vit(10, 63, 50, 14), vit(15, 75, 38, 15), vit(20, 72, 36, 15)] + self.assertEqual(self.stages(v, [], 0.0), ['light'] * 5) + + def test_low_hr_ratio_is_deep(self): + v = [vit(0, 80, 35, 15), vit(5, 65, 40, 15), vit(10, 63, 50, 14), vit(15, 75, 38, 15), vit(20, 72, 36, 15)] + self.assertEqual(self.stages(v, [], 1.0)[2], 'deep') + + def test_elevated_hr_low_hrv_low_movement_is_rem(self): + v = [vit(0, 55, 20, 14), vit(5, 62, 20, 14), vit(10, 65, 20, 14), vit(15, 68, 20, 14), vit(20, 78, 18, 16)] + self.assertEqual(self.stages(v, [mov(20, 10)], 1.0)[4], 'rem') + + def test_missing_movement_does_not_produce_rem(self): + v = [vit(m, 70, 20, 14) for m in (0, 5, 10, 15, 20)] + self.assertNotIn('rem', self.stages(v, [], 1.0)) + + def test_aba_smoothing(self): + v = [vit(0, 60, 40, 14), vit(5, 60, 40, 14), vit(10, 60, 40, 14)] + self.assertEqual(self.stages(v, [mov(0, 10), mov(5, 300), mov(10, 10)]), ['light'] * 3) + + def test_transition_wake_to_deep_passes_through_light(self): + # Epoch 0 wake by movement; epoch 1 deep by HR ratio; HR window kept varied so + # the outlier filter keeps it. + v = [vit(0, 70, 40, 14), vit(5, 58, 40, 14), vit(10, 66, 40, 14), vit(15, 72, 40, 14)] + raw = [ds.classify_epoch(h, x, m, 66.5, 1.0) for (_, h, x, _), m in zip(v, [300, None, None, None])] + self.assertEqual(raw[:2], ['wake', 'deep']) + self.assertEqual(self.stages(v, [mov(0, 300)], 1.0)[1], 'light') + + def test_outlier_filter_hard_limits_and_median(self): + out = ds.filter_outliers([vit(0, 40, 0.5, 30), vit(1, 60, 50, 14), vit(2, 61, 50, 14), + vit(3, 100, 50, 14), vit(4, 60, 50, 14), vit(5, 61, 50, 14)]) + self.assertEqual(out[0][1:], (None, None, None)) + self.assertIsNone(out[3][1]) # 100 vs window median 61, std ~15.6 → > 2σ + self.assertEqual(out[1][1], 60) + + def test_single_sample_window_keeps_hr(self): + v = [vit(0, None), vit(5, None), vit(10, 68, 40, 14), vit(15, None), vit(20, None)] + self.assertEqual(ds.filter_outliers(v)[2][1], 68) + + def test_movement_uses_five_minute_buckets_last_row_wins(self): + # Vitals at minute 1 rounds to bucket 0; movement rows at minutes 0 and 2 share it. + self.assertEqual(self.stages([vit(1, 60)], [mov(0, 10), mov(2, 300)]), ['wake']) + self.assertEqual(self.stages([vit(1, 60)], [mov(0, 300), mov(2, 10)]), ['light']) + + +class FormatTest(unittest.TestCase): + def test_fmt(self): + self.assertEqual([ds._fmt(v) for v in (None, 0.0, 0.5, 61.23456, 7)], ['', '0', '0.5', '61.235', 7]) + self.assertEqual([ds._fmt(v) for v in (1e-7, -0.0003)], ['1e-07', '-0.0003']) + + +class SummaryTest(Fixture): + def test_summary_reports_agreement_per_night(self): + write_export(self.export) + make_db(self.db_path).close() + out, _ = self.run_main(self.export, self.db_path, '--summary') + line = next(ln for ln in out.splitlines() if ln.startswith('2026-10-08')) + _night, side, rows, labeled, _mad, deployed, _ios, light = line.split() + self.assertEqual((side, rows, labeled), ('left', '40', '30')) + # 10 light / 10 deep / 10 rem Watch minutes. Movement-only mode only ever says + # light or wake, so both it and always-light score the 10 light minutes. + self.assertEqual(light, '33%') + self.assertEqual(deployed, '33%') + + def test_agreement_ignores_unlabeled_rows(self): + rows = [{'ts': 1, 'watch_stage': 'light'}, {'ts': 2, 'watch_stage': None}, {'ts': 3, 'watch_stage': 'deep'}] + self.assertEqual(ds.agreement(rows, {1: 'light', 2: 'deep', 3: 'light'}), 50.0) + self.assertIsNone(ds.agreement([{'ts': 1, 'watch_stage': None}], {})) + + +if __name__ == '__main__': + unittest.main() diff --git a/src/automation/signals.biometrics.ts b/src/automation/signals.biometrics.ts index 1ad964d8c..78585237c 100644 --- a/src/automation/signals.biometrics.ts +++ b/src/automation/signals.biometrics.ts @@ -18,6 +18,12 @@ * `streaming/capFramePersistence`). (`reduceCap` also derives a peakZone index, * used by the UI zone visualization rather than the engine.) * + * `{side}.presence` is the shared virtual occupancy sensor (`lib/occupancy`) + * collapsed to 1 (a person is in bed) / 0 (empty, or a light load such as a + * pet). It reads `undefined` while presence can't be sensed — no fresh + * capSense2 frame or no calibration — so rules skip rather than fire blind. + * `{side}.presence.zones` is the lit head/torso/legs count behind that call. + * * Automation temperature signals are Fahrenheit by contract because automation * actions target hardware setpoints, which are also Fahrenheit. */ @@ -26,6 +32,7 @@ import { desc, eq } from 'drizzle-orm' import { biometricsDb } from '@/src/db' import { ambientLight, bedTemp, freezerTemp, movement, vitals } from '@/src/db/biometrics-schema' import { centiDegreesToF, centiPercentToPercent } from '@/src/lib/tempUtils' +import { getOccupancy } from '@/src/lib/occupancy' import { getLatestCapSenseSnapshot } from '@/src/streaming/piezoStream' import { mean, reduceCap } from './capReduce' import type { SignalReader, SignalSnapshot } from './signals' @@ -94,6 +101,13 @@ export class BiometricsSignalReader implements SignalReader { out['ambient.light'] = al.lux } + for (const side of SIDES) { + const occ = getOccupancy(side) + if (!occ.available) continue + out[`${side}.presence`] = occ.classification === 'person' ? 1 : 0 + if (occ.level.zones !== null) out[`${side}.presence.zones`] = occ.level.zones + } + const cap = getLatestCapSenseSnapshot() if (cap && fresh(cap.receivedAtMs, CAP_FRESH_MS)) { for (const side of SIDES) { diff --git a/src/automation/tests/signals.biometrics.test.ts b/src/automation/tests/signals.biometrics.test.ts index 205fa7ab7..8cebc89f8 100644 --- a/src/automation/tests/signals.biometrics.test.ts +++ b/src/automation/tests/signals.biometrics.test.ts @@ -44,6 +44,17 @@ vi.mock('@/src/streaming/piezoStream', () => ({ getLatestCapSenseSnapshot: () => capMock.snapshot, })) +const occupancyMock = vi.hoisted(() => ({ + bySide: {} as Record, +})) + +vi.mock('@/src/lib/occupancy', () => ({ + getOccupancy: (side: 'left' | 'right') => { + const o = occupancyMock.bySide[side] ?? { available: false, classification: 'none', zones: null } + return { available: o.available, classification: o.classification, level: { zones: o.zones } } + }, +})) + import { ambientLight, bedTemp, freezerTemp, movement, vitals } from '@/src/db/biometrics-schema' import { reduceCap } from '../capReduce' import { BiometricsSignalReader } from '../signals.biometrics' @@ -90,6 +101,7 @@ describe('BiometricsSignalReader', () => { dbMock.state.sideCalls = {} dbMock.state.throws = false capMock.snapshot = null + occupancyMock.bySide = {} }) afterEach(() => { @@ -188,6 +200,26 @@ describe('BiometricsSignalReader', () => { expect(new BiometricsSignalReader().read()).toEqual({}) }) + it('exposes the classified presence sensor as 1/0 only while presence is sensable', () => { + occupancyMock.bySide = { + left: { available: true, classification: 'person', zones: 3 }, + right: { available: true, classification: 'light', zones: 1 }, + } + const out = new BiometricsSignalReader().read() + expect(out['left.presence']).toBe(1) + expect(out['left.presence.zones']).toBe(3) + expect(out['right.presence']).toBe(0) + expect(out['right.presence.zones']).toBe(1) + }) + + it('leaves presence undefined when the sensor is unavailable, so rules skip', () => { + occupancyMock.bySide = { left: { available: false, classification: 'person', zones: null } } + const out = new BiometricsSignalReader().read() + expect(out['left.presence']).toBeUndefined() + expect(out['left.presence.zones']).toBeUndefined() + expect(out['right.presence']).toBeUndefined() + }) + it('returns an empty snapshot when there are no rows and no cap snapshot', () => { expect(new BiometricsSignalReader().read()).toEqual({}) }) diff --git a/src/biometrics/algorithms/canonical-json.ts b/src/biometrics/algorithms/canonical-json.ts new file mode 100644 index 000000000..eddcd59fa --- /dev/null +++ b/src/biometrics/algorithms/canonical-json.ts @@ -0,0 +1,65 @@ +/** + * finite-binary64-json-v1: sorted UTF-16 keys, ordered arrays, exact decimal + * expansion of each finite IEEE-754 number, negative zero normalized to zero. + * Exact expansion avoids differences between Python and JS float printers. + * This is valid numeric JSON, not RFC 8785. Version it in every manifest. + */ +const utf8 = new TextEncoder() + +function numberToken(value: number): string { + if (!Number.isFinite(value)) throw new Error('Non-finite JSON number') + if (value === 0) return '0' + const view = new DataView(new ArrayBuffer(8)) + view.setFloat64(0, Math.abs(value)) + const bits = view.getBigUint64(0) + const exponent = Number((bits >> 52n) & 2047n) + let coefficient = bits & ((1n << 52n) - 1n) + if (exponent !== 0) coefficient += 1n << 52n + const power = (exponent || 1) - 1023 - 52 + let token: string + if (power >= 0) token = (coefficient << BigInt(power)).toString() + else { + const places = -power + const digits = (coefficient * 5n ** BigInt(places)).toString().padStart(places + 1, '0') + token = `${digits.slice(0, -places)}.${digits.slice(-places)}`.replace(/0+$/, '').replace(/\.$/, '') + } + return value < 0 ? `-${token}` : token +} + +export function canonicalWithLimit(value: unknown, maxNodes: number, maxBytes = Infinity): string { + let nodes = 0, bytes = 0 + const token = (text: string) => { + bytes += utf8.encode(text).byteLength + if (bytes > maxBytes) throw new Error('Canonical data byte limit') + return text + } + function visit(item: unknown, depth: number): string { + if (++nodes > maxNodes || depth > 32) throw new Error('JSON resource limit') + if (item === null) return token('null') + if (typeof item === 'number') return token(numberToken(item)) + if (typeof item === 'boolean') return token(String(item)) + if (typeof item === 'string') { + if (item.length > 65536 || /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(? maxNodes - nodes) throw new Error('JSON resource limit') + token('[' + ','.repeat(Math.max(0, item.length - 1)) + ']') + return `[${Array.from(item, child => visit(child, depth + 1)).join(',')}]` + } + if (typeof item === 'object' && (Object.getPrototypeOf(item) === Object.prototype || Object.getPrototypeOf(item) === null)) { + const keys = Object.keys(item) + if (keys.length * 2 > maxNodes - nodes) throw new Error('JSON resource limit') + keys.sort() + token('{' + ':'.repeat(keys.length) + ','.repeat(Math.max(0, keys.length - 1)) + '}') + return `{${keys.map(key => `${visit(key, depth + 1)}:${visit((item as Record)[key], depth + 1)}`).join(',')}}` + } + throw new Error('Expected finite JSON data') + } + return visit(value, 0) +} + +export function canonicalJson(value: unknown): string { + return canonicalWithLimit(value, 20000) +} + diff --git a/src/biometrics/algorithms/canonical.ts b/src/biometrics/algorithms/canonical.ts new file mode 100644 index 000000000..3fc4b0c2f --- /dev/null +++ b/src/biometrics/algorithms/canonical.ts @@ -0,0 +1,13 @@ +import { createHash } from 'node:crypto' +import { canonicalJson, canonicalWithLimit } from './canonical-json' +export { canonicalJson } from './canonical-json' + +/** Same canonical encoding; larger bounded budget for retained rows/reports. */ +export function dataHash(value: unknown): string { + const encoded = canonicalWithLimit(value, 2000000, 32 * 1024 * 1024) + return createHash('sha256').update(encoded).digest('hex') +} + +export function configHash(value: unknown): string { + return createHash('sha256').update(canonicalJson(value)).digest('hex') +} diff --git a/src/biometrics/algorithms/registry-core.ts b/src/biometrics/algorithms/registry-core.ts new file mode 100644 index 000000000..2f62a2f65 --- /dev/null +++ b/src/biometrics/algorithms/registry-core.ts @@ -0,0 +1,135 @@ +import contract from '../../../modules/common/biometrics/manifest.v1.json' +import type { StageClassifier } from '@/src/lib/sleep-stage-rules' +import { canonicalJson } from './canonical-json' + +interface ParameterSchema { + type?: string + properties?: Record + required?: string[] + additionalProperties?: boolean + items?: ParameterSchema + maxItems?: number + minimum?: number + maximum?: number + const?: unknown + enum?: unknown[] +} + +export interface BiometricConfig { + heart_rate: { trackingToleranceBpm: number, trackingHistoryLength: number } + breathing_rate: { intervalLowerRatio: number, intervalUpperRatio: number } + hrv_index: { harmonicTolerance: number, minimumWindows: number } + sleep_stages: StageClassifier +} +export interface Override { path: string, value: unknown } + +function deepFreeze(value: T): T { + if (value && typeof value === 'object') { + for (const child of Object.values(value)) deepFreeze(child) + Object.freeze(value) + } + return value +} + +export const biometricManifest = deepFreeze(contract) +export const defaultConfig = biometricManifest.defaults as BiometricConfig + +/** Validated adapters implement this deliberately small shared schema dialect. */ +function validateNode(schema: ParameterSchema, value: unknown, path: string): void { + const fail = (): never => { + throw new Error(`Invalid biometric parameter: ${path}`) + } + if (schema.const !== undefined && value !== schema.const) fail() + if (schema.enum && !schema.enum.includes(value)) fail() + switch (schema.type) { + case 'object': { + if (!value || typeof value !== 'object' || Array.isArray(value)) return fail() + const record = value as Record + const properties = schema.properties ?? {} + for (const key of schema.required ?? []) if (!Object.hasOwn(record, key)) fail() + for (const [key, child] of Object.entries(record)) { + if (!Object.hasOwn(properties, key)) fail() + validateNode(properties[key], child, `${path}.${key}`) + } + break + } + case 'array': + if (!Array.isArray(value) || value.length > (schema.maxItems ?? Infinity)) return fail() + if (!schema.items) throw new Error('Missing array schema') + for (const child of value) validateNode(schema.items, child, path) + break + case 'number': + case 'integer': + if (typeof value !== 'number' || !Number.isFinite(value) + || (schema.type === 'integer' && !Number.isSafeInteger(value)) + || value < (schema.minimum ?? -Infinity) || value > (schema.maximum ?? Infinity)) fail() + break + case 'string': + if (typeof value !== 'string') fail() + break + case 'boolean': + if (typeof value !== 'boolean') fail() + break + default: throw new Error(`Unsupported contract schema at ${path}`) + } +} + +function atPath(value: unknown, path: string[]): unknown { + return path.reduce((parent, key) => (parent as Record)[key], value) +} + +export function validateConfig(value: unknown): BiometricConfig { + canonicalJson(value) // finite JSON, depth/size bounds, no class instances + validateNode(biometricManifest.parameters, value, 'config') + for (const path of biometricManifest.orderedRanges) { + const range = atPath(value, path) as { min: number, max: number } + if (range.min > range.max) throw new Error(`Invalid ordered range: ${path.join('.')}`) + } + return value as BiometricConfig +} + +/** Scalars inherit independently; ordered arrays are replaced whole. */ +export function resolveOverrides(overrides: Override[]) { + if (overrides.length > 64) throw new Error('RESOURCE_LIMIT: maximum 64 overrides') + const resolved = structuredClone(defaultConfig) + const paths = new Set() + for (const override of overrides) { + if (Object.keys(override).some(k => k !== 'path' && k !== 'value')) throw new Error('Unknown override field') + const parts = override.path.split('.') + if (paths.has(override.path)) throw new Error('Duplicate override path') + paths.add(override.path) + let schema: ParameterSchema = biometricManifest.parameters + let parent = resolved as unknown as Record + for (const [i, part] of parts.entries()) { + if (schema.type !== 'object' || !schema.properties || !Object.hasOwn(schema.properties, part)) throw new Error(`Unknown override path: ${override.path}`) + schema = schema.properties[part] + if (i === parts.length - 1) { + if (schema.type === 'object') throw new Error('Override leaves or entire ordered arrays') + validateNode(schema, override.value, override.path) + parent[part] = structuredClone(override.value) + } + else parent = parent[part] as Record + } + } + validateConfig(resolved) + return { resolved: deepFreeze(resolved), overrides: structuredClone(overrides), provenance: Object.fromEntries(overrides.map(o => [o.path, 'override' as const])) } +} + +/** Removing a path restores inheritance; null is never a removal sentinel. */ +export function removeOverride(overrides: Override[], path: string): Override[] { + return overrides.filter(override => override.path !== path) +} + +/** Migrate every legacy leaf, including explicit defaults, to preserve intent. */ +export function legacyStageOverrides(classifier: StageClassifier): Override[] { + const result: Override[] = [] + function walk(value: unknown, path: string) { + if (value !== null && typeof value === 'object' && !Array.isArray(value)) { + for (const [key, child] of Object.entries(value)) walk(child, `${path}.${key}`) + } + else result.push({ path, value }) + } + walk(classifier, 'sleep_stages') + resolveOverrides(result) + return result +} diff --git a/src/biometrics/algorithms/registry.ts b/src/biometrics/algorithms/registry.ts new file mode 100644 index 000000000..0dc8371ff --- /dev/null +++ b/src/biometrics/algorithms/registry.ts @@ -0,0 +1,13 @@ +import { configHash } from './canonical' +import { biometricManifest, resolveOverrides, type Override } from './registry-core' +export { biometricManifest, defaultConfig, validateConfig, removeOverride, legacyStageOverrides } from './registry-core' +export type { BiometricConfig, Override } from './registry-core' + +export const manifestHash = configHash(biometricManifest) + +/** Resolve against the exact shipped manifest, retaining the canonical identity. */ +export function resolveConfig(overrides: Override[], baseHash = manifestHash) { + if (baseHash !== manifestHash) throw new Error('PROFILE_INCOMPATIBLE: unknown base manifest') + const result = resolveOverrides(overrides) + return { ...result, hash: configHash(result.resolved), manifestHash: baseHash } +} diff --git a/src/biometrics/algorithms/tests/data-hash.test.ts b/src/biometrics/algorithms/tests/data-hash.test.ts new file mode 100644 index 000000000..c5f411d87 --- /dev/null +++ b/src/biometrics/algorithms/tests/data-hash.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it } from 'vitest' +import { configHash, dataHash } from '../canonical' + +describe('bounded retained-data hashing', () => { + it('keeps canonical identity while permitting realistic multi-night data beyond config budgets', () => { + const small = { timestampMs: 1791673200000, heartRate: 60.1 } + expect(dataHash(small)).toBe(configHash(small)) + const large = Array.from({ length: 10000 }, (_, id) => ({ id, ...small })) + expect(() => configHash(large)).toThrow('resource limit') + expect(dataHash(large)).toMatch(/^[a-f0-9]{64}$/) + expect(dataHash(large)).toBe(dataHash(JSON.parse(JSON.stringify(large)))) + }) + it('continues to reject nonfinite and excessively deep data', () => { + expect(() => dataHash({ value: Infinity })).toThrow('Non-finite') + let nested: unknown = 0 + for (let index = 0; index < 40; index++) nested = [nested] + expect(() => dataHash(nested)).toThrow('resource limit') + expect(() => dataHash(new Array(3000000))).toThrow('resource limit') + expect(() => dataHash(Array(600).fill('x'.repeat(65536)))).toThrow('byte limit') + }) +}) diff --git a/src/biometrics/algorithms/tests/registry.test.ts b/src/biometrics/algorithms/tests/registry.test.ts new file mode 100644 index 000000000..e26371b13 --- /dev/null +++ b/src/biometrics/algorithms/tests/registry.test.ts @@ -0,0 +1,159 @@ +// @vitest-environment node +import { execFileSync } from 'node:child_process' +import { createHash } from 'node:crypto' +import { readFileSync } from 'node:fs' +import { describe, expect, it } from 'vitest' +import { canonicalJson, configHash } from '../canonical' +import { biometricManifest, defaultConfig, legacyStageOverrides, manifestHash, removeOverride, resolveConfig, validateConfig } from '../registry' +import { DEFAULT_STAGE_CLASSIFIER, stageClassifierSchema } from '@/src/lib/sleep-stage-rules' + +const fixtures = [ + [], + [{ path: 'heart_rate.trackingToleranceBpm', value: 20 }], + [{ path: 'sleep_stages.ruleset.smoothIsolatedEpochs', value: false }, { path: 'sleep_stages.ruleset.lowSignalBelow', value: 0 }], + [{ path: 'sleep_stages.ruleset.rules', value: [{ stage: 'wake', when: [] }, { stage: 'light', when: [] }] }], + [{ path: 'sleep_stages.ruleset.rules', value: [] }], + [{ path: 'heart_rate.trackingHistoryLength', value: 2.5 }], + [{ path: 'heart_rate.trackingToleranceBpm', value: true }], + [{ path: 'sleep_stages.ruleset.outliers.heartRate.min', value: 140 }], + [{ path: 'sleep_stages.ruleset.rules.0.stage', value: 'rem' }], + [{ path: 'sleep_stages.ruleset.rules', value: [{ stage: 'unknown', when: [] }] }], + [{ path: 'sleep_stages.ruleset.fallback', value: null }], + [{ path: 'heart_rate', value: { trackingToleranceBpm: 10 } }], + [{ path: 'heart_rate.trackingToleranceBpm', value: 10 }, { path: 'heart_rate.trackingToleranceBpm', value: 12 }], + [{ path: '__proto__.polluted', value: true }], + [{ path: 'sleep_stages.ruleset.lowSignalBelow', value: 1.01 }], + [{ path: 'sleep_stages.ruleset.rules', value: Array(33).fill({ stage: 'wake', when: [] }) }], +] + +describe('versioned biometric contract', () => { + it('pins the actual numerical source and locked dependency files', () => { + for (const [path, hash] of Object.entries({ ...biometricManifest.sourceHashes, ...biometricManifest.dependencyHashes })) { + expect(createHash('sha256').update(readFileSync(path)).digest('hex'), path).toBe(hash) + } + }) + + it('preserves the shipped classifier and labels HRV correctly', () => { + expect(resolveConfig([]).resolved.sleep_stages).toEqual(DEFAULT_STAGE_CLASSIFIER) + expect(stageClassifierSchema.parse(defaultConfig.sleep_stages)).toEqual(defaultConfig.sleep_stages) + expect(biometricManifest.components.hrv_index.metricId).toBe('window_ibi_variability_v1') + expect(biometricManifest.components.sleep_stages.causal).toBe(false) + expect(validateConfig(defaultConfig)).toEqual(defaultConfig) + }) + + it('keeps false/zero intent, replaces ordered arrays, and removes overrides explicitly', () => { + const overrides = fixtures[2] + const profile = resolveConfig(overrides) + expect(profile.resolved.sleep_stages.ruleset.lowSignalBelow).toBe(0) + expect(profile.resolved.sleep_stages.ruleset.smoothIsolatedEpochs).toBe(false) + expect(resolveConfig(removeOverride(overrides, 'sleep_stages.ruleset.lowSignalBelow')).resolved.sleep_stages.ruleset.lowSignalBelow).toBe(0.3) + expect(resolveConfig(fixtures[4]).resolved.sleep_stages.ruleset.rules).toEqual([]) + const reversed = structuredClone(fixtures[3]) + ;(reversed[0].value as unknown[]).reverse() + expect(resolveConfig(reversed).hash).not.toBe(resolveConfig(fixtures[3]).hash) + expect(() => { + profile.resolved.heart_rate.trackingToleranceBpm = 25 + }).toThrow() + }) + + it('preserves every legacy leaf and first-match rule ordering', () => { + const legacy = structuredClone(DEFAULT_STAGE_CLASSIFIER) + legacy.ruleset.rules.reverse() + legacy.ruleset.smoothIsolatedEpochs = false + const result = resolveConfig(legacyStageOverrides(legacy)) + expect(result.resolved.sleep_stages).toEqual(legacy) + expect(result.resolved.heart_rate).toEqual(defaultConfig.heart_rate) + expect(() => resolveConfig([], 'unknown-engine')).toThrow('PROFILE_INCOMPATIBLE') + }) + + it('rejects every invalid contract fixture', () => { + for (const fixture of fixtures.slice(5)) expect(() => resolveConfig(fixture)).toThrow() + expect(() => validateConfig({ ...defaultConfig, sensorCalibration: {} })).toThrow() + }) + + it('matches Python validation and configuration hashes on the same fixtures', () => { + const python = execFileSync('python3', ['-c', ` +import json, sys +sys.path.insert(0, 'modules') +from common.biometrics.config import resolve_config +results = [] +for fixture in json.load(sys.stdin): + try: + result = resolve_config(fixture) + results.append({'hash': result['hash'], 'manifestHash': result['manifestHash']}) + except ValueError: + results.append(None) +print(json.dumps(results)) +`], { input: JSON.stringify(fixtures), encoding: 'utf8' }) + expect(JSON.parse(python)).toEqual(fixtures.map((fixture) => { + try { + const result = resolveConfig(fixture) + return { hash: result.hash, manifestHash } + } + catch { return null } + })) + }) +}) + +describe('finite-binary64-json-v1', () => { + it('uses deterministic key order and finite numeric JSON, preserving array order', () => { + expect(canonicalJson({ b: -0, a: 0.5 })).toBe('{"a":0.5,"b":0}') + expect(configHash({ a: 1, b: 2 })).toBe(configHash({ b: 2, a: 1 })) + for (const invalid of [NaN, Infinity, undefined, new Date(), { a: undefined }, '\ud800', Array(2)]) { + expect(() => canonicalJson(invalid)).toThrow() + } + expect(() => canonicalJson(Array(20001).fill(null))).toThrow('resource limit') + let deep: unknown = null + for (let i = 0; i < 33; i++) deep = [deep] + expect(() => canonicalJson(deep)).toThrow('resource limit') + }) + + it('matches Python for float edge cases, control characters, and Unicode key order', () => { + const values = [0.1, -0, 1e-7, 1e21, Number.MIN_VALUE, Number.MAX_VALUE, 1.23456789012345, + { '\u{10000}': '🌙\n\t', '\ue000': 'é', '\u0000': '\b\f\r' }] + const output = execFileSync('python3', ['-c', ` +import json, sys +sys.path.insert(0, 'modules') +from common.biometrics.config import canonical_json +print(json.dumps([canonical_json(v) for v in json.load(sys.stdin)], ensure_ascii=False)) +`], { input: JSON.stringify(values), encoding: 'utf8' }) + expect(JSON.parse(output)).toEqual(values.map(canonicalJson)) + for (const value of values) expect(JSON.parse(canonicalJson(value))).toEqual(value === 0 ? 0 : value) + }) +}) + +it.each([ + ['wrong object type', { ...defaultConfig, heart_rate: [] }], + ['missing required property', { ...defaultConfig, heart_rate: { trackingToleranceBpm: 15 } }], + ['null component', { ...defaultConfig, heart_rate: null }], + ['wrong contract version', { ...defaultConfig, sleep_stages: { ...defaultConfig.sleep_stages, version: 99 } }], +])('rejects %s instead of silently filling an immutable snapshot', (_reason, config) => { + expect(() => validateConfig(config)).toThrow() +}) +it.each([ + { path: 'heart_rate.trackingToleranceBpm', value: 4 }, + { path: 'heart_rate.trackingToleranceBpm', value: 31 }, + { path: 'sleep_stages.ruleset.smoothIsolatedEpochs', value: 1 }, + { path: 'sleep_stages.ruleset.rules', value: {} }, + { path: 'sleep_stages.ruleset.rules', value: [null] }, + { path: 'sleep_stages.ruleset.rules', value: [{ stage: 'wake' }] }, + { path: '', value: 1 }, + { path: 'heart_rate.trackingToleranceBpm', value: 15, extra: true }, +])('rejects malformed override $path with value $value', (override) => { + expect(() => resolveConfig([override])).toThrow() +}) +it('bounds override intake before resolving or mutating defaults', () => { + expect(() => resolveConfig(Array.from({ length: 65 }, () => ({ path: 'heart_rate.trackingToleranceBpm', value: 15 })))).toThrow('maximum 64') + expect(defaultConfig.heart_rate.trackingToleranceBpm).toBe(15) +}) + +it('validates constants, required keys, array bounds and string leaves from the manifest', () => { + expect(() => validateConfig({ ...defaultConfig, sleep_stages: { ...defaultConfig.sleep_stages, engine: 'other' } })).toThrow('Invalid biometric parameter') + expect(() => validateConfig({ ...defaultConfig, sleep_stages: { ...defaultConfig.sleep_stages, engine: 7 } })).toThrow('Invalid biometric parameter') + const missing = Object.fromEntries(Object.entries(defaultConfig).filter(([key]) => key !== 'heart_rate')) + expect(() => validateConfig(missing)).toThrow('Invalid biometric parameter') + expect(() => validateConfig({ ...defaultConfig, heart_rate: [] })).toThrow('Invalid biometric parameter') + const rules = Array.from({ length: 65 }, () => defaultConfig.sleep_stages.ruleset.rules[0]) + expect(() => validateConfig({ ...defaultConfig, sleep_stages: { ...defaultConfig.sleep_stages, ruleset: { ...defaultConfig.sleep_stages.ruleset, rules } } })).toThrow('Invalid biometric parameter') + expect(() => validateConfig({ ...defaultConfig, sleep_stages: { ...defaultConfig.sleep_stages, ruleset: { ...defaultConfig.sleep_stages.ruleset, outliers: { ...defaultConfig.sleep_stages.ruleset.outliers, heartRate: { min: 100, max: 50 } } } } })).toThrow('Invalid ordered range') +}) diff --git a/src/biometrics/background-admission.ts b/src/biometrics/background-admission.ts new file mode 100644 index 000000000..9299d2b75 --- /dev/null +++ b/src/biometrics/background-admission.ts @@ -0,0 +1,35 @@ +import { ReferenceInputError } from './references/time' + +type Lane = 'read' | 'work' +interface Slot { token: symbol, quarantined: boolean } +const shared = globalThis as typeof globalThis & { __biometricBackgroundSlotsV1?: Partial> } + +/** Application-process admission, shared across separately bundled modules. + * Kernel numerical-worker locking remains necessary across processes/restarts. */ +export function withBiometricBackgroundSlot(lane: Lane, action: () => Promise): Promise { + const slots = shared.__biometricBackgroundSlotsV1 ??= {} + if (slots[lane]) return Promise.reject(new ReferenceInputError('RESOURCE_LIMIT', slots[lane].quarantined + ? 'Biometrics background cleanup is unconfirmed; restart after verifying process cleanup' + : 'Biometrics background capacity is busy; retry later')) + const slot = { token: Symbol(lane), quarantined: false } + slots[lane] = slot + const release = (error?: unknown) => { + if (slots[lane]?.token !== slot.token) return + const code = error && typeof error === 'object' && 'code' in error ? error.code : null + if (code === 'SHADOW_CLEANUP_UNCONFIRMED' || code === 'BIOMETRIC_COORDINATOR_CLEANUP_UNCONFIRMED') slot.quarantined = true + else slots[lane] = undefined + } + try { + return action().then((result) => { + release() + return result + }, (error: unknown) => { + release(error) + throw error + }) + } + catch (error) { + release(error) + throw error + } +} diff --git a/src/biometrics/backtest/admission-service.ts b/src/biometrics/backtest/admission-service.ts new file mode 100644 index 000000000..6c19726a4 --- /dev/null +++ b/src/biometrics/backtest/admission-service.ts @@ -0,0 +1,59 @@ +import type { HealthAdmission } from './job-operations' +import { dirname } from 'node:path' +import { sql } from 'drizzle-orm' +import { z } from 'zod' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import type * as schema from '@/src/db/biometrics-schema' +import { admissionPolicySchema, collectAdmissionSample, evaluateAdmission } from './admission' + +export const replayAdmissionSchema = z.object({ + configured: z.boolean(), qualificationId: z.string().nullable(), admitted: z.boolean(), + reasons: z.array(z.enum(['unqualified', 'health_unavailable', 'stale_sample', 'clock_skew', + 'memory_pressure', 'disk_pressure', 'cpu_pressure', 'event_loop_delay', 'piezo_stale', 'temperature'])), +}).strict() + +/** Server-owned policy only. Missing or malformed configuration never enables + * background I/O. The qualification ID is operator-supplied evidence attribution, + * not an assertion that supplying JSON has qualified the hardware. + */ +async function measuredReplayAdmission(db: BetterSQLite3Database): Promise & { receipt?: Awaited> }> { + const policy = configuredReplayPolicy() + if (!policy) return { configured: false, qualificationId: null, admitted: false, reasons: ['unqualified'] } + try { + const path = db.all<{ name: string, file: string }>(sql`PRAGMA database_list`).find(row => row.name === 'main')?.file + if (!path) throw new Error('Missing persistent database') + const { getSensorFrameTimes } = await import('@/src/streaming/piezoStream') + const sample = await collectAdmissionSample(dirname(path), () => getSensorFrameTimes()['piezo-dual']) + const checkedAtMs = Date.now() + const decision = evaluateAdmission(sample, policy, checkedAtMs) + return { configured: true, qualificationId: policy.qualificationId, ...decision, + receipt: { allowed: decision.admitted, sample: decision.admitted && sample + ? { qualificationId: policy.qualificationId, checkedAtMs, ingestionAgeMs: checkedAtMs - sample.piezoReceivedAtMs } + : null } } + } + catch { return { configured: true, qualificationId: policy.qualificationId, admitted: false, reasons: ['health_unavailable'] } } +} + +export async function replayAdmission(db: BetterSQLite3Database) { + const { configured, qualificationId, admitted, reasons } = await measuredReplayAdmission(db) + return replayAdmissionSchema.parse({ configured, qualificationId, admitted, reasons }) +} + +export function replayHealthAdmission(db: BetterSQLite3Database): HealthAdmission { + return async () => { + const measured = await measuredReplayAdmission(db) + return measured.receipt ?? { allowed: false, sample: null } + } +} + +export function replayAdmitter(db: BetterSQLite3Database) { + return async () => (await replayAdmission(db)).admitted +} + +/** Reading configuration has no database, streaming or health side effects. */ +export function configuredReplayPolicy() { + try { + return admissionPolicySchema.parse(JSON.parse(process.env.SLEEPYPOD_BIOMETRICS_ADMISSION_POLICY ?? 'null')) + } + catch { return null } +} diff --git a/src/biometrics/backtest/admission.ts b/src/biometrics/backtest/admission.ts new file mode 100644 index 000000000..fdcaec507 --- /dev/null +++ b/src/biometrics/backtest/admission.ts @@ -0,0 +1,82 @@ +import { readFile, statfs } from 'node:fs/promises' +import { availableParallelism, loadavg } from 'node:os' +import { performance } from 'node:perf_hooks' +import { z } from 'zod' + +/** Limits must come from an explicitly qualified deployment policy. There is + * no fallback that silently enables replay on an unmeasured device. */ +export const admissionPolicySchema = z.object({ + qualificationId: z.string().min(1).max(200), + minimumAvailableMemoryBytes: z.number().int().positive(), + minimumFreeDiskBytes: z.number().int().positive(), + maximumLoadPerCpu: z.number().positive(), + maximumEventLoopDelayMs: z.number().positive(), + maximumPiezoAgeMs: z.number().int().positive(), + maximumTemperatureC: z.number().positive(), + maximumSampleAgeMs: z.number().int().positive(), +}).strict() +export type AdmissionPolicy = z.infer +const sampleSchema = z.object({ + measuredAtMs: z.number().int().nonnegative(), + availableMemoryBytes: z.number().int().nonnegative(), + freeDiskBytes: z.number().int().nonnegative(), + loadPerCpu: z.number().nonnegative(), + eventLoopDelayMs: z.number().nonnegative(), + piezoReceivedAtMs: z.number().int().nonnegative(), + temperatureC: z.number().min(-50).max(150), +}).strict() +export type AdmissionSample = z.infer +export type AdmissionReason = 'unqualified' | 'health_unavailable' | 'stale_sample' | 'clock_skew' + | 'memory_pressure' | 'disk_pressure' | 'cpu_pressure' | 'event_loop_delay' | 'piezo_stale' | 'temperature' + +export function evaluateAdmission(rawSample: unknown, rawPolicy: unknown, now = Date.now()): { admitted: boolean, reasons: AdmissionReason[] } { + const policy = admissionPolicySchema.safeParse(rawPolicy) + if (!policy.success) return { admitted: false, reasons: ['unqualified'] } + const sample = sampleSchema.safeParse(rawSample) + if (!sample.success || !Number.isSafeInteger(now) || now < 0) return { admitted: false, reasons: ['health_unavailable'] } + const s = sample.data, p = policy.data + const reasons: AdmissionReason[] = [] + if (s.measuredAtMs > now || s.piezoReceivedAtMs > now) reasons.push('clock_skew') + if (now - s.measuredAtMs > p.maximumSampleAgeMs) reasons.push('stale_sample') + if (s.availableMemoryBytes < p.minimumAvailableMemoryBytes) reasons.push('memory_pressure') + if (s.freeDiskBytes < p.minimumFreeDiskBytes) reasons.push('disk_pressure') + if (s.loadPerCpu > p.maximumLoadPerCpu) reasons.push('cpu_pressure') + if (s.eventLoopDelayMs > p.maximumEventLoopDelayMs) reasons.push('event_loop_delay') + if (now - s.piezoReceivedAtMs > p.maximumPiezoAgeMs) reasons.push('piezo_stale') + if (s.temperatureC > p.maximumTemperatureC) reasons.push('temperature') + return { admitted: reasons.length === 0, reasons } +} + +export function parseAvailableMemory(text: string) { + const match = /^MemAvailable:\s+(\d+)\s+kB\s*$/m.exec(text) + if (!match) throw new Error('Available memory is unavailable') + const bytes = Number(match[1]) * 1024 + if (!Number.isSafeInteger(bytes) || bytes < 0) throw new Error('Invalid available memory') + return bytes +} + +/** No hardware commands, database queries, or lifetime histogram. Measure the + * scheduling delay of this process and only the live piezo frame timestamp. */ +export async function collectAdmissionSample(databaseDirectory: string, piezoReceivedAt: () => number | undefined): Promise { + const measuredAtMs = Date.now() + try { + // Probe Linux health files directly. Turbopack folds both process.platform + // and os.platform() to the build host; missing runtime files fail closed. + const probeStarted = performance.now() + const [memory, disk, thermal, delay] = await Promise.all([ + readFile('/proc/meminfo', 'utf8'), + statfs(databaseDirectory), + readFile('/sys/class/thermal/thermal_zone0/temp', 'utf8'), + new Promise(resolve => setTimeout(() => resolve(Math.max(0, performance.now() - probeStarted - 20)), 20)), + ]) + const rawTemperature = thermal.trim() + if (!/^-?\d+$/.test(rawTemperature)) return null + const parsed = sampleSchema.safeParse({ + measuredAtMs, availableMemoryBytes: parseAvailableMemory(memory), freeDiskBytes: disk.bavail * disk.bsize, + loadPerCpu: loadavg()[0] / availableParallelism(), eventLoopDelayMs: delay, + piezoReceivedAtMs: piezoReceivedAt(), temperatureC: Number(rawTemperature) / 1000, + }) + return parsed.success ? parsed.data : null + } + catch { return null } +} diff --git a/src/biometrics/backtest/artifacts.ts b/src/biometrics/backtest/artifacts.ts new file mode 100644 index 000000000..b82330bb9 --- /dev/null +++ b/src/biometrics/backtest/artifacts.ts @@ -0,0 +1,161 @@ +import Database from 'better-sqlite3' +import { createHash, randomUUID } from 'node:crypto' +import { closeSync, fsyncSync, lstatSync, mkdirSync, openSync, readFileSync, readdirSync, renameSync, unlinkSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { z } from 'zod' +import { ReferenceInputError } from '../references/time' + +export const REPORT_BUDGET_BYTES = 64 * 1024 * 1024 +export const MAX_REPORT_BYTES = 16 * 1024 * 1024 +export const reportArtifactSchema = z.object({ kind: z.literal('biometric-report-artifact-v1'), + name: z.string().regex(/^[a-f0-9-]{36}\.[1-9][0-9]*\.[a-f0-9]{64}\.json$/), + byteLength: z.number().int().positive().max(MAX_REPORT_BYTES), sha256: z.string().regex(/^[a-f0-9]{64}$/), +}).strict() +export type ReportArtifact = z.infer +const ownedName = /^[a-f0-9-]{36}\.[1-9][0-9]*\.[a-f0-9]{64}\.json(?:\.[a-f0-9-]{36}\.tmp)?$/ + +/** Trusted evaluator artifacts only. The root comes from the server's DB path, + * never a request. Files are immutable; a fenced DB transaction publishes the + * descriptor only after fsync + atomic rename. Failed publication leaves an + * unreferenced artifact for bounded cleanup, never a partially visible result. */ +export class ReportArtifactStore { + private publishing = false + constructor(private readonly root: string) {} + + /** A separate SQLite file provides an OS-released cross-process lock. Long + * file writes never hold the live biometrics DB transaction. All publishers + * and collectors take this lock before reading live descriptor/quota state. */ + withPublication(operation: () => T): T { + if (this.publishing) return operation() + mkdirSync(this.root, { recursive: true, mode: 0o700 }) + const lock = new Database(join(this.root, '.publication.sqlite'), { timeout: 1000 }) + try { + return lock.transaction(() => { + this.publishing = true + try { + return operation() + } + finally { + this.publishing = false + } + }).immediate() + } + finally { + lock.close() + } + } + + usage() { + mkdirSync(this.root, { recursive: true, mode: 0o700 }) + const names = readdirSync(this.root) + if (names.length > 4096) throw new ReferenceInputError('RESOURCE_LIMIT', 'Too many report artifacts') + let bytes = 0 + for (const name of names) { + if (!ownedName.test(name)) continue + const stat = lstatSync(join(this.root, name)) + if (!stat.isFile()) throw new ReferenceInputError('RESULT_CORRUPT', 'Unexpected report artifact type') + bytes += stat.size + } + return bytes + } + + write(jobId: string, token: number, serialized: string, inlineBytes: number): ReportArtifact { + return this.withPublication(() => { + z.string().uuid().parse(jobId) + z.number().int().positive().parse(token) + z.number().int().nonnegative().parse(inlineBytes) + const byteLength = Buffer.byteLength(serialized) + if (!byteLength || byteLength > MAX_REPORT_BYTES) throw new ReferenceInputError('RESOURCE_LIMIT', 'Report exceeds 16 MiB') + const sha256 = createHash('sha256').update(serialized).digest('hex') + const name = `${jobId}.${token}.${sha256}.json` + const descriptor = reportArtifactSchema.parse({ kind: 'biometric-report-artifact-v1', name, byteLength, sha256 }) + const used = this.usage() + try { + const stat = lstatSync(join(this.root, name)) + if (stat.isFile() && stat.size === byteLength) { + const bytes = readFileSync(join(this.root, name)) + if (createHash('sha256').update(bytes).digest('hex') === sha256) { + if (used + inlineBytes > REPORT_BUDGET_BYTES) throw new ReferenceInputError('RESOURCE_LIMIT', 'Reports reached the 64 MiB storage budget') + return descriptor + } + } + } + catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error + } + if (used + byteLength + inlineBytes > REPORT_BUDGET_BYTES) throw new ReferenceInputError('RESOURCE_LIMIT', 'Reports reached the 64 MiB storage budget') + const temporary = join(this.root, `${name}.${randomUUID()}.tmp`) + let fd: number | undefined + try { + fd = openSync(temporary, 'wx', 0o600) + writeFileSync(fd, serialized) + fsyncSync(fd) + closeSync(fd) + fd = undefined + renameSync(temporary, join(this.root, name)) + const directory = openSync(this.root, 'r') + try { + fsyncSync(directory) + } + finally { + closeSync(directory) + } + return descriptor + } + finally { + if (fd !== undefined) closeSync(fd) + try { + unlinkSync(temporary) + } + catch { /* Atomic rename or cleanup after a failed write. */ } + } + }) + } + + read(raw: unknown): unknown { + const parsed = reportArtifactSchema.safeParse(raw) + if (!parsed.success) throw new ReferenceInputError('RESULT_CORRUPT', 'Invalid report artifact descriptor') + const descriptor = parsed.data + try { + const path = join(this.root, descriptor.name) + const stat = lstatSync(path) + if (!stat.isFile() || stat.size !== descriptor.byteLength) throw new Error('Artifact size/type mismatch') + const bytes = readFileSync(path) + if (createHash('sha256').update(bytes).digest('hex') !== descriptor.sha256) throw new Error('Artifact hash mismatch') + return JSON.parse(bytes.toString('utf8')) + } + catch { + throw new ReferenceInputError('RESULT_CORRUPT', 'Report artifact is missing or corrupt') + } + } + + /** Called only after the descriptor has been detached in a committed DB + * transaction under the publication lock. Failed unlink remains GC work. */ + removeReleased(raw: ReportArtifact) { + const descriptor = reportArtifactSchema.parse(raw) + if (!this.publishing) throw new Error('Artifact release requires the publication lock') + try { + unlinkSync(join(this.root, descriptor.name)) + } + catch { /* Retained orphan remains counted against the filesystem quota. */ } + } + + /** Protect freshly staged files from a concurrent publisher. Only files + * outside the committed descriptor set and older than one hour are removed. */ + collect(referenced: ReadonlySet, now = Date.now()) { + return this.withPublication(() => { + this.usage() + let removed = 0 + for (const name of readdirSync(this.root)) { + if (!ownedName.test(name) || referenced.has(name)) continue + const path = join(this.root, name) + const stat = lstatSync(path) + if (stat.isFile() && now - stat.mtimeMs > 3600000) { + unlinkSync(path) + removed++ + } + } + return removed + }) + } +} diff --git a/src/biometrics/backtest/child.ts b/src/biometrics/backtest/child.ts new file mode 100644 index 000000000..6fc6330a2 --- /dev/null +++ b/src/biometrics/backtest/child.ts @@ -0,0 +1,89 @@ +import { createHash } from 'node:crypto' +import { readFile } from 'node:fs/promises' +import { spawn } from 'node:child_process' +import { StringDecoder } from 'node:string_decoder' +import { tmpdir, setPriority } from 'node:os' +import { z } from 'zod' + +const eventSchema = z.discriminatedUnion('type', [ + z.object({ type: z.literal('progress'), completedNights: z.number().int().min(0).max(60), totalNights: z.number().int().min(0).max(60) }).strict(), + z.object({ type: z.literal('result'), report: z.object({ version: z.enum(['processed-backtest-report-v1', 'waveform-backtest-report-v1']) }).passthrough() }).strict(), +]) + +export async function workerBuildHash(workerPath: string) { + return createHash('sha256').update(await readFile(workerPath)).digest('hex') +} + +/** Executes only the shipped worker path selected by the server, never uploads. */ +export function runBacktestChild(workerPath: string, input: unknown, options: { + signal?: AbortSignal + timeoutMs?: number + onProgress?: (completed: number, total: number) => void +} = {}): Promise { + const serialized = JSON.stringify(input) + if (!serialized || Buffer.byteLength(serialized) > 10 * 1024 * 1024) return Promise.reject(new Error('Replay input exceeds 10 MiB')) + const timeoutMs = options.timeoutMs ?? 120000 + if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 10 || timeoutMs > 1800000) return Promise.reject(new Error('Invalid replay time budget')) + if (options.signal?.aborted) return Promise.reject(new Error('Replay cancelled')) + return new Promise((resolve, reject) => { + const child = spawn(process.execPath, ['--max-old-space-size=128', workerPath], { + cwd: tmpdir(), env: { NODE_ENV: 'production', LANG: 'C.UTF-8', TZ: 'UTC' }, stdio: ['pipe', 'pipe', 'pipe'], + }) + const decoder = new StringDecoder('utf8') + let stdout = '', outputBytes = 0, stderrBytes = 0 + let result: unknown, failure: Error | undefined, killTimer: ReturnType | undefined + const stop = (error: Error) => { + failure ??= error + child.kill('SIGTERM') + killTimer ??= setTimeout(() => child.kill('SIGKILL'), 500) + } + if (child.pid) { + try { + setPriority(child.pid, 19) + } + catch { + stop(new Error('Could not lower replay process priority')) + } + } + const timeout = setTimeout(() => stop(new Error('Replay exceeded its time budget')), timeoutMs) + const abort = () => stop(new Error('Replay cancelled')) + options.signal?.addEventListener('abort', abort, { once: true }) + child.on('error', error => stop(error)) + child.stdin.on('error', () => stop(new Error('Replay input pipe closed'))) + child.stderr.on('data', (chunk: Buffer) => { + stderrBytes += chunk.length + // Do not retain or log health exports through a worker error channel. + if (stderrBytes > 16384) stop(new Error('Replay diagnostic output exceeded its bound')) + }) + child.stdout.on('data', (chunk: Buffer) => { + outputBytes += chunk.length + if (outputBytes > 16 * 1024 * 1024) return stop(new Error('Replay output exceeded 16 MiB')) + stdout += decoder.write(chunk) + let newline: number + while ((newline = stdout.indexOf('\n')) >= 0) { + const line = stdout.slice(0, newline) + stdout = stdout.slice(newline + 1) + try { + const event = eventSchema.parse(JSON.parse(line)) + if (result !== undefined) throw new Error('Worker emitted data after its result') + if (event.type === 'progress') options.onProgress?.(event.completedNights, event.totalNights) + else result = event.report + } + catch { + stop(new Error('Replay emitted an invalid event or progress update')) + return + } + } + }) + child.on('close', (code) => { + clearTimeout(timeout) + if (killTimer) clearTimeout(killTimer) + options.signal?.removeEventListener('abort', abort) + stdout += decoder.end() + if (failure) reject(failure) + else if (code !== 0 || result === undefined || stdout.length) reject(new Error('Replay worker failed or returned an incomplete report')) + else resolve(result) + }) + child.stdin.end(serialized) + }) +} diff --git a/src/biometrics/backtest/common-support.ts b/src/biometrics/backtest/common-support.ts new file mode 100644 index 000000000..102c12a33 --- /dev/null +++ b/src/biometrics/backtest/common-support.ts @@ -0,0 +1,34 @@ +export interface SupportInterval { start: number, end: number } + +function ordered(intervals: T[]) { + if (intervals.length > 100000) throw new Error('Too many support intervals') + const sorted = [...intervals].sort((a, b) => a.start - b.start) + if (sorted.some((row, index) => !Number.isSafeInteger(row.start) || !Number.isSafeInteger(row.end) || row.end <= row.start + || (index > 0 && sorted[index - 1].end > row.start))) throw new Error('Support intervals must be valid and nonoverlapping') + return sorted +} + +/** Linear sweep after validation/sorting; preserve labels from the left input. */ +export function restrictToSupport(input: T[], support: SupportInterval[]): T[] { + const left = ordered(input), right = ordered(support) + const result: T[] = [] + let i = 0, j = 0 + while (i < left.length && j < right.length) { + const a = left[i], b = right[j] + const start = Math.max(a.start, b.start), end = Math.min(a.end, b.end) + if (end > start) result.push({ ...a, start, end }) + if (a.end <= b.end) i++ + else j++ + } + return result +} + +/** One common denominator across every submitted configuration. An absent + * output empties the intersection; it never silently drops out of the set. */ +export function intersectOutputSupport(outputs: SupportInterval[][], start: number, end: number) { + if (!Number.isSafeInteger(start) || !Number.isSafeInteger(end) || end <= start || end - start > 86400000 + || outputs.length < 1 || outputs.length > 3) throw new Error('Invalid bounded common-support input') + let support: SupportInterval[] = [{ start, end }] + for (const output of outputs) support = restrictToSupport(support, output) + return support +} diff --git a/src/biometrics/backtest/contracts.ts b/src/biometrics/backtest/contracts.ts new file mode 100644 index 000000000..2e64f84b5 --- /dev/null +++ b/src/biometrics/backtest/contracts.ts @@ -0,0 +1,88 @@ +import { stageObjectivePolicySchema } from './stage-objective' +import { z } from 'zod' +import { objectivePolicySchema, type ObjectivePolicy } from './objective-policy' +import { waveformDatasetInputsSchema } from './waveform-contracts' +import { nightBootstrapSchema } from './uncertainty-policy' +import { normalizedMeasurementSchema } from '../references/contracts' +const key = z.string().min(1).max(200) +const hash = z.string().regex(/^[a-f0-9]{64}$/) +export const experimentProtocolSchema = z.object({ + version: z.literal('descriptive-evaluation-v1'), + mode: z.enum(['comparison', 'stage_replay', 'signal_replay']), + waveformStages: z.boolean().optional(), + // Explicit approximation, never represented as measured sensor calibration. + assumedStageCalibrationQuality: z.number().min(0).max(1), + pointToleranceMs: z.number().int().min(0).max(300000), + clockOffsetMs: z.number().int().min(-3600000).max(3600000), + nightUncertainty: nightBootstrapSchema.optional(), + objectivePolicy: objectivePolicySchema.optional(), + objectivePolicies: z.array(objectivePolicySchema).min(1).max(3).optional(), + stageObjectivePolicy: stageObjectivePolicySchema.optional(), +}).strict().refine(protocol => !(protocol.objectivePolicy && protocol.objectivePolicies), 'Use one numeric objective policy representation') + .refine(protocol => new Set(numericObjectivePolicies(protocol).map(policy => policy.metric)).size === numericObjectivePolicies(protocol).length, 'Numeric objective metrics must be unique') + .refine(protocol => !numericObjectivePolicies(protocol).length || protocol.mode === 'signal_replay', 'Numeric objectives require waveform replay') + .refine(protocol => !protocol.stageObjectivePolicy || protocol.mode === 'stage_replay' || (protocol.mode === 'signal_replay' && protocol.waveformStages === true), 'Stage objectives require stage replay') +/** Preserve old protocol hashes: normalize at consumption, never mutate snapshots. */ +export function numericObjectivePolicies(protocol: { objectivePolicy?: ObjectivePolicy, objectivePolicies?: ObjectivePolicy[] }): ObjectivePolicy[] { + return protocol.objectivePolicies ?? (protocol.objectivePolicy ? [protocol.objectivePolicy] : []) +} +export const replayBaselineProvenanceSchema = z.object({ source: z.enum(['acknowledged_runtime', 'unacknowledged_defaults']), + generation: z.number().int().nonnegative().nullable(), effectiveSampleTimeMs: z.number().int().nonnegative().nullable(), warning: z.string().max(1000).nullable() }).strict() +export const enqueueBacktestSchema = z.object({ + experimentId: z.string().uuid(), idempotencyKey: key, evaluatorBuildHash: hash, submissionHash: hash.optional(), + waveform: z.object({ pythonBuildHash: hash, baselineProvenance: z.object({ current: replayBaselineProvenanceSchema, otherSide: replayBaselineProvenanceSchema }).strict().optional(), otherSide: z.object({ manifestHash: hash, resolvedHash: hash, resolved: z.unknown() }).strict() }).strict().optional(), + split: z.enum(['tuning', 'validation', 'holdout']), + configs: z.array(z.object({ label: z.enum(['default', 'current', 'candidate']), revisionId: z.string().uuid().nullable(), + manifestHash: hash, resolvedHash: hash, resolved: z.unknown() }).strict()).min(1).max(3), +}).strict() + +export const backtestJobSnapshotSchema = enqueueBacktestSchema.extend({ version: z.literal('backtest-job-v1'), datasetId: z.string().uuid(), + datasetHash: z.string(), protocol: experimentProtocolSchema, protocolHash: z.string(), holdoutUse: z.number().int().nonnegative(), + evidence: z.enum(['descriptive_only', 'reused_holdout_is_validation']), +}) + +const vitalSchema = z.object({ id: z.number().int(), timestampMs: z.number().int(), heartRate: z.number().nullable(), + hrv: z.number().nullable(), breathingRate: z.number().nullable(), processingRunId: z.string().nullable() }) +const movementSchema = z.object({ id: z.number().int(), timestampMs: z.number().int(), totalMovement: z.number() }) +const processedDatasetSnapshotSchema = z.object({ + version: z.literal('processed-dataset-v1'), side: z.enum(['left', 'right']), subjectId: z.string(), + sessions: z.array(z.object({ id: z.number().int(), startMs: z.number().int(), endMs: z.number().int(), + sleepDurationSeconds: z.number(), timesExitedBed: z.number(), presentIntervals: z.unknown(), notPresentIntervals: z.unknown(), + vitals: z.array(vitalSchema), movement: z.array(movementSchema), + })), + references: z.array(z.object({ importId: z.string(), revision: z.number(), contentHash: z.string(), source: z.string(), + timePolicy: z.unknown(), measurements: z.array(normalizedMeasurementSchema) })), + processingRuns: z.array(z.object({ id: z.string(), side: z.string(), kind: z.string(), bundle: z.unknown(), + configHash: z.string(), calibrationHash: z.string().nullable(), inputIdentity: z.unknown(), startedAtMs: z.number(), endedAtMs: z.number().nullable() })), +}) +export const datasetSnapshotSchema = z.discriminatedUnion('version', [ + processedDatasetSnapshotSchema, + processedDatasetSnapshotSchema.extend({ version: z.literal('waveform-dataset-v1'), waveformInputs: waveformDatasetInputsSchema }), +]) +export type DatasetSnapshot = z.infer + +export const coverageSchema = z.object({ count: z.number(), duplicateTimestamps: z.number(), firstMs: z.number().nullable(), lastMs: z.number().nullable(), + expectedCadenceMs: z.number(), gaps: z.array(z.object({ startMs: z.number(), endMs: z.number() })), gapDurationMs: z.number(), + interpretation: z.literal('cadence_availability_not_measured_support') }) +export const datasetManifestSchema = z.object({ + version: z.enum(['processed-dataset-manifest-v1', 'waveform-dataset-manifest-v1']), + parentDataset: z.object({ id: z.string().uuid(), contentHash: z.string().regex(/^[a-f0-9]{64}$/) }).optional(), + splitProtocol: z.object({ version: z.literal('chronological-whole-night-v1'), timezone: z.string(), + timezoneRuntime: z.object({ icu: z.string(), tz: z.string() }), grouping: z.literal('local_noon_connected_sessions'), + assignments: z.array(z.object({ cohortId: z.string(), nightLabels: z.array(z.string()), sessionIds: z.array(z.number()), + startMs: z.number(), endMs: z.number(), split: z.enum(['tuning', 'validation', 'holdout']) })), + evidence: z.enum(['partitioned_not_validated', 'descriptive_only']), limitations: z.array(z.string()), + }), + coverage: z.array(z.object({ sessionId: z.number(), vitals: coverageSchema, movement: coverageSchema })), + capabilities: z.object({ comparison: z.boolean(), stageReplay: z.boolean(), signalReplay: z.boolean() }), limitations: z.array(z.string()), +}) +export const datasetDescriptorSchema = z.object({ id: z.string().uuid(), side: z.enum(['left', 'right']), subjectId: z.string(), + contentHash: z.string(), manifest: datasetManifestSchema, byteLength: z.number(), createdAtMs: z.number(), releasedAtMs: z.number().nullable(), available: z.boolean() }) + +export const createDatasetSchema = z.object({ + side: z.enum(['left', 'right']), subjectId: z.string().min(1).max(200), + sessionIds: z.array(z.number().int().positive()).min(1).max(60), + referenceImportIds: z.array(z.string().uuid()).max(20), + timezone: z.string().min(1).max(100), + idempotencyKey: z.string().min(1).max(200), +}).strict() diff --git a/src/biometrics/backtest/coordinator-entry.ts b/src/biometrics/backtest/coordinator-entry.ts new file mode 100644 index 000000000..9a0759424 --- /dev/null +++ b/src/biometrics/backtest/coordinator-entry.ts @@ -0,0 +1,76 @@ +import { admissionReceiptSchema, type AdmissionReceipt } from './job-operations' +import { replayBudgetSchema } from './execution-budget' +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import * as schema from '@/src/db/biometrics-schema' +import { ReferenceInputError } from '../references/time' +import { performCoordinatorTask } from './coordinator-tasks' + +// One operation per process. The supervisor owns this process group, including +// the evaluator grandchild; losing the supervisor IPC channel cancels all work. +const [databasePath, workerPath, rawLocationJson, replayBudgetJson] = process.argv.slice(2) +let started = false, completed = false, sequence = 0 +const controller = new AbortController() +let lastAdmission: AdmissionReceipt | null = null +const pending = new Map void>() +process.on('disconnect', () => { + if (completed) return + controller.abort() + // The launcher creates this process group. If it dies, clean up our child too. + if (process.platform !== 'win32') { + try { + process.kill(-process.pid, 'SIGKILL') + } + catch { /* Fall through if the supervisor already removed the group. */ } + } + process.exit(1) +}) +process.on('message', async (message: unknown) => { + const data = message as { type?: string, sequence?: number, allowed?: boolean, sample?: unknown, memory?: unknown, task?: unknown } + if (data?.type === 'cancel') { + controller.abort() + return + } + if (data?.type === 'admission' && typeof data.sequence === 'number') { + const receipt = admissionReceiptSchema.safeParse({ allowed: data.allowed, sample: data.sample ?? null, ...(data.memory === undefined ? {} : { memory: data.memory }) }) + pending.get(data.sequence)?.(receipt.success ? receipt.data : { allowed: false, sample: null }) + return + } + if (data?.type !== 'run' || started) return + started = true + let sqlite: Database.Database | undefined + try { + const replayBudget = replayBudgetJson && replayBudgetJson !== 'null' ? replayBudgetSchema.parse(JSON.parse(replayBudgetJson)) : undefined + sqlite = new Database(databasePath, { fileMustExist: true, timeout: 1000 }) + sqlite.pragma('foreign_keys = ON') + const db = drizzle(sqlite, { schema }) + const result = await performCoordinatorTask(db, data.task, { workerPath, replayBudget, rawLocation: rawLocationJson ? JSON.parse(rawLocationJson) ?? undefined : undefined, signal: controller.signal, admissionReceipt: () => lastAdmission, admit: () => new Promise((resolve) => { + const id = ++sequence + const finish = (receipt: AdmissionReceipt) => { + clearTimeout(timer) + pending.delete(id) + lastAdmission = receipt + resolve(receipt.allowed) + } + const timer = setTimeout(() => finish({ allowed: false, sample: null }), 2000) + pending.set(id, finish) + process.send?.({ type: 'admission', sequence: id }) + }) }) + if (Buffer.byteLength(JSON.stringify(result)) > 17 * 1024 * 1024) throw new Error('Coordinator response exceeds byte limit') + process.send?.({ type: 'result', result }, () => { + completed = true + process.disconnect?.() + }) + } + catch (error) { + // Known input errors preserve their codes; generic errors never log exports. + process.send?.({ type: 'error', code: error instanceof ReferenceInputError ? error.code : 'COORDINATOR_FAILED', + message: error instanceof ReferenceInputError ? error.message.slice(0, 1000) : 'Biometrics coordinator failed' }, () => { + completed = true + process.disconnect?.() + }) + } + finally { + sqlite?.close() + } +}) diff --git a/src/biometrics/backtest/coordinator-lanes.ts b/src/biometrics/backtest/coordinator-lanes.ts new file mode 100644 index 000000000..14477eec2 --- /dev/null +++ b/src/biometrics/backtest/coordinator-lanes.ts @@ -0,0 +1,8 @@ +import type { CoordinatorTask } from './coordinator-tasks' +export type CoordinatorLane = 'read' | 'work' +export const coordinatorTaskLanes: Record = { + readShadowReport: 'read', + pageRawSources: 'read', datasetDescriptor: 'read', readResult: 'read', readSummary: 'read', readTrace: 'read', readNumericTrace: 'read', readReferenceTrace: 'read', + createWaveformDataset: 'work', inspectRawSource: 'work', createDataset: 'work', releaseDataset: 'work', + releaseResult: 'work', createExperiment: 'work', collectArtifacts: 'work', execute: 'work', +} diff --git a/src/biometrics/backtest/coordinator-queue.ts b/src/biometrics/backtest/coordinator-queue.ts new file mode 100644 index 000000000..e39683dcd --- /dev/null +++ b/src/biometrics/backtest/coordinator-queue.ts @@ -0,0 +1,91 @@ +import { coordinatorTaskLanes as taskLanes, type CoordinatorLane as Lane } from './coordinator-lanes' +import { ReferenceInputError } from '../references/time' +import { runCoordinatorTask, type CoordinatorOptions } from './coordinator' +import type { CoordinatorTask, CoordinatorResults } from './coordinator-tasks' + +interface Control { signal?: AbortSignal, admit?: () => Promise } +interface Entry { lane: Lane, start: () => Promise, reject: (error: Error) => void } + +/** One serialized work slot plus one read slot, so replay cannot starve report + * access. No writes or scans use the read slot. Eight requests total and seven + * per lane reserve capacity for the other lane; pending waits remain bounded. + * Both processes count toward the deployment's admission/resource qualification. + */ +export class CoordinatorQueue { + private pending: Entry[] = [] + private active = new Map>() + private stopped = false + private abort = new AbortController() + constructor(private readonly options: CoordinatorOptions, private readonly execute = runCoordinatorTask) {} + + request(task: T, control: Control = {}): Promise { + if (this.stopped) return Promise.reject(new Error('Coordinator stopped')) + if (control.signal?.aborted) return Promise.reject(new Error('Coordinator cancelled')) + const lane = taskLanes[task.type] ?? 'work' + if (this.pending.length + this.active.size >= 8 + || this.pending.filter(entry => entry.lane === lane).length + (this.active.has(lane) ? 1 : 0) >= 7) return Promise.reject(new ReferenceInputError('RESOURCE_LIMIT', 'Biometrics coordinator queue is full')) + return new Promise((resolve, reject) => { + const controller = new AbortController() + let started = false, finished = false + const cleanup = () => { + clearTimeout(timer) + control.signal?.removeEventListener('abort', cancel) + this.abort.signal.removeEventListener('abort', cancel) + } + const fail = (error: Error) => { + if (finished) return + finished = true + cleanup() + this.pending = this.pending.filter(row => row !== entry) + reject(error) + } + const cancel = () => { + controller.abort() + if (!started) fail(new Error('Coordinator cancelled')) + } + const timer = setTimeout(() => fail(new ReferenceInputError('RESOURCE_LIMIT', 'Biometrics coordinator queue wait exceeded')), 30000) + const entry: Entry = { + lane, + reject: fail, + start: async () => { + started = true + clearTimeout(timer) + try { + const result = await this.execute(this.options, task, { ...control, signal: controller.signal }) + if (!finished) resolve(result) + } + catch (error) { if (!finished) reject(error) } + finally { + finished = true + cleanup() + } + }, + } + control.signal?.addEventListener('abort', cancel, { once: true }) + this.abort.signal.addEventListener('abort', cancel, { once: true }) + this.pending.push(entry) + this.pump() + }) + } + + private pump() { + if (this.stopped) return + for (const lane of ['read', 'work'] as const) { + if (this.active.has(lane)) continue + const index = this.pending.findIndex(entry => entry.lane === lane) + if (index === -1) continue + const [next] = this.pending.splice(index, 1) + this.active.set(lane, next.start().finally(() => { + this.active.delete(lane) + this.pump() + })) + } + } + + async stop() { + this.stopped = true + this.abort.abort() + for (const entry of [...this.pending]) entry.reject(new Error('Coordinator stopped')) + await Promise.all(this.active.values()) + } +} diff --git a/src/biometrics/backtest/coordinator-service.ts b/src/biometrics/backtest/coordinator-service.ts new file mode 100644 index 000000000..280884c99 --- /dev/null +++ b/src/biometrics/backtest/coordinator-service.ts @@ -0,0 +1,32 @@ +import type { HealthAdmission } from './job-operations' +import { replayBudgetSchema } from './execution-budget' +import { sql } from 'drizzle-orm' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import type * as schema from '@/src/db/biometrics-schema' +import { ReferenceInputError } from '../references/time' +import { CoordinatorQueue } from './coordinator-queue' +import { packagedCoordinatorPath, packagedWorkerPath } from './location' +import type { CoordinatorTask } from './coordinator-tasks' + +const shared = globalThis as typeof globalThis & { __biometricCoordinator?: { path: string, queue: CoordinatorQueue } } +export function coordinatorRequest(db: BetterSQLite3Database, task: T, control: { + signal?: AbortSignal + admit?: () => Promise + healthAdmission?: HealthAdmission +} = {}) { + const path = db.all<{ name: string, file: string }>(sql`PRAGMA database_list`).find(row => row.name === 'main')?.file + if (!path) throw new ReferenceInputError('WORKER_UNAVAILABLE', 'Coordinator requires a persistent biometrics database') + if (shared.__biometricCoordinator && shared.__biometricCoordinator.path !== path) throw new Error('Stop the coordinator before changing its database') + shared.__biometricCoordinator ??= { path, queue: new CoordinatorQueue({ databasePath: path, + coordinatorPath: packagedCoordinatorPath(), workerPath: packagedWorkerPath(), + replayBudget: process.env.SLEEPYPOD_BIOMETRICS_REPLAY_BUDGET === undefined + ? undefined + : replayBudgetSchema.parse(JSON.parse(process.env.SLEEPYPOD_BIOMETRICS_REPLAY_BUDGET)) }) } + return shared.__biometricCoordinator.queue.request(task, control) +} +export async function stopBiometricCoordinator() { + const current = shared.__biometricCoordinator + if (!current) return + await current.queue.stop() + if (shared.__biometricCoordinator === current) delete shared.__biometricCoordinator +} diff --git a/src/biometrics/backtest/coordinator-tasks.ts b/src/biometrics/backtest/coordinator-tasks.ts new file mode 100644 index 000000000..bcc8e6140 --- /dev/null +++ b/src/biometrics/backtest/coordinator-tasks.ts @@ -0,0 +1,170 @@ +import type { AdmissionReceipt } from './job-operations' +import type Database from 'better-sqlite3' +import { shadowReportRequestSchema } from '../shadow/report-contracts' +import { readShadowReport } from '../shadow/report-operation' +import { referenceTraceRequestSchema } from './reference-trace-page' +import { sql } from 'drizzle-orm' +import { dirname, join } from 'node:path' +import { runRawCollectionChild } from './raw-collection-child' +import type { ReplayBudget } from './execution-budget' +import { z } from 'zod' +import { summarizeBacktestResult } from './report-summary' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import type * as schema from '@/src/db/biometrics-schema' +import { DatasetStore, createDatasetSchema } from './datasets' +import { BacktestJobStore, experimentProtocolSchema } from './jobs' +import { readStageTraces, readNumericTraces, readReferenceTraces } from './traces' +import { traceRequestSchema } from './trace-contracts' +import { executeNextBacktest } from './runner' + +import { RawSourceCatalog, rawSourcePageSchema } from './raw-catalog' +import { rawScanInputSchema, runRawInspectionChild, podRawInspectionLocation, type RawInspectionLocation } from './raw-inspection-child' + +import { createWaveformDatasetSchema } from './waveform-selection' +import { publishSelectedWaveformDataset } from './waveform-publication' + +const idInput = z.object({ id: z.string().uuid() }).strict() +export const coordinatorTaskSchema = z.discriminatedUnion('type', [ + z.object({ type: z.literal('readShadowReport'), input: shadowReportRequestSchema }).strict(), + z.object({ type: z.literal('createWaveformDataset'), input: createWaveformDatasetSchema }).strict(), + z.object({ type: z.literal('inspectRawSource'), input: rawScanInputSchema }).strict(), + z.object({ type: z.literal('pageRawSources'), input: rawSourcePageSchema }).strict(), + z.object({ type: z.literal('createDataset'), input: createDatasetSchema }).strict(), + z.object({ type: z.literal('datasetDescriptor'), input: idInput }).strict(), + z.object({ type: z.literal('readResult'), input: idInput }).strict(), + z.object({ type: z.literal('readSummary'), input: idInput }).strict(), + z.object({ type: z.literal('readReferenceTrace'), input: referenceTraceRequestSchema }).strict(), + z.object({ type: z.literal('readNumericTrace'), input: traceRequestSchema }).strict(), + z.object({ type: z.literal('readTrace'), input: traceRequestSchema }).strict(), + z.object({ type: z.literal('releaseDataset'), input: idInput }).strict(), + z.object({ type: z.literal('releaseResult'), input: idInput.extend({ expectedContentHash: z.string().regex(/^[a-f0-9]{64}$/) }) }).strict(), + z.object({ type: z.literal('createExperiment'), input: z.object({ datasetId: z.string().uuid(), protocol: experimentProtocolSchema, idempotencyKey: z.string().min(1).max(200) }).strict() }).strict(), + z.object({ type: z.literal('collectArtifacts') }).strict(), + z.object({ type: z.literal('execute'), owner: z.string().min(1).max(200) }).strict(), +]) +export type CoordinatorTask = z.input +export interface CoordinatorResults { + readShadowReport: ReturnType + createWaveformDataset: Awaited> + inspectRawSource: Awaited> + pageRawSources: ReturnType + createDataset: ReturnType + datasetDescriptor: ReturnType + readResult: ReturnType + readSummary: ReturnType>>> + readReferenceTrace: ReturnType + readNumericTrace: ReturnType + readTrace: ReturnType + releaseDataset: ReturnType + releaseResult: ReturnType + createExperiment: ReturnType + collectArtifacts: null + execute: Awaited> +} + +/** Fixed shipped operations; neither executable code nor paths come from RPC input. */ +export async function performCoordinatorTask(db: BetterSQLite3Database & { $client: Database.Database }, raw: unknown, options: { + workerPath: string + rawLocation?: RawInspectionLocation + replayBudget?: ReplayBudget + signal: AbortSignal + admit: () => Promise + admissionReceipt?: () => AdmissionReceipt | null +}) { + const task = coordinatorTaskSchema.parse(raw) + switch (task.type) { + case 'readShadowReport': return readShadowReport(db.$client, task.input) + case 'createWaveformDataset': return publishSelectedWaveformDataset(db, task.input, options) + case 'inspectRawSource': return inspectRawSource(db, task.input, options) + case 'pageRawSources': return new RawSourceCatalog(db).page(task.input) + case 'createDataset': return new DatasetStore(db).create(task.input) + case 'datasetDescriptor': return new DatasetStore(db).descriptor(task.input.id) + case 'readResult': return new BacktestJobStore(db).result(task.input.id) + case 'readSummary': return summarizeBacktestResult(new BacktestJobStore(db).result(task.input.id)) + case 'readReferenceTrace': return readReferenceTraces(db, task.input) + case 'readNumericTrace': return readNumericTraces(db, task.input) + case 'readTrace': return readStageTraces(db, task.input) + case 'releaseDataset': return new DatasetStore(db).release(task.input.id) + case 'releaseResult': return new BacktestJobStore(db).releaseResult(task.input.id, task.input.expectedContentHash) + case 'createExperiment': return new BacktestJobStore(db).createExperiment(task.input.datasetId, task.input.protocol, task.input.idempotencyKey) + case 'collectArtifacts': + new BacktestJobStore(db).collectOrphanedArtifacts() + await collectRawArtifacts(db, options) + return null + case 'execute': return executeNextBacktest(db, { owner: task.owner, ...options }) + } +} + +/** Recheck resource health during decompression and before committing evidence. */ +async function inspectRawSource(db: BetterSQLite3Database, input: z.input, options: { + signal: AbortSignal + admit: () => Promise + rawLocation?: RawInspectionLocation + replayBudget?: ReplayBudget +}) { + const deferred = { status: 'deferred' as const, nextCursor: input.afterName } + if (options.signal.aborted || !await options.admit()) return deferred + const controller = new AbortController() + const abort = () => controller.abort() + options.signal.addEventListener('abort', abort, { once: true }) + if (options.signal.aborted) abort() + let checking: Promise | undefined, denied = false + const timer = setInterval(() => { + if (checking) return + checking = Promise.resolve().then(() => options.admit()).catch(() => false).then((allowed) => { + if (!allowed) { + denied = true + controller.abort() + } + }).finally(() => { checking = undefined }) + }, 5000) + try { + const result = await runRawInspectionChild(options.rawLocation ?? podRawInspectionLocation, input, { signal: controller.signal }) + clearInterval(timer) + await checking + if (denied || options.signal.aborted || !await options.admit() || options.signal.aborted) return deferred + if (result.status !== 'inspected') return result + const id = new RawSourceCatalog(db).record(result.inspection) + return { status: 'inspected' as const, nextCursor: result.nextCursor, source: new RawSourceCatalog(db).read(id) } + } + catch (error) { + if (denied) return deferred + throw error + } + finally { + clearInterval(timer) + await checking + options.signal.removeEventListener('abort', abort) + } +} + +/** Same work lane as publication; Python additionally locks across processes. */ +async function collectRawArtifacts(db: BetterSQLite3Database, options: { + signal: AbortSignal + admit: () => Promise + rawLocation?: RawInspectionLocation +}) { + if (options.signal.aborted || !await options.admit()) return + const databasePath = db.all<{ name: string, file: string }>(sql`PRAGMA database_list`).find(row => row.name === 'main')?.file + if (!databasePath) return + const controller = new AbortController() + const abort = () => controller.abort() + options.signal.addEventListener('abort', abort, { once: true }) + if (options.signal.aborted) abort() + let checking: Promise | undefined + const timer = setInterval(() => { + if (checking) return + checking = Promise.resolve().then(options.admit).catch(() => false).then((allowed) => { + if (!allowed) controller.abort() + }).finally(() => { checking = undefined }) + }, 5000) + try { + await runRawCollectionChild({ ...(options.rawLocation ?? podRawInspectionLocation), databasePath, + retainedDirectory: join(dirname(databasePath), 'biometric-raw') }, { signal: controller.signal }) + } + finally { + clearInterval(timer) + await checking + options.signal.removeEventListener('abort', abort) + } +} diff --git a/src/biometrics/backtest/coordinator.ts b/src/biometrics/backtest/coordinator.ts new file mode 100644 index 000000000..07f92761c --- /dev/null +++ b/src/biometrics/backtest/coordinator.ts @@ -0,0 +1,123 @@ +import { withBiometricBackgroundSlot } from '../background-admission' +import { coordinatorTaskLanes } from './coordinator-lanes' +import { confirmShadowProcessCleanup } from '../shadow/process-cleanup' +import { monitorProcessGroupMemory } from './process-group-memory' +import { admissionReceiptSchema, type HealthAdmission } from './job-operations' +import { fork } from 'node:child_process' +import { setPriority } from 'node:os' +import { dirname, resolve } from 'node:path' +import { ReferenceInputError } from '../references/time' +import { coordinatorTaskSchema, type CoordinatorTask, type CoordinatorResults } from './coordinator-tasks' + +import { replayBudgetSchema, type ReplayBudget } from './execution-budget' +import type { RawInspectionLocation } from './raw-inspection-child' + +export interface CoordinatorOptions { + databasePath: string + coordinatorPath: string + workerPath: string + rawLocation?: RawInspectionLocation + modulePath?: string + timeoutMs?: number + replayBudget?: ReplayBudget +} + +/** Own an OS process group so crashes/timeouts cannot orphan the evaluator. */ +export function runCoordinatorTask(options: CoordinatorOptions, task: T, control: { + signal?: AbortSignal + admit?: () => Promise + healthAdmission?: HealthAdmission +} = {}): Promise { + const parsed = coordinatorTaskSchema.parse(task) + if (Buffer.byteLength(JSON.stringify(parsed)) > 65536) return Promise.reject(new ReferenceInputError('RESOURCE_LIMIT', 'Coordinator input exceeds 64 KiB')) + const replayBudget = options.replayBudget === undefined ? undefined : replayBudgetSchema.parse(options.replayBudget) + const timeoutMs = options.timeoutMs ?? (parsed.type === 'execute' ? replayBudget?.jobWallMs : undefined) ?? 150000 + const maximumTimeout = parsed.type === 'execute' && replayBudget ? replayBudget.jobWallMs : 180000 + if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 10 || timeoutMs > maximumTimeout) throw new Error('Invalid coordinator timeout') + if (control.signal?.aborted) return Promise.reject(new Error('Coordinator cancelled')) + return withBiometricBackgroundSlot(coordinatorTaskLanes[parsed.type], () => new Promise((resolveResult, reject) => { + const child = fork(resolve(options.coordinatorPath), [resolve(options.databasePath), resolve(options.workerPath), JSON.stringify(options.rawLocation ?? null), JSON.stringify(parsed.type === 'execute' ? replayBudget ?? null : null)], { + cwd: dirname(resolve(options.databasePath)), detached: process.platform !== 'win32', serialization: 'advanced', + execArgv: ['--max-old-space-size=384'], silent: true, + env: { NODE_ENV: 'production', LANG: 'C.UTF-8', TZ: 'UTC', NODE_PATH: options.modulePath ?? resolve('node_modules') }, + }) + const memory = parsed.type === 'execute' && child.pid ? monitorProcessGroupMemory(child.pid) : null + let settled = false, received = false, result: CoordinatorResults[T['type']], failure: Error | undefined + let grace: ReturnType | undefined + let cancellationGrace: ReturnType | undefined + let diagnostics = 0, admissionActive = false + const killGroup = (signal: NodeJS.Signals) => { + if (!child.pid) return + try { + if (process.platform === 'win32') child.kill(signal) + else process.kill(-child.pid, signal) + } + catch { /* Already exited. */ } + } + const stop = (error: Error) => { + failure ??= error + killGroup('SIGTERM') + grace ??= setTimeout(() => killGroup('SIGKILL'), 500) + } + const timer = setTimeout(() => stop(new Error('Coordinator time budget exceeded')), timeoutMs) + const cancel = () => { + failure ??= new Error('Coordinator cancelled') + if (child.connected) child.send({ type: 'cancel' }, () => {}) + cancellationGrace ??= setTimeout(() => stop(new Error('Coordinator cancellation grace expired')), 2000) + } + control.signal?.addEventListener('abort', cancel, { once: true }) + if (control.signal?.aborted) cancel() + const count = (chunk: Buffer) => { + diagnostics += chunk.length + if (diagnostics > 16384) stop(new Error('Coordinator diagnostics exceeded byte limit')) + } + child.stdout?.on('data', count) + child.stderr?.on('data', count) + child.on('error', stop) + child.on('message', (message: unknown) => { + const data = message as { type?: string, sequence?: number, result?: CoordinatorResults[T['type']], code?: string, message?: string } + if (data?.type === 'admission' && (parsed.type === 'execute' || parsed.type === 'inspectRawSource' || parsed.type === 'createWaveformDataset' || parsed.type === 'collectArtifacts') && !received && !admissionActive && Number.isSafeInteger(data.sequence)) { + admissionActive = true + Promise.resolve().then(async () => control.healthAdmission + ? admissionReceiptSchema.parse(await control.healthAdmission()) + : { allowed: await control.admit?.() ?? false, sample: null }).catch(() => ({ allowed: false, sample: null })).then(async (receipt) => { + await memory?.sample() + if (child.connected && !failure) child.send({ type: 'admission', sequence: data.sequence, ...receipt, ...(memory ? { memory: memory.snapshot() } : {}) }, () => {}) + }).finally(() => { admissionActive = false }) + } + else if (data?.type === 'result' && !received) { + received = true + result = data.result as CoordinatorResults[T['type']] + } + else if (data?.type === 'error' && typeof data.code === 'string' && typeof data.message === 'string') { + stop(new ReferenceInputError(data.code, data.message)) + } + else stop(new Error('Invalid coordinator response')) + }) + child.on('close', async (code) => { + if (settled) return + settled = true + void memory?.stop() + clearTimeout(timer) + if (grace) clearTimeout(grace) + if (cancellationGrace) clearTimeout(cancellationGrace) + control.signal?.removeEventListener('abort', cancel) + // Also kill descendants if the coordinator crashed before reporting them. + killGroup('SIGKILL') + if (process.platform !== 'win32' && !await confirmShadowProcessCleanup(child.pid, () => true)) { + reject(Object.assign(new Error('Biometrics coordinator process-group cleanup unconfirmed', { cause: failure }), { code: 'BIOMETRIC_COORDINATOR_CLEANUP_UNCONFIRMED' })) + return + } + if (failure) reject(failure) + else if (!received || code !== 0) reject(new Error('Coordinator exited without a complete result')) + else resolveResult(result) + }) + try { + if (child.pid) setPriority(child.pid, 19) + child.send({ type: 'run', task: parsed }, (error) => { + if (error) stop(error) + }) + } + catch { stop(new Error('Could not start coordinator')) } + })) +} diff --git a/src/biometrics/backtest/coverage.ts b/src/biometrics/backtest/coverage.ts new file mode 100644 index 000000000..988c3625b --- /dev/null +++ b/src/biometrics/backtest/coverage.ts @@ -0,0 +1,18 @@ +/** No interpolation: expose actual timestamps and gaps under an explicit cadence. */ +export function describeCoverage(timestamps: number[], startMs: number, endMs: number, expectedCadenceMs: number) { + if (![startMs, endMs, expectedCadenceMs].every(Number.isSafeInteger) || endMs <= startMs || expectedCadenceMs <= 0 + || timestamps.length > 100000 || timestamps.some(time => !Number.isSafeInteger(time))) throw new Error('Invalid bounded coverage input') + const times = [...new Set(timestamps.filter(time => time >= startMs && time < endMs))].sort((a, b) => a - b) + const gaps: Array<{ startMs: number, endMs: number }> = [] + let cursor = startMs + for (const time of times) { + if (time > cursor) gaps.push({ startMs: cursor, endMs: time }) + cursor = Math.min(endMs, time + expectedCadenceMs) + } + if (cursor < endMs) gaps.push({ startMs: cursor, endMs }) + return { count: times.length, duplicateTimestamps: timestamps.filter(time => time >= startMs && time < endMs).length - times.length, + firstMs: times[0] ?? null, lastMs: times.at(-1) ?? null, + expectedCadenceMs, gaps, gapDurationMs: gaps.reduce((sum, gap) => sum + gap.endMs - gap.startMs, 0), + // Cadence-based availability is not sensor support duration. + interpretation: 'cadence_availability_not_measured_support' as const } +} diff --git a/src/biometrics/backtest/datasets.ts b/src/biometrics/backtest/datasets.ts new file mode 100644 index 000000000..b86ce0d2c --- /dev/null +++ b/src/biometrics/backtest/datasets.ts @@ -0,0 +1,194 @@ +import { randomUUID } from 'node:crypto' +import { and, desc, eq, gte, inArray, isNull, lt, or, sql } from 'drizzle-orm' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import type { z } from 'zod' +import * as schema from '@/src/db/biometrics-schema' +import { configHash, dataHash } from '../algorithms/canonical' +import { normalizedMeasurementSchema } from '../references/contracts' +import { ReferenceStore } from '../references/store' +import { ReferenceInputError } from '../references/time' +import { assignWholeNightSplits } from './splits' +import { describeCoverage } from './coverage' + +import { createDatasetSchema } from './contracts' +export { createDatasetSchema } from './contracts' + +export { datasetSnapshotSchema } from './contracts' +import { datasetSnapshotSchema } from './contracts' + +const LIMITS = { rows: 20000, datasetBytes: 8 * 1024 * 1024, totalBytes: 64 * 1024 * 1024 } + +/** Creates processed snapshots; reads and releases both retained dataset formats. */ +export class DatasetStore { + constructor(private readonly db: BetterSQLite3Database, private readonly now = Date.now) {} + + create(raw: z.input) { + const input = createDatasetSchema.parse(raw) + if (new Set(input.sessionIds).size !== input.sessionIds.length || new Set(input.referenceImportIds).size !== input.referenceImportIds.length) throw new ReferenceInputError('DUPLICATE_SELECTION', 'Select each session and import once') + input.sessionIds.sort((a, b) => a - b) + input.referenceImportIds.sort() + const requestHash = configHash(input) + const existing = this.db.select().from(schema.biometricDatasets).where(eq(schema.biometricDatasets.idempotencyKey, input.idempotencyKey)).get() + if (existing) { + if (existing.requestHash !== requestHash) throw new ReferenceInputError('REVISION_CONFLICT', 'Dataset idempotency key was already used for a different selection') + return this.describe(existing) + } + const captured = this.db.transaction(() => { + const sessions = this.db.select().from(schema.sleepRecords).where(and(eq(schema.sleepRecords.side, input.side), inArray(schema.sleepRecords.id, input.sessionIds))) + .orderBy(schema.sleepRecords.enteredBedAt, schema.sleepRecords.id).all() + if (sessions.length !== input.sessionIds.length) throw new ReferenceInputError('NOT_FOUND', 'A selected session is missing or belongs to the other side') + const splits = assignWholeNightSplits(sessions.map(row => ({ id: row.id, startMs: row.enteredBedAt.getTime(), endMs: row.leftBedAt.getTime() })), input.timezone) + let remaining = LIMITS.rows + const snapshotSessions = sessions.map((session) => { + const vitals = this.db.select().from(schema.vitals).where(and(eq(schema.vitals.side, input.side), + gte(schema.vitals.timestamp, session.enteredBedAt), lt(schema.vitals.timestamp, session.leftBedAt))) + .orderBy(schema.vitals.timestamp).limit(remaining + 1).all() + remaining -= vitals.length + if (remaining < 0) throw new ReferenceInputError('RESOURCE_LIMIT', 'Select fewer sessions; processed snapshot row budget exceeded') + const movement = this.db.select().from(schema.movement).where(and(eq(schema.movement.side, input.side), + gte(schema.movement.timestamp, session.enteredBedAt), lt(schema.movement.timestamp, session.leftBedAt))) + .orderBy(schema.movement.timestamp, schema.movement.id).limit(remaining + 1).all() + remaining -= movement.length + if (remaining < 0) throw new ReferenceInputError('RESOURCE_LIMIT', 'Select fewer sessions; processed snapshot row budget exceeded') + return { id: session.id, startMs: session.enteredBedAt.getTime(), endMs: session.leftBedAt.getTime(), + sleepDurationSeconds: session.sleepDurationSeconds, timesExitedBed: session.timesExitedBed, + presentIntervals: session.presentIntervals, notPresentIntervals: session.notPresentIntervals, + vitals: vitals.map(row => ({ id: row.id, timestampMs: row.timestamp.getTime(), heartRate: row.heartRate, hrv: row.hrv, breathingRate: row.breathingRate, processingRunId: row.processingRunId })), + movement: movement.map(row => ({ id: row.id, timestampMs: row.timestamp.getTime(), totalMovement: row.totalMovement })), + } + }) + const store = new ReferenceStore(this.db) + const families = new Set() + const references = input.referenceImportIds.map((importId) => { + const descriptor = store.get(importId) + if (descriptor.summary.side !== input.side || descriptor.summary.subjectId !== input.subjectId) throw new ReferenceInputError('SUBJECT_MISMATCH', 'All references must match the selected side and sleeper') + if (families.has(descriptor.familyId)) throw new ReferenceInputError('REVISION_CONFLICT', 'Select only one revision from each reference family') + families.add(descriptor.familyId) + const measurements = [] + let cursor = 0 + do { + const page = store.measurements(importId, cursor) + remaining -= page.items.length + if (remaining < 0) throw new ReferenceInputError('RESOURCE_LIMIT', 'Reference selection exceeds snapshot row budget') + measurements.push(...page.items.map(row => normalizedMeasurementSchema.parse(Object.fromEntries(Object.entries(row).filter(([key]) => key !== 'id'))))) + if (page.nextCursor === null) break + cursor = page.nextCursor + } while (remaining >= 0) + return { importId, revision: descriptor.revision, contentHash: descriptor.summary.contentHash, + source: descriptor.summary.source, timePolicy: descriptor.summary.timePolicy, measurements } + }) + const runIds = [...new Set(snapshotSessions.flatMap(session => session.vitals.flatMap(row => row.processingRunId ? [row.processingRunId] : [])))] + const processingRuns: Array = [] + for (let index = 0; index < runIds.length; index += 100) { + processingRuns.push(...this.db.select().from(schema.biometricProcessingRuns) + .where(and(eq(schema.biometricProcessingRuns.side, input.side), inArray(schema.biometricProcessingRuns.id, runIds.slice(index, index + 100)))).all()) + } + processingRuns.sort((a, b) => a.id.localeCompare(b.id)) + return { snapshot: datasetSnapshotSchema.parse({ version: 'processed-dataset-v1', side: input.side, subjectId: input.subjectId, + sessions: snapshotSessions, references, processingRuns }), splits } + }) + // No write lock is held while computing hashes and coverage. + const snapshotBytes = Buffer.byteLength(JSON.stringify(captured.snapshot)) + if (snapshotBytes > LIMITS.datasetBytes) throw new ReferenceInputError('RESOURCE_LIMIT', 'Processed dataset exceeds the 8 MiB snapshot limit') + const manifest = { version: 'processed-dataset-manifest-v1', splitProtocol: captured.splits, + coverage: captured.snapshot.sessions.map(session => ({ sessionId: session.id, + vitals: describeCoverage(session.vitals.map(row => row.timestampMs), session.startMs, session.endMs, 60000), + movement: describeCoverage(session.movement.map(row => row.timestampMs), session.startMs, session.endMs, 60000), + })), + capabilities: { comparison: true, stageReplay: captured.snapshot.sessions.some(session => session.vitals.length > 0), signalReplay: false }, + limitations: ['Raw channels, pump context and measured calibration bytes are not retained in this processed dataset; signal replay is unavailable.', + 'Historical stage replay must declare its calibration quality assumption when original calibration is unavailable.', + 'Reference deletion does not erase this separately retained dataset.', + 'Session side is known; historical occupant identity is asserted by the dataset creator, not measured by the pod.'], + } + const byteLength = snapshotBytes + Buffer.byteLength(JSON.stringify(manifest)) + if (byteLength > LIMITS.datasetBytes) throw new ReferenceInputError('RESOURCE_LIMIT', 'Dataset and manifest exceed the 8 MiB snapshot limit') + const contentHash = dataHash({ manifest, snapshot: captured.snapshot }) + return this.db.transaction((tx) => { + const raced = tx.select().from(schema.biometricDatasets).where(eq(schema.biometricDatasets.idempotencyKey, input.idempotencyKey)).get() + if (raced) { + if (raced.requestHash !== requestHash) throw new ReferenceInputError('REVISION_CONFLICT', 'Dataset idempotency key changed') + return this.describe(raced) + } + const total = tx.select({ bytes: sql`coalesce(sum(${schema.biometricDatasets.byteLength}), 0)` }).from(schema.biometricDatasets).get()?.bytes ?? 0 + if (total + byteLength > LIMITS.totalBytes) throw new ReferenceInputError('RESOURCE_LIMIT', 'Processed datasets have reached their 64 MiB retained storage budget') + const row = tx.insert(schema.biometricDatasets).values({ id: randomUUID(), side: input.side, subjectId: input.subjectId, + idempotencyKey: input.idempotencyKey, requestHash, contentHash, manifest, snapshot: captured.snapshot, byteLength, createdAtMs: this.now() }).returning().get() + return this.describe(row) + }, { behavior: 'immediate' }) + } + + private describe(row: typeof schema.biometricDatasets.$inferSelect) { + return { id: row.id, side: row.side, subjectId: row.subjectId, contentHash: row.contentHash, + manifest: row.manifest, byteLength: row.byteLength, createdAtMs: row.createdAtMs, releasedAtMs: row.releasedAtMs, available: row.releasedAtMs === null } + } + + read(id: string) { + const row = this.db.select().from(schema.biometricDatasets).where(eq(schema.biometricDatasets.id, id)).get() + if (!row) throw new ReferenceInputError('NOT_FOUND', 'Dataset is unavailable') + if (row.releasedAtMs !== null) throw new ReferenceInputError('DATASET_RELEASED', 'Retained dataset inputs were explicitly released') + const snapshot = datasetSnapshotSchema.parse(row.snapshot) + if (dataHash({ manifest: row.manifest, snapshot }) !== row.contentHash) throw new ReferenceInputError('DATASET_CORRUPT', 'Retained dataset integrity check failed') + return { ...this.describe(row), snapshot } + } + + release(id: string) { + return this.db.transaction((tx) => { + const dataset = tx.select().from(schema.biometricDatasets).where(eq(schema.biometricDatasets.id, id)).get() + if (!dataset) throw new ReferenceInputError('NOT_FOUND', 'Dataset not found') + if (dataset.releasedAtMs !== null) return this.describe(dataset) + const active = tx.select({ id: schema.biometricBacktestJobs.id }).from(schema.biometricBacktestJobs) + .innerJoin(schema.biometricExperiments, eq(schema.biometricBacktestJobs.experimentId, schema.biometricExperiments.id)) + .where(and(eq(schema.biometricExperiments.datasetId, id), inArray(schema.biometricBacktestJobs.state, ['queued', 'running', 'cancel_requested', 'interrupted']))).get() + if (active) throw new ReferenceInputError('REVISION_CONFLICT', 'Cancel or finish jobs using this dataset before releasing its inputs') + const releasedAtMs = this.now() + const references = schema.biometricDatasetRawArtifacts + const artifacts = schema.biometricRawArtifacts + tx.update(references).set({ releasedAtMs }) + .where(and(eq(references.datasetId, id), isNull(references.releasedAtMs))).run() + // Mark storage eligible for collection only after its last live reference + // is released. The file collector separately serializes with publication. + tx.update(artifacts).set({ releasedAtMs }).where(and(isNull(artifacts.releasedAtMs), + inArray(artifacts.contentHash, tx.select({ hash: references.artifactHash }).from(references).where(eq(references.datasetId, id))), + sql`NOT EXISTS (SELECT 1 FROM biometric_dataset_raw_artifacts AS live + WHERE live.artifact_hash = ${artifacts.contentHash} AND live.released_at_ms IS NULL)`, + )).run() + const tombstone = { released: true } + const byteLength = Buffer.byteLength(JSON.stringify(dataset.manifest)) + Buffer.byteLength(JSON.stringify(tombstone)) + const released = tx.update(schema.biometricDatasets).set({ snapshot: tombstone, byteLength, releasedAtMs }) + .where(eq(schema.biometricDatasets.id, id)).returning().get() + return this.describe(released) + }, { behavior: 'immediate' }) + } + + descriptor(id: string) { + const row = this.db.select().from(schema.biometricDatasets).where(eq(schema.biometricDatasets.id, id)).get() + if (!row) throw new ReferenceInputError('NOT_FOUND', 'Dataset not found') + if (row.releasedAtMs === null) { + const snapshot = datasetSnapshotSchema.parse(row.snapshot) + if (dataHash({ manifest: row.manifest, snapshot }) !== row.contentHash) throw new ReferenceInputError('DATASET_CORRUPT', 'Retained dataset integrity check failed') + } + return this.describe(row) + } + + page(side: 'left' | 'right', cursor?: { createdAtMs: number, id: string }) { + const rows = this.db.select({ id: schema.biometricDatasets.id, side: schema.biometricDatasets.side, + subjectId: schema.biometricDatasets.subjectId, contentHash: schema.biometricDatasets.contentHash, + manifest: schema.biometricDatasets.manifest, byteLength: schema.biometricDatasets.byteLength, createdAtMs: schema.biometricDatasets.createdAtMs, releasedAtMs: schema.biometricDatasets.releasedAtMs, + }).from(schema.biometricDatasets).where(and(eq(schema.biometricDatasets.side, side), cursor + ? or(lt(schema.biometricDatasets.createdAtMs, cursor.createdAtMs), and(eq(schema.biometricDatasets.createdAtMs, cursor.createdAtMs), lt(schema.biometricDatasets.id, cursor.id))) + : undefined)) + .orderBy(desc(schema.biometricDatasets.createdAtMs), desc(schema.biometricDatasets.id)).limit(51).all() + const items = rows.slice(0, 50).map(row => ({ ...row, available: row.releasedAtMs === null })) + const last = items.at(-1) + return { items, nextCursor: rows.length > 50 && last ? { createdAtMs: last.createdAtMs, id: last.id } : null } + } + + list(side: 'left' | 'right') { + return this.db.select({ id: schema.biometricDatasets.id, side: schema.biometricDatasets.side, + subjectId: schema.biometricDatasets.subjectId, contentHash: schema.biometricDatasets.contentHash, + manifest: schema.biometricDatasets.manifest, byteLength: schema.biometricDatasets.byteLength, createdAtMs: schema.biometricDatasets.createdAtMs, releasedAtMs: schema.biometricDatasets.releasedAtMs, + }).from(schema.biometricDatasets).where(eq(schema.biometricDatasets.side, side)) + .orderBy(desc(schema.biometricDatasets.createdAtMs), desc(schema.biometricDatasets.id)).limit(100).all() + } +} diff --git a/src/biometrics/backtest/descriptors.ts b/src/biometrics/backtest/descriptors.ts new file mode 100644 index 000000000..7cf6dd0b7 --- /dev/null +++ b/src/biometrics/backtest/descriptors.ts @@ -0,0 +1,23 @@ +import { z } from 'zod' +import type { biometricBacktestJobs } from '@/src/db/biometrics-schema' + +export const jobDescriptorSchema = z.object({ id: z.string().uuid(), experimentId: z.string().uuid(), + state: z.enum(['queued', 'running', 'cancel_requested', 'cancelled', 'succeeded', 'failed', 'interrupted']), attempt: z.number(), + progress: z.object({ phase: z.string(), completedNights: z.number(), totalNights: z.number() }), + cancellationRequestedAtMs: z.number().nullable(), error: z.string().nullable(), createdAtMs: z.number(), updatedAtMs: z.number(), + datasetId: z.string().uuid(), datasetHash: z.string(), evaluatorBuildHash: z.string(), split: z.enum(['tuning', 'validation', 'holdout']), + configurations: z.array(z.object({ label: z.string(), revisionId: z.string().uuid().nullable(), hash: z.string() })), + evaluationScope: z.enum(['original_readings_comparison', 'stage_rules_on_original_readings', 'retained_waveform_replay']), +}) +export function describeJob(row: typeof biometricBacktestJobs.$inferSelect) { + const input = z.object({ datasetId: z.string(), datasetHash: z.string(), evaluatorBuildHash: z.string(), split: z.string(), + protocol: z.object({ mode: z.enum(['comparison', 'stage_replay', 'signal_replay']) }), + configs: z.array(z.object({ label: z.string(), revisionId: z.string().nullable(), resolvedHash: z.string() })), + }).parse(row.input) + return jobDescriptorSchema.parse({ ...row, datasetId: input.datasetId, datasetHash: input.datasetHash, evaluatorBuildHash: input.evaluatorBuildHash, + split: input.split, configurations: input.configs.map(config => ({ label: config.label, revisionId: config.revisionId, hash: config.resolvedHash })), + evaluationScope: input.protocol.mode === 'signal_replay' ? 'retained_waveform_replay' : input.protocol.mode === 'comparison' ? 'original_readings_comparison' : 'stage_rules_on_original_readings' }) +} + +export const resultDescriptorSchema = z.object({ jobId: z.string(), fencingToken: z.number(), report: z.unknown(), + contentHash: z.string(), completedAtMs: z.number(), releasedAtMs: z.number().nullable(), available: z.boolean(), currentHoldoutUseCount: z.number() }) diff --git a/src/biometrics/backtest/discovery.ts b/src/biometrics/backtest/discovery.ts new file mode 100644 index 000000000..7f4f1ffda --- /dev/null +++ b/src/biometrics/backtest/discovery.ts @@ -0,0 +1,60 @@ +import { and, desc, eq, gt, gte, lt, or, sql } from 'drizzle-orm' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import { z } from 'zod' +import * as schema from '@/src/db/biometrics-schema' +import { ReferenceInputError } from '../references/time' +import { coverageSchema } from './contracts' +import { describeCoverage } from './coverage' + +const side = z.enum(['left', 'right']) +const sessionSchema = z.object({ id: z.number(), side, startMs: z.number(), endMs: z.number(), sleepDurationSeconds: z.number(), timesExitedBed: z.number() }) +export const sessionPageSchema = z.object({ items: z.array(sessionSchema), nextCursor: z.object({ startMs: z.number(), id: z.number() }).nullable() }) +export const sessionCapabilitiesSchema = z.object({ + session: sessionSchema, + processed: z.object({ vitals: coverageSchema, movement: coverageSchema, legacyUnknownProvenanceCount: z.number() }), + referenceMetrics: z.array(z.object({ metric: z.string(), count: z.number() })), + comparison: z.boolean(), stageReplay: z.boolean(), signalReplay: z.literal(false), rawWaveforms: z.literal('not_inventoried'), + limitations: z.array(z.string()), +}) +const describeSession = (row: typeof schema.sleepRecords.$inferSelect) => ({ id: row.id, side: row.side, startMs: row.enteredBedAt.getTime(), + endMs: row.leftBedAt.getTime(), sleepDurationSeconds: row.sleepDurationSeconds, timesExitedBed: row.timesExitedBed }) + +export function listReplaySessions(db: BetterSQLite3Database, selectedSide: 'left' | 'right', cursor?: { startMs: number, id: number }) { + const records = db.select().from(schema.sleepRecords).where(and(eq(schema.sleepRecords.side, selectedSide), cursor + ? or(lt(schema.sleepRecords.enteredBedAt, new Date(cursor.startMs)), and(eq(schema.sleepRecords.enteredBedAt, new Date(cursor.startMs)), lt(schema.sleepRecords.id, cursor.id))) + : undefined)) + .orderBy(desc(schema.sleepRecords.enteredBedAt), desc(schema.sleepRecords.id)).limit(21).all() + const items = records.slice(0, 20).map(describeSession) + const last = items.at(-1) + return { items, nextCursor: records.length > 20 && last ? { startMs: last.startMs, id: last.id } : null } +} + +export function sessionCapabilities(db: BetterSQLite3Database, selectedSide: 'left' | 'right', sessionId: number) { + return db.transaction(() => { + const record = db.select().from(schema.sleepRecords).where(and(eq(schema.sleepRecords.side, selectedSide), eq(schema.sleepRecords.id, sessionId))).get() + if (!record) throw new ReferenceInputError('NOT_FOUND', 'Session not found for this side') + const session = describeSession(record) + if (session.endMs <= session.startMs || session.endMs - session.startMs > 36 * 3600000) throw new ReferenceInputError('RESOURCE_LIMIT', 'Session boundaries exceed the inspection budget') + const vitals = db.select({ timestamp: schema.vitals.timestamp, run: schema.vitals.processingRunId }).from(schema.vitals).where(and(eq(schema.vitals.side, selectedSide), + gte(schema.vitals.timestamp, record.enteredBedAt), lt(schema.vitals.timestamp, record.leftBedAt))).orderBy(schema.vitals.timestamp).limit(20001).all() + const movement = db.select({ timestamp: schema.movement.timestamp }).from(schema.movement).where(and(eq(schema.movement.side, selectedSide), + gte(schema.movement.timestamp, record.enteredBedAt), lt(schema.movement.timestamp, record.leftBedAt))).orderBy(schema.movement.timestamp).limit(20001).all() + if (vitals.length + movement.length > 20000) throw new ReferenceInputError('RESOURCE_LIMIT', 'Session exceeds the retained processed-data row budget') + const referenceMetrics = db.select({ metric: schema.referenceMeasurements.metric, count: sql`count(*)` }).from(schema.referenceMeasurements) + .where(and(eq(schema.referenceMeasurements.side, selectedSide), lt(schema.referenceMeasurements.startMs, session.endMs), + or(gt(schema.referenceMeasurements.endMs, session.startMs), and(eq(schema.referenceMeasurements.temporalKind, 'point'), gte(schema.referenceMeasurements.startMs, session.startMs))))) + .groupBy(schema.referenceMeasurements.metric).all() + return { session, + processed: { vitals: describeCoverage(vitals.map(row => row.timestamp.getTime()), session.startMs, session.endMs, 60000), + movement: describeCoverage(movement.map(row => row.timestamp.getTime()), session.startMs, session.endMs, 60000), + legacyUnknownProvenanceCount: vitals.filter(row => row.run === null).length }, + referenceMetrics, comparison: vitals.length > 0, stageReplay: vitals.length > 0 && session.endMs - session.startMs <= 86400000, + signalReplay: false as const, rawWaveforms: 'not_inventoried' as const, + limitations: ['Reference counts include available revisions; select one revision per source family when freezing inputs.', + 'Raw archives and pump/occupancy context have not been retained by this processed-data service.', + 'Movement and vitals availability use nominal one-minute cadence, not inferred continuous signal support.', + ...(session.endMs - session.startMs > 86400000 ? ['Stage replay currently requires a merged window of at most one day.'] : []), + ...(!vitals.length ? ['No stored vitals are available for comparison or stage replay.'] : [])], + } + }) +} diff --git a/src/biometrics/backtest/evaluator-core.ts b/src/biometrics/backtest/evaluator-core.ts new file mode 100644 index 000000000..1db9c3fde --- /dev/null +++ b/src/biometrics/backtest/evaluator-core.ts @@ -0,0 +1,245 @@ +import { assessStageObjective } from './stage-objective' +import { numericSubgroupNames, subgroupReferences, subgroupObjectiveEvidence, subgroupBoundaryExclusions } from './numeric-subgroups' +import { summarizeWaveformResources } from './waveform-resources' +import { assessNumericObjective } from './objective-policy' +import { createNumericNightSummary } from './numeric-night-summary' +import { compareNumericConfigurations } from './numeric-comparison' +import { z } from 'zod' +import { waveformEvaluationSchema } from './waveform-output' +import { compareStageOutputs } from './stage-comparison' +import { createNightSummary } from './night-summary-core' +import { createNightUncertainty } from './uncertainty-core' +import { intersectOutputSupport } from './common-support' +import { classifySleepStages, type SleepStageBlock } from '@/src/lib/sleep-stages' +import { biometricManifest, validateConfig } from '../algorithms/registry-core' +import { alignReferences, DEFAULT_ALIGNMENT } from '../references/alignment' +import { scoreStageDurations } from '../references/stage-scoring' +import { datasetSnapshotSchema, backtestJobSnapshotSchema, numericObjectivePolicies } from './contracts' + +export { waveformEvaluationSchema } from './waveform-output' + +export const backtestWorkSchema = z.object({ + job: backtestJobSnapshotSchema, + dataset: z.object({ manifest: z.unknown(), snapshot: datasetSnapshotSchema }), + waveform: waveformEvaluationSchema.optional(), +}) +const assignmentsSchema = z.object({ splitProtocol: z.object({ assignments: z.array(z.object({ + cohortId: z.string(), sessionIds: z.array(z.number()), split: z.enum(['tuning', 'validation', 'holdout']), startMs: z.number(), endMs: z.number(), +})) }) }) + +function mergeWindows(windows: Array<{ startMs: number, endMs: number }>) { + const merged: typeof windows = [] + for (const window of [...windows].sort((a, b) => a.startMs - b.startMs)) { + const last = merged.at(-1) + if (last && last.endMs >= window.startMs) last.endMs = Math.max(last.endMs, window.endMs) + else merged.push({ ...window }) + } + return merged +} + +/** Pure retained-data evaluator. No DB, network, filesystem or hardware writes. */ +export function createBacktestEvaluator(dependencies: { configHash: (value: unknown) => string, dataHash: (value: unknown) => string, digest: (text: string) => Uint8Array }) { + const { configHash, dataHash, digest } = dependencies + const manifestHash = configHash(biometricManifest) + const summarizeNumericNights = createNumericNightSummary(createNightUncertainty(digest, 'equal_night_mean_common_mae_difference')) + const summarizeStageNights = createNightSummary(createNightUncertainty(digest)) + return function evaluateBacktest(raw: z.input, progress: (completed: number, total: number) => void = () => {}) { + const { job, dataset, waveform } = backtestWorkSchema.parse(raw) + const signalReplay = job.protocol.mode === 'signal_replay' + const stages = job.protocol.mode === 'stage_replay' || (signalReplay && job.protocol.waveformStages === true) + if (dataHash(dataset) !== job.datasetHash || configHash(job.protocol) !== job.protocolHash) throw new Error('Evaluation input integrity mismatch') + const assignments = assignmentsSchema.parse(dataset.manifest).splitProtocol.assignments.filter(row => row.split === job.split) + if (!assignments.length) throw new Error('No nights in selected split') + const configs = job.configs.map((config) => { + const resolved = validateConfig(config.resolved) + if (config.manifestHash !== manifestHash || configHash(resolved) !== config.resolvedHash) throw new Error('Evaluation engine/configuration mismatch') + return { ...config, resolved } + }) + const waveformOutputs = new Map['outputs'][number]>() + if (signalReplay) { + if (!job.waveform || !waveform || dataset.snapshot.version !== 'waveform-dataset-v1' + || dataHash(waveform.outputs) !== waveform.hash) throw new Error('Missing or corrupt waveform evaluation inputs') + const other = validateConfig(job.waveform.otherSide.resolved) + if (job.waveform.otherSide.manifestHash !== manifestHash || configHash(other) !== job.waveform.otherSide.resolvedHash) throw new Error('Invalid other-side waveform configuration') + const expectedConfigs = new Set(configs.map(config => config.resolvedHash)) + const otherSide = dataset.snapshot.side === 'left' ? 'right' : 'left' + for (const output of waveform.outputs) { + const retained = dataset.snapshot.waveformInputs.find(row => row.cohortId === output.cohortId) + const key = `${output.cohortId}:${output.configHash}` + if (!retained || !assignments.some(row => row.cohortId === output.cohortId) || !expectedConfigs.has(output.configHash) + || waveformOutputs.has(key) || output.run.build_hash !== job.waveform.pythonBuildHash + || output.run.config_hashes[dataset.snapshot.side] !== output.configHash + || output.run.config_hashes[otherSide] !== job.waveform.otherSide.resolvedHash + || output.run.counts.scored_outputs !== output.readings.length + || output.run.raw_records < output.run.decoded_records || output.run.decoded_records < output.run.counts.records + || JSON.stringify(output.run.artifact_hashes) !== JSON.stringify(retained.input.artifacts.map(row => row.artifact.sha256)) + || output.readings.some(row => row.configHash !== output.run.config_hashes[row.side] + || row.timestampMs < retained.input.score_start_ms || row.timestampMs >= retained.input.score_end_ms)) throw new Error('Waveform output identity mismatch') + waveformOutputs.set(key, output) + } + if (waveformOutputs.size !== assignments.length * expectedConfigs.size) throw new Error('Missing waveform cohort/configuration output') + } + else if (waveform || job.waveform) throw new Error('Unexpected waveform inputs for processed replay') + const references = dataset.snapshot.references.flatMap(source => source.measurements.map(row => ({ ...row, recordKey: `${source.importId}:${row.recordKey}` }))) + const nights = assignments.map((assignment, nightIndex) => { + const sessions = assignment.sessionIds.map((id) => { + const session = dataset.snapshot.sessions.find(row => row.id === id) + if (!session) throw new Error('Retained session missing') + return session + }) + const windows = mergeWindows(sessions.map(row => ({ startMs: row.startMs, endMs: row.endMs }))) + const vitals = [...new Map(sessions.flatMap(session => session.vitals).map(row => [row.id, row])).values()].sort((a, b) => a.timestampMs - b.timestampMs) + const movement = [...new Map(sessions.flatMap(session => session.movement).map(row => [row.id, row])).values()].sort((a, b) => a.timestampMs - b.timestampMs || a.id - b.id) + // Each unique configuration executes once even when default/current match. + const computed = new Map>() + const evaluateConfig = (config: typeof configs[number]) => windows.map((window) => { + if (window.endMs - window.startMs > 86400000) throw new Error('Merged replay window exceeds the one-day evaluation budget') + const output = waveformOutputs.get(`${assignment.cohortId}:${config.resolvedHash}`) + const observations = signalReplay + ? (output?.readings.filter(row => row.side === dataset.snapshot.side).map((row, id) => ({ + id, timestampMs: row.timestampMs, heartRate: row.heartRate, hrv: row.hrvIndex, breathingRate: row.breathingRate, processingRunId: null, + })) ?? []) + : vitals + const rows = observations.filter(row => row.timestampMs >= window.startMs && row.timestampMs < window.endMs) + const moves = movement.filter(row => row.timestampMs >= window.startMs && row.timestampMs < window.endMs) + const numeric = (['heart_rate', 'breathing_rate', 'window_ibi_variability_v1'] as const).map((metric) => { + const aligned = alignReferences(metric, metric === 'window_ibi_variability_v1' ? 300000 : null, + rows.map(row => ({ timestampMs: row.timestampMs, value: metric === 'heart_rate' ? row.heartRate : metric === 'breathing_rate' ? row.breathingRate : row.hrv })), + references.filter(row => row.metric === metric && row.startMs >= window.startMs && (row.temporalKind === 'point' ? row.startMs < window.endMs : row.endMs <= window.endMs)), + { ...DEFAULT_ALIGNMENT, pointToleranceMs: job.protocol.pointToleranceMs, clockOffsetMs: job.protocol.clockOffsetMs }) + // The report stores summaries; full input samples remain in the dataset. + return { metric, status: aligned.status, metrics: aligned.metrics, pairedCount: aligned.pairedCount, + referenceCoverage: aligned.referenceCoverage, outputCoverage: aligned.outputCoverage, unmatchedReferences: aligned.unmatchedReferences, + unmatchedObservations: aligned.unmatchedObservations, exclusions: aligned.exclusions } + }) + let blocks: SleepStageBlock[] = [] + if (stages) { + const epochs = classifySleepStages(rows.map(row => ({ ...row, timestamp: new Date(row.timestampMs) })), + moves.map(row => ({ ...row, timestamp: new Date(row.timestampMs) })), job.protocol.assumedStageCalibrationQuality, config.resolved.sleep_stages.ruleset) + blocks = epochs.map(epoch => ({ start: epoch.start, end: Math.min(window.endMs, epoch.start + epoch.duration), stage: epoch.stage })).filter(row => row.end > row.start) + } + return { ...window, observationCount: rows.length, numeric, + stage: stages + ? scoreStageDurations(blocks.map(block => ({ ...block, start: block.start + job.protocol.clockOffsetMs, end: block.end + job.protocol.clockOffsetMs })), references.filter(row => row.metric === 'sleep_stages'), window.startMs, window.endMs) + : null, + // Bounded complete traces (the dataset has at most 20k rows). + blocks, + } + }) + const evaluations = configs.map((config) => { + let windows = computed.get(config.resolvedHash) + if (!windows) { + windows = evaluateConfig(config) + computed.set(config.resolvedHash, windows) + } + return { label: config.label, revisionId: config.revisionId, configHash: config.resolvedHash, windows } + }) + const shift = (blocks: SleepStageBlock[]) => blocks.map(block => ({ ...block, + start: block.start + job.protocol.clockOffsetMs, end: block.end + job.protocol.clockOffsetMs })) + const sharedSupport = stages + ? windows.map((window, index) => + intersectOutputSupport(evaluations.map(evaluation => shift(evaluation.windows[index].blocks)), window.startMs, window.endMs)) + : [] + const comparisons = stages + ? evaluations.flatMap(candidate => evaluations + .filter(baseline => (candidate.label === 'candidate' && baseline.label !== 'candidate') || (candidate.label === 'current' && baseline.label === 'default')) + .map(baseline => ({ baseline: baseline.label, candidate: candidate.label, + baselineHash: baseline.configHash, candidateHash: candidate.configHash, + windows: candidate.windows.map((window, index) => { + return { startMs: window.startMs, endMs: window.endMs, + ...compareStageOutputs(shift(baseline.windows[index].blocks), shift(window.blocks), + references.filter(row => row.metric === 'sleep_stages'), window.startMs, window.endMs, sharedSupport[index]) } + }), + }))) + : [] + const compareWindow = (window: typeof windows[number], metric: 'heart_rate' | 'breathing_rate' | 'window_ibi_variability_v1', selectedReferences: typeof references) => compareNumericConfigurations( + metric, metric === 'window_ibi_variability_v1' ? 300000 : null, + configs.map(config => ({ label: config.label, configHash: config.resolvedHash, + observations: (waveformOutputs.get(`${assignment.cohortId}:${config.resolvedHash}`)?.readings ?? []) + .filter(row => row.side === dataset.snapshot.side && row.timestampMs >= window.startMs && row.timestampMs < window.endMs) + .map(row => ({ timestampMs: row.timestampMs, value: metric === 'heart_rate' ? row.heartRate : metric === 'breathing_rate' ? row.breathingRate : row.hrvIndex })), + })), selectedReferences, + { ...DEFAULT_ALIGNMENT, pointToleranceMs: job.protocol.pointToleranceMs, clockOffsetMs: job.protocol.clockOffsetMs }) + const windowReferences = (window: typeof windows[number], metric: string) => references.filter(row => row.metric === metric && row.startMs >= window.startMs + && (row.temporalKind === 'point' ? row.startMs < window.endMs : row.endMs <= window.endMs)) + const numericComparisons = signalReplay + ? windows.map(window => ({ ...window, + metrics: (['heart_rate', 'breathing_rate', 'window_ibi_variability_v1'] as const).flatMap(metric => compareWindow(window, metric, windowReferences(window, metric))), + })) + : [] + const subgroupMetrics = numericObjectivePolicies(job.protocol).map(policy => policy.metric) + const numericSubgroups = signalReplay && subgroupMetrics.length + ? numericSubgroupNames.map(name => ({ name, numericComparisons: windows.map(window => ({ ...window, + metrics: subgroupMetrics.flatMap(metric => compareWindow(window, metric, subgroupReferences(name, windowReferences(window, metric), window))), + excludedBoundaryReferenceCount: subgroupMetrics.reduce((sum, metric) => sum + subgroupBoundaryExclusions(name, windowReferences(window, metric), window), 0), + excludedByMetric: Object.fromEntries(subgroupMetrics.map(metric => [metric, subgroupBoundaryExclusions(name, windowReferences(window, metric), window)])), + })) })) + : [] + progress(nightIndex + 1, assignments.length) + return { cohortId: assignment.cohortId, sessionIds: assignment.sessionIds, + uniqueObservationCount: signalReplay + ? new Set([...waveformOutputs.values()].filter(output => output.cohortId === assignment.cohortId) + .flatMap(output => output.readings.filter(row => row.side === dataset.snapshot.side + && windows.some(window => row.timestampMs >= window.startMs && row.timestampMs < window.endMs)).map(row => row.timestampMs))).size + : vitals.length, + ...(signalReplay ? { numericComparisons, numericSubgroups, recordedObservationCount: vitals.length, observationCountBasis: 'union_of_derived_timestamps' } : {}), evaluations, comparisons } + }) + const stageNightSummary = summarizeStageNights(nights, job.protocol.nightUncertainty) + const stagePolicy = job.protocol.stageObjectivePolicy + const stageObjectiveAssessments = stagePolicy + ? stageNightSummary.filter(row => row.candidate === 'candidate').map(summary => ({ + baseline: summary.baseline, candidate: summary.candidate, baselineHash: summary.baselineHash, candidateHash: summary.candidateHash, + ...assessStageObjective(stagePolicy, { split: job.split, holdoutUse: job.holdoutUse, intervalLower: summary.uncertainty.interval?.lower ?? null, operational: null, + nights: summary.nights.map(night => ({ cohortId: night.cohortId, pairedDurationMs: night.commonSupport.candidate.pairedDurationMs, + agreementDifference: night.commonAgreementDifference, macroF1Difference: night.commonMacroF1Difference, + candidateOutputCoverage: night.ownSupport.candidate.outputCoverage, baselineOutputMs: night.ownSupport.baseline.outputMs, commonOutputMs: night.commonOutputDurationMs, + perStage: night.commonSupport.candidate.perStage.map((row) => { + const baseline = night.commonSupport.baseline.perStage.find(item => item.stage === row.stage) + return { stage: row.stage, referenceMs: row.referenceMs, f1Difference: row.f1 === null || baseline?.f1 == null ? null : row.f1 - baseline.f1 } + }), + })), + }, null), + })) + : [] + const numericNightSummary = signalReplay ? summarizeNumericNights(nights, job.protocol.nightUncertainty) : [] + const numericSubgroupSummary = signalReplay && numericObjectivePolicies(job.protocol).length + ? numericSubgroupNames.flatMap(name => summarizeNumericNights(nights.map(night => ({ cohortId: night.cohortId, + numericComparisons: night.numericSubgroups?.find(group => group.name === name)?.numericComparisons }))) + .map(summary => ({ name, version: 'reference-defined-numeric-subgroups-v1' as const, + referenceMembershipBasis: 'native_reference_time' as const, observationCoverageScope: 'complete_scoring_windows' as const, + excludedBoundaryReferenceCount: nights.reduce((sum, night) => sum + (night.numericSubgroups?.find(group => group.name === name)?.numericComparisons.reduce((count, window) => count + (window.excludedByMetric[summary.metric] ?? 0), 0) ?? 0), 0), summary }))) + : [] + const objectiveAssessments = numericObjectivePolicies(job.protocol).flatMap(objectivePolicy => numericNightSummary + .filter(summary => summary.metric === objectivePolicy.metric && summary.candidate === 'candidate') + .map(summary => ({ baseline: summary.baseline, candidate: summary.candidate, baselineHash: summary.baselineHash, candidateHash: summary.candidateHash, + ...assessNumericObjective(objectivePolicy, { + metricCompatible: !nights.some(night => night.numericComparisons?.some(window => window.metrics.some(row => row.metric === summary.metric + && row.baseline === summary.baseline && row.candidate === summary.candidate && row.commonSupport.candidate.status === 'incompatible'))), + split: job.split, holdoutUse: job.holdoutUse, + nights: summary.nights.map(night => ({ cohortId: night.cohortId, pairedReferences: night.commonSupport.candidate.pairedCount, + maeDifference: night.commonMaeDifference, lostObservations: night.lostObservationTimestamps, + baselineObservationCount: nights.find(row => row.cohortId === night.cohortId)?.numericComparisons?.reduce((sum, window) => sum + + (window.metrics.find(row => row.metric === summary.metric && row.baseline === summary.baseline && row.candidate === summary.candidate)?.ownSupport.baseline.validObservationCount ?? 0), 0) ?? 0 })), + intervalUpper: summary.uncertainty.interval?.upper ?? null, + subgroups: subgroupObjectiveEvidence(numericSubgroupSummary.map(group => ({ ...group.summary, name: group.name })), summary, objectivePolicy.minimumPairedReferencesPerNight), operational: null, + }, null), + }))) + return { side: dataset.snapshot.side, version: signalReplay ? 'waveform-backtest-report-v1' as const : 'processed-backtest-report-v1' as const, + ...(signalReplay ? { waveform: { ...(job.waveform?.baselineProvenance ? { baselineProvenance: job.waveform.baselineProvenance } : {}), pythonBuildHash: job.waveform?.pythonBuildHash, otherSideConfigHash: job.waveform?.otherSide.resolvedHash, ...waveform, resources: summarizeWaveformResources(waveform?.outputs ?? []) } } : {}), datasetId: job.datasetId, datasetHash: job.datasetHash, + evaluatorBuildHash: job.evaluatorBuildHash, protocol: job.protocol, protocolHash: job.protocolHash, manifestHash, support: signalReplay ? 'identical_retained_waveform_inputs' as const : 'identical_retained_epoch_intervals' as const, split: job.split, holdoutUse: job.holdoutUse, + ...(signalReplay ? { numericNightSummary, numericSubgroupSummary } : {}), + objectiveAssessments, stageObjectiveAssessments, + status: 'insufficient_evidence' as const, evidence: job.evidence, nights, stageNightSummary, + calibration: { source: 'explicit_assumption' as const, quality: job.protocol.assumedStageCalibrationQuality }, + limitations: ['Only a qualified, complete independent evaluation can establish improvement in reference agreement.', + ...(signalReplay ? ['Numeric comparisons retain own-support scores and scores on observation timestamps shared by all configurations; coverage loss and ambiguous timestamps are reported separately. Common-support differences require qualified objective and coverage assessment before they support promotion.'] : []), + signalReplay + ? 'Numeric readings are replayed from retained waveforms; missing historical context, event-time substitution and cold starts constrain parity.' + : 'Numeric readings are original observations; parameter changes to waveform processing are not evaluated by processed-data replay.', + 'Stages use the shared retrospective classifier, including its bounded gap-duration semantics; they are not causal predictions.', + 'Calibration quality is an explicit assumption, not recovered historical sensor calibration.', + 'Overlapping sessions share one input set; disconnected sessions are classified separately.', + 'Holdout exposure must be rechecked before any later promotion decision.'], + } + } +} diff --git a/src/biometrics/backtest/evaluator.ts b/src/biometrics/backtest/evaluator.ts new file mode 100644 index 000000000..4ba5ebb8a --- /dev/null +++ b/src/biometrics/backtest/evaluator.ts @@ -0,0 +1,5 @@ +import { createHash } from 'node:crypto' +import { configHash, dataHash } from '../algorithms/canonical' +import { createBacktestEvaluator } from './evaluator-core' +export { backtestWorkSchema, waveformEvaluationSchema } from './evaluator-core' +export const evaluateBacktest = createBacktestEvaluator({ configHash, dataHash, digest: text => createHash('sha256').update(text).digest() }) diff --git a/src/biometrics/backtest/execution-budget.ts b/src/biometrics/backtest/execution-budget.ts new file mode 100644 index 000000000..539bcdf49 --- /dev/null +++ b/src/biometrics/backtest/execution-budget.ts @@ -0,0 +1,14 @@ +import { z } from 'zod' + +/** Trusted deployment settings, never an experiment/configuration or RPC field. + * These ceilings permit qualification of whole nights; they do not enable a + * worker host or replace ongoing resource admission and lease fencing. + */ +export const replayBudgetSchema = z.object({ + jobWallMs: z.number().int().min(1000).max(48 * 60 * 60 * 1000), + waveformWallMs: z.number().int().min(1000).max(60 * 60 * 1000), + waveformCpuSeconds: z.number().int().min(1).max(3600), +}).strict().refine(budget => budget.waveformWallMs <= budget.jobWallMs + && budget.waveformCpuSeconds * 1000 <= budget.waveformWallMs, +'Waveform CPU and wall budgets must fit within the job budget') +export type ReplayBudget = z.infer diff --git a/src/biometrics/backtest/host-startup.ts b/src/biometrics/backtest/host-startup.ts new file mode 100644 index 000000000..e0f257bff --- /dev/null +++ b/src/biometrics/backtest/host-startup.ts @@ -0,0 +1,11 @@ +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import type * as schema from '@/src/db/biometrics-schema' +import { configuredReplayPolicy, replayAdmitter, replayHealthAdmission } from './admission-service' +import { startBacktestHost } from './host' + +/** Enable polling only for an explicitly configured deployment. Startup does + * not substitute for qualification or fresh health checks on every operation. */ +export function startConfiguredBacktestHost(db: BetterSQLite3Database) { + if (!configuredReplayPolicy()) return null + return startBacktestHost(db, replayAdmitter(db), replayHealthAdmission(db)) +} diff --git a/src/biometrics/backtest/host.ts b/src/biometrics/backtest/host.ts new file mode 100644 index 000000000..ce3cd189e --- /dev/null +++ b/src/biometrics/backtest/host.ts @@ -0,0 +1,101 @@ +import type { HealthAdmission } from './job-operations' +import { randomUUID } from 'node:crypto' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import type * as schema from '@/src/db/biometrics-schema' +import { coordinatorRequest } from './coordinator-service' + +interface HostOptions { + execute: (signal: AbortSignal) => Promise + collect: (signal: AbortSignal) => void | Promise + onError?: (phase: 'execution' | 'cleanup') => void + intervalMs?: number + clock?: () => number +} + +/** One serialized host loop. The runner still owns the database lease, so + * multiple processes cannot bypass fencing by each creating a local host. */ +export class BacktestHost { + private stopped = false + private active: Promise | null = null + private timer: ReturnType | null = null + private controller = new AbortController() + private lastCollection = -Infinity + private readonly clock: () => number + constructor(private readonly options: HostOptions) { + this.clock = options.clock ?? Date.now + if (!Number.isSafeInteger(options.intervalMs ?? 5000) || (options.intervalMs ?? 5000) < 10) throw new Error('Invalid backtest host interval') + } + + start() { + if (!this.stopped && !this.active && !this.timer) this.tick() + } + + private report(phase: 'execution' | 'cleanup') { + try { + this.options.onError?.(phase) + } + catch { /* Observability must not stop recovery. */ } + } + + private tick() { + if (this.stopped || this.active) return + this.timer = null + this.active = this.run().finally(() => { + this.active = null + if (!this.stopped) { + this.timer = setTimeout(() => this.tick(), this.options.intervalMs ?? 5000) + this.timer.unref() + } + }) + } + + private async run() { + if (this.clock() - this.lastCollection >= 1800000) { + this.lastCollection = this.clock() + try { + await this.options.collect(this.controller.signal) + } + catch { + this.report('cleanup') + } + } + if (this.stopped) return + try { + await this.options.execute(this.controller.signal) + } + catch { + this.report('execution') + } + } + + /** Abort the child, then await its lease interruption before DB closure. */ + async stop() { + this.stopped = true + if (this.timer) clearTimeout(this.timer) + this.timer = null + this.controller.abort() + await this.active + } +} + +const shared = globalThis as typeof globalThis & { __biometricBacktestHost?: BacktestHost } +/** Admission is mandatory and supplied by the deployment's resource policy. + * There is deliberately no permissive fallback for missing health evidence. */ +export function startBacktestHost(db: BetterSQLite3Database, admit: () => Promise, healthAdmission?: HealthAdmission) { + if (shared.__biometricBacktestHost) return shared.__biometricBacktestHost + const owner = `core:${process.pid}:${randomUUID()}` + const host = new BacktestHost({ + execute: signal => coordinatorRequest(db, { type: 'execute', owner }, { signal, admit, healthAdmission }), + collect: signal => coordinatorRequest(db, { type: 'collectArtifacts' }, { signal, admit, healthAdmission }).then(() => {}), + onError: phase => console.warn(`[biometrics] Backtest ${phase} failed; host will retry`), + }) + shared.__biometricBacktestHost = host + host.start() + return host +} +export async function stopBacktestHost() { + const host = shared.__biometricBacktestHost + if (!host) return + await host.stop() + if (shared.__biometricBacktestHost === host) delete shared.__biometricBacktestHost +} diff --git a/src/biometrics/backtest/job-operations.ts b/src/biometrics/backtest/job-operations.ts new file mode 100644 index 000000000..fbc554110 --- /dev/null +++ b/src/biometrics/backtest/job-operations.ts @@ -0,0 +1,64 @@ +import { processGroupMemorySchema, type ProcessGroupMemory } from './process-group-memory-contract' +import { z } from 'zod' + +/** Internal supervisor evidence, never accepted from experiment/RPC payloads. */ +export const admissionReceiptSchema = z.object({ + allowed: z.boolean(), + memory: processGroupMemorySchema.optional(), + sample: z.object({ qualificationId: z.string().min(1).max(200), checkedAtMs: z.number().int().nonnegative(), + ingestionAgeMs: z.number().int().nonnegative() }).strict().nullable(), +}).strict() +export const jobOperationsSchema = z.object({ + version: z.literal('backtest-job-operations-v1'), wallMs: z.number().finite().nonnegative(), + scope: z.literal('admission_through_prepublication'), + memory: processGroupMemorySchema.nullable().optional(), + ingestion: z.object({ source: z.literal('supervisor_live_piezo_receipt_age'), + checks: z.number().int().nonnegative(), samples: z.number().int().nonnegative(), allChecksMeasured: z.boolean(), + qualificationId: z.string().nullable(), maximumObservedAgeMs: z.number().int().nonnegative().nullable(), + maximumObservationGapMs: z.number().finite().nonnegative(), nominalIntervalMs: z.literal(5000), continuousMeasurement: z.literal(false), + }).strict(), +}).strict() + +export type AdmissionReceipt = z.infer +export type HealthAdmission = () => Promise + +/** Bounded accumulation; no health frames or unbounded sample arrays. Clock + * gaps are measured monotonically, independently of wall-clock adjustments. */ +export function createJobOperations(now: () => number = () => performance.now()) { + const start = now() + let last = start, checks = 0, samples = 0, maximumGapMs = 0, maximumIngestionAgeMs = 0 + let memory: ProcessGroupMemory | null = null + let qualificationId: string | null = null, consistent = true + return { + record(raw: unknown) { + const at = now() + maximumGapMs = Math.max(maximumGapMs, at - last) + last = at + checks++ + const receipt = admissionReceiptSchema.safeParse(raw) + if (receipt.success && receipt.data.memory) memory = receipt.data.memory + if (!receipt.success || !receipt.data.allowed || !receipt.data.sample) { + consistent = false + return + } + const sample = receipt.data.sample + if (qualificationId !== null && qualificationId !== sample.qualificationId) consistent = false + qualificationId ??= sample.qualificationId + maximumIngestionAgeMs = Math.max(maximumIngestionAgeMs, sample.ingestionAgeMs) + samples++ + }, + finish() { + const end = now() + return { version: 'backtest-job-operations-v1' as const, wallMs: Math.max(0, end - start), + scope: 'admission_through_prepublication' as const, memory, + ingestion: { source: 'supervisor_live_piezo_receipt_age' as const, checks, samples, + allChecksMeasured: consistent && checks >= 2 && samples === checks, + qualificationId: consistent ? qualificationId : null, + maximumObservedAgeMs: samples > 0 ? maximumIngestionAgeMs : null, + maximumObservationGapMs: Math.max(maximumGapMs, end - last), + nominalIntervalMs: 5000, + continuousMeasurement: false as const }, + } + }, + } +} diff --git a/src/biometrics/backtest/jobs.ts b/src/biometrics/backtest/jobs.ts new file mode 100644 index 000000000..afdd33f19 --- /dev/null +++ b/src/biometrics/backtest/jobs.ts @@ -0,0 +1,340 @@ +import { randomUUID } from 'node:crypto' +import { dirname, join } from 'node:path' +import { and, asc, desc, eq, gt, inArray, lt, lte, or, sql } from 'drizzle-orm' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import { z } from 'zod' +import * as schema from '@/src/db/biometrics-schema' +import { configHash, dataHash } from '../algorithms/canonical' +import { manifestHash, validateConfig } from '../algorithms/registry' +import { ReferenceInputError } from '../references/time' +import { DatasetStore } from './datasets' +import { MAX_REPORT_BYTES, REPORT_BUDGET_BYTES, ReportArtifactStore, reportArtifactSchema } from './artifacts' + +const jobs = schema.biometricBacktestJobs +const experiments = schema.biometricExperiments +const results = schema.biometricBacktestResults +const exposure = schema.biometricHoldoutAccesses +const manifestSchema = z.object({ splitProtocol: z.object({ assignments: z.array(z.object({ split: z.string(), startMs: z.number(), endMs: z.number() })) }) }) +const key = z.string().min(1).max(200) +export { experimentProtocolSchema, enqueueBacktestSchema } from './contracts' +import { experimentProtocolSchema, enqueueBacktestSchema } from './contracts' +const progressSchema = z.object({ phase: z.enum(['queued', 'loading', 'evaluating', 'reporting', 'complete']), + completedNights: z.number().int().min(0).max(60), totalNights: z.number().int().min(0).max(60) }).strict() +const leaseSchema = z.object({ id: z.string().uuid(), owner: key, token: z.number().int().positive() }).strict() +export type JobLease = z.infer + +/** Leases fence every publication; all expensive work happens outside SQLite. */ +export class BacktestJobStore { + constructor(private readonly db: BetterSQLite3Database, private readonly now = Date.now, + private readonly artifactStore?: ReportArtifactStore) {} + + private storage() { + if (this.artifactStore) return this.artifactStore + const main = this.db.all<{ name: string, file: string }>(sql`PRAGMA database_list`).find(row => row.name === 'main') + return main?.file ? new ReportArtifactStore(join(dirname(main.file), 'biometric-report-artifacts')) : undefined + } + + private artifacts() { + const store = this.storage() + if (!store) throw new ReferenceInputError('RESOURCE_LIMIT', 'Large reports require a persistent artifact directory') + return store + } + + collectOrphanedArtifacts() { + const store = this.artifacts() + return store.withPublication(() => { + const referenced = new Set(this.db.select({ report: results.report }).from(results).all().flatMap((row) => { + // Preserve named future-version descriptors during downgrade. + const artifact = z.object({ name: z.string() }).safeParse(row.report) + return artifact.success ? [artifact.data.name] : [] + })) + return store.collect(referenced, this.now()) + }) + } + + createExperiment(datasetId: string, rawProtocol: z.input, idempotencyKey: string = randomUUID()) { + const protocol = experimentProtocolSchema.parse(rawProtocol) + if (protocol.waveformStages === undefined) delete protocol.waveformStages + key.parse(idempotencyKey) + const requestHash = configHash({ datasetId, protocol }) + const dataset = new DatasetStore(this.db).read(datasetId) + if (protocol.mode === 'signal_replay' && dataset.snapshot.version !== 'waveform-dataset-v1') throw new ReferenceInputError('INSUFFICIENT_RAW_DATA', 'Signal replay requires retained waveform inputs') + return this.db.transaction((tx) => { + const existing = tx.select().from(experiments).where(eq(experiments.idempotencyKey, idempotencyKey)).get() + if (existing) { + if (existing.requestHash !== requestHash) throw new ReferenceInputError('REVISION_CONFLICT', 'Experiment idempotency key changed') + return existing + } + const total = tx.select({ count: sql`count(*)` }).from(experiments).get()?.count ?? 0 + if (total >= 1000) throw new ReferenceInputError('RESOURCE_LIMIT', 'Experiment history has reached its 1000-entry bound') + return tx.insert(experiments).values({ id: randomUUID(), datasetId, idempotencyKey, requestHash, side: dataset.side, + protocol, protocolHash: configHash(protocol), createdAtMs: this.now() }).returning().get() + }, { behavior: 'immediate' }) + } + + getExperiment(id: string) { + const experiment = this.db.select().from(experiments).where(eq(experiments.id, id)).get() + if (!experiment) throw new ReferenceInputError('NOT_FOUND', 'Experiment not found') + return { ...experiment, protocol: experimentProtocolSchema.parse(experiment.protocol) } + } + + pageExperiments(side: 'left' | 'right', cursor?: { createdAtMs: number, id: string }) { + const rows = this.db.select().from(experiments).where(and(eq(experiments.side, side), cursor + ? or(lt(experiments.createdAtMs, cursor.createdAtMs), and(eq(experiments.createdAtMs, cursor.createdAtMs), lt(experiments.id, cursor.id))) + : undefined)) + .orderBy(desc(experiments.createdAtMs), desc(experiments.id)).limit(51).all() + const items = rows.slice(0, 50).map(row => ({ ...row, protocol: experimentProtocolSchema.parse(row.protocol) })) + const last = items.at(-1) + return { items, nextCursor: rows.length > 50 && last ? { createdAtMs: last.createdAtMs, id: last.id } : null } + } + + holdoutUseCount(datasetId: string) { + const dataset = this.db.select({ side: schema.biometricDatasets.side, manifest: schema.biometricDatasets.manifest }) + .from(schema.biometricDatasets).where(eq(schema.biometricDatasets.id, datasetId)).get() + if (!dataset) throw new ReferenceInputError('NOT_FOUND', 'Dataset unavailable') + const windows = manifestSchema.parse(dataset.manifest).splitProtocol.assignments.filter(row => row.split === 'holdout') + if (!windows.length) return 0 + return this.db.selectDistinct({ jobId: exposure.jobId }).from(exposure).where(and(eq(exposure.side, dataset.side), + or(...windows.map(window => and(lt(exposure.startMs, window.endMs), gt(exposure.endMs, window.startMs)))), + )).all().length + } + + enqueue(raw: z.input) { + const input = enqueueBacktestSchema.parse(raw) + if (input.waveform === undefined) delete input.waveform + if (new Set(input.configs.map(config => config.label)).size !== input.configs.length) throw new ReferenceInputError('INVALID_CONFIG', 'Baseline labels must be unique') + for (const config of input.configs) { + const resolved = validateConfig(config.resolved) + if (config.manifestHash !== manifestHash || configHash(resolved) !== config.resolvedHash) throw new ReferenceInputError('INVALID_CONFIG', 'Resolved config or engine hash does not match this build') + } + if (Buffer.byteLength(JSON.stringify(input)) > 64 * 1024) throw new ReferenceInputError('RESOURCE_LIMIT', 'Backtest configuration snapshot exceeds 64 KiB') + const requestHash = configHash(input) + return this.db.transaction((tx) => { + const existing = tx.select().from(jobs).where(eq(jobs.idempotencyKey, input.idempotencyKey)).get() + if (existing) { + if (existing.requestHash !== requestHash) throw new ReferenceInputError('REVISION_CONFLICT', 'Backtest idempotency key changed') + return existing + } + const total = tx.select({ count: sql`count(*)` }).from(jobs).get()?.count ?? 0 + if (total >= 1000) throw new ReferenceInputError('RESOURCE_LIMIT', 'Job history has reached its 1000-entry bound') + const experiment = tx.select().from(experiments).where(eq(experiments.id, input.experimentId)).get() + if (!experiment) throw new ReferenceInputError('NOT_FOUND', 'Experiment not found') + const protocol = experimentProtocolSchema.parse(experiment.protocol) + if ((protocol.mode === 'signal_replay') !== Boolean(input.waveform)) throw new ReferenceInputError('INVALID_CONFIG', 'Waveform replay requires pinned Python and other-side configuration') + if (input.waveform && (input.waveform.otherSide.manifestHash !== manifestHash + || configHash(validateConfig(input.waveform.otherSide.resolved)) !== input.waveform.otherSide.resolvedHash)) throw new ReferenceInputError('INVALID_CONFIG', 'Other-side configuration identity mismatch') + const dataset = tx.select({ contentHash: schema.biometricDatasets.contentHash, releasedAtMs: schema.biometricDatasets.releasedAtMs, manifest: schema.biometricDatasets.manifest }) + .from(schema.biometricDatasets).where(eq(schema.biometricDatasets.id, experiment.datasetId)).get() + if (!dataset) throw new ReferenceInputError('NOT_FOUND', 'Dataset unavailable') + if (dataset.releasedAtMs !== null) throw new ReferenceInputError('DATASET_RELEASED', 'Retained dataset inputs were explicitly released') + const manifest = manifestSchema.parse(dataset.manifest) + const totalNights = manifest.splitProtocol.assignments.filter(group => group.split === input.split).length + if (!totalNights) throw new ReferenceInputError('INSUFFICIENT_NIGHTS', 'The frozen dataset has no nights assigned to this split') + let holdoutUse = this.holdoutUseCount(experiment.datasetId) + if (input.split === 'holdout') { + holdoutUse++ + tx.update(experiments).set({ holdoutAccesses: experiment.holdoutAccesses + 1 }).where(eq(experiments.id, experiment.id)).run() + } + const now = this.now() + const job = tx.insert(jobs).values({ id: randomUUID(), experimentId: experiment.id, idempotencyKey: input.idempotencyKey, requestHash, + input: { version: 'backtest-job-v1', ...input, datasetId: experiment.datasetId, datasetHash: dataset.contentHash, + protocol: experiment.protocol, protocolHash: experiment.protocolHash, holdoutUse, + evidence: input.split === 'holdout' && holdoutUse > 1 ? 'reused_holdout_is_validation' : 'descriptive_only' }, + state: 'queued', progress: { phase: 'queued', completedNights: 0, totalNights }, createdAtMs: now, updatedAtMs: now, + }).returning().get() + if (input.split === 'holdout') { + for (const window of manifest.splitProtocol.assignments.filter(row => row.split === 'holdout')) { + tx.insert(exposure).values({ id: randomUUID(), jobId: job.id, side: experiment.side, startMs: window.startMs, endMs: window.endMs, accessedAtMs: now }).run() + } + } + return job + }, { behavior: 'immediate' }) + } + + page(side: 'left' | 'right', cursor?: { createdAtMs: number, id: string }) { + const rows = this.db.select({ job: jobs }).from(jobs).innerJoin(experiments, eq(jobs.experimentId, experiments.id)) + .where(and(eq(experiments.side, side), cursor + ? or(lt(jobs.createdAtMs, cursor.createdAtMs), and(eq(jobs.createdAtMs, cursor.createdAtMs), lt(jobs.id, cursor.id))) + : undefined)) + .orderBy(desc(jobs.createdAtMs), desc(jobs.id)).limit(51).all().map(row => row.job) + const items = rows.slice(0, 50) + const last = items.at(-1) + return { items, nextCursor: rows.length > 50 && last ? { createdAtMs: last.createdAtMs, id: last.id } : null } + } + + get(id: string) { + const job = this.db.select().from(jobs).where(eq(jobs.id, id)).get() + if (!job) throw new ReferenceInputError('NOT_FOUND', 'Backtest job not found') + return job + } + + result(id: string) { + const storage = this.storage() + const read = () => { + const job = this.get(id) + if (job.state !== 'succeeded') return null + const result = this.db.select().from(results).where(eq(results.jobId, id)).get() + if (!result || result.fencingToken !== job.fencingToken) throw new ReferenceInputError('RESULT_CORRUPT', 'Backtest result integrity check failed') + const input = z.object({ datasetId: z.string() }).parse(job.input) + const currentHoldoutUseCount = this.holdoutUseCount(input.datasetId) + if (result.releasedAtMs !== null) return { ...result, report: null, available: false, currentHoldoutUseCount } + const artifact = reportArtifactSchema.safeParse(result.report) + const report = artifact.success ? this.artifacts().read(artifact.data) : result.report + if (dataHash(report) !== result.contentHash) throw new ReferenceInputError('RESULT_CORRUPT', 'Backtest result integrity check failed') + return { ...result, report, available: true, currentHoldoutUseCount } + } + // A release cannot unlink an artifact between reading its descriptor and bytes. + return storage ? storage.withPublication(read) : read() + } + + /** Release payload bytes, retaining winning-attempt and holdout audit records. + * The caller must identify the exact report hash it intends to release. */ + releaseResult(id: string, expectedContentHash: string) { + z.string().regex(/^[a-f0-9]{64}$/).parse(expectedContentHash) + const storage = this.storage() + const release = () => { + const previous = this.db.transaction((tx) => { + const job = this.get(id) + if (job.state !== 'succeeded') throw new ReferenceInputError('INVALID_STATE', 'Only completed reports can be released') + const result = tx.select().from(results).where(eq(results.jobId, id)).get() + if (!result || result.fencingToken !== job.fencingToken) throw new ReferenceInputError('RESULT_CORRUPT', 'Backtest result integrity check failed') + if (result.contentHash !== expectedContentHash) throw new ReferenceInputError('REVISION_CONFLICT', 'Report hash changed') + if (result.releasedAtMs === null) { + tx.update(results).set({ report: {}, releasedAtMs: this.now() }).where(eq(results.jobId, id)).run() + } + return result.report + }, { behavior: 'immediate' }) + const artifact = reportArtifactSchema.safeParse(previous) + if (artifact.success && storage) { + const stillReferenced = this.db.select({ report: results.report }).from(results).all().some((row) => { + const named = z.object({ name: z.string() }).safeParse(row.report) + return named.success && named.data.name === artifact.data.name + }) + if (!stillReferenced) storage.removeReleased(artifact.data) + } + } + if (storage) storage.withPublication(release) + else release() + return this.result(id) + } + + /** Recover expired attempts without granting stale owners publication rights. */ + recover() { + const now = this.now() + return this.db.transaction((tx) => { + const expired = tx.select().from(jobs).where(and(inArray(jobs.state, ['running', 'cancel_requested']), lte(jobs.leaseExpiresAtMs, now))).all() + for (const job of expired) { + tx.update(jobs).set({ state: job.state === 'cancel_requested' ? 'cancelled' : job.attempt >= 3 ? 'failed' : 'interrupted', + leaseOwner: null, leaseExpiresAtMs: null, error: job.state === 'cancel_requested' ? null : 'Worker lease expired', updatedAtMs: now, + }).where(eq(jobs.id, job.id)).run() + } + return expired.length + }, { behavior: 'immediate' }) + } + + claim(owner: string, leaseMs = 30000) { + key.parse(owner) + z.number().int().min(1000).max(120000).parse(leaseMs) + this.recover() + return this.db.transaction((tx) => { + const now = this.now() + const active = tx.select({ id: jobs.id }).from(jobs).where(and(inArray(jobs.state, ['running', 'cancel_requested']), gt(jobs.leaseExpiresAtMs, now))).get() + if (active) return null // one background replay at a time across processes + const job = tx.select().from(jobs).where(inArray(jobs.state, ['queued', 'interrupted'])).orderBy(asc(jobs.createdAtMs), asc(jobs.id)).get() + if (!job) return null + return tx.update(jobs).set({ state: 'running', leaseOwner: owner, leaseExpiresAtMs: now + leaseMs, + heartbeatAtMs: now, fencingToken: job.fencingToken + 1, attempt: job.attempt + 1, + progress: { ...progressSchema.parse(job.progress), phase: 'loading', completedNights: 0 }, error: null, updatedAtMs: now, + }).where(eq(jobs.id, job.id)).returning().get() + }, { behavior: 'immediate' }) + } + + private owned(lease: JobLease) { + leaseSchema.parse(lease) + const job = this.get(lease.id) + if (job.leaseOwner !== lease.owner || job.fencingToken !== lease.token || (job.leaseExpiresAtMs ?? 0) <= this.now() + || !['running', 'cancel_requested'].includes(job.state)) throw new ReferenceInputError('STALE_LEASE', 'Worker lease is no longer authoritative') + return job + } + + heartbeat(lease: JobLease, rawProgress: z.input, leaseMs = 30000) { + const progress = progressSchema.parse(rawProgress) + z.number().int().min(1000).max(120000).parse(leaseMs) + return this.db.transaction((tx) => { + const job = this.owned(lease) + const previous = progressSchema.parse(job.progress) + if (progress.totalNights !== previous.totalNights || progress.completedNights < previous.completedNights || progress.completedNights > progress.totalNights) throw new ReferenceInputError('INVALID_PROGRESS', 'Progress must be monotonic within the frozen night count') + const now = this.now() + tx.update(jobs).set({ progress, heartbeatAtMs: now, leaseExpiresAtMs: now + leaseMs, updatedAtMs: now }).where(eq(jobs.id, job.id)).run() + return { cancelRequested: job.state === 'cancel_requested' } + }, { behavior: 'immediate' }) + } + + cancel(id: string) { + return this.db.transaction((tx) => { + const job = this.get(id) + if (['succeeded', 'failed', 'cancelled'].includes(job.state)) return job + const now = this.now() + return tx.update(jobs).set({ state: job.state === 'queued' || job.state === 'interrupted' ? 'cancelled' : 'cancel_requested', + cancellationRequestedAtMs: job.cancellationRequestedAtMs ?? now, updatedAtMs: now }).where(eq(jobs.id, id)).returning().get() + }, { behavior: 'immediate' }) + } + + acknowledgeCancellation(lease: JobLease) { + return this.db.transaction((tx) => { + const job = this.owned(lease) + if (job.state !== 'cancel_requested') throw new ReferenceInputError('INVALID_STATE', 'Cancellation has not been requested') + return tx.update(jobs).set({ state: 'cancelled', leaseOwner: null, leaseExpiresAtMs: null, updatedAtMs: this.now() }).where(eq(jobs.id, job.id)).returning().get() + }, { behavior: 'immediate' }) + } + + interrupt(lease: JobLease, reason: 'Worker host stopped' | 'Resource admission withdrawn' = 'Worker host stopped') { + return this.db.transaction((tx) => { + const job = this.owned(lease) + return tx.update(jobs).set({ state: job.state === 'cancel_requested' ? 'cancelled' : job.attempt >= 3 ? 'failed' : 'interrupted', + error: reason, leaseOwner: null, leaseExpiresAtMs: null, updatedAtMs: this.now() }).where(eq(jobs.id, job.id)).returning().get() + }, { behavior: 'immediate' }) + } + + fail(lease: JobLease, message: string) { + return this.db.transaction((tx) => { + const job = this.owned(lease) + return tx.update(jobs).set({ state: job.state === 'cancel_requested' ? 'cancelled' : 'failed', + error: message.slice(0, 1000), leaseOwner: null, leaseExpiresAtMs: null, updatedAtMs: this.now() }).where(eq(jobs.id, job.id)).returning().get() + }, { behavior: 'immediate' }) + } + + complete(lease: JobLease, report: unknown) { + // Concrete evaluator schemas validate reports before calling this sink. + const serialized = JSON.stringify(report) + if (!serialized || Buffer.byteLength(serialized) > MAX_REPORT_BYTES) throw new ReferenceInputError('RESOURCE_LIMIT', 'Backtest report exceeds 16 MiB') + const contentHash = dataHash(report) + const artifacts = this.storage() + const publish = () => { + const owned = this.owned(lease) + if (owned.state !== 'running') throw new ReferenceInputError('INVALID_STATE', 'Cancelled work cannot publish a result') + let stored: unknown = report + const inlineBytes = this.db.select({ bytes: sql`coalesce(sum(case when json_extract(${results.report}, '$.kind') = 'biometric-report-artifact-v1' then 0 else length(cast(${results.report} as blob)) end), 0)` }).from(results).get()?.bytes ?? 0 + if (Buffer.byteLength(serialized) > 2 * 1024 * 1024) { + if (!artifacts) throw new ReferenceInputError('RESOURCE_LIMIT', 'Large reports require a persistent artifact directory') + stored = artifacts.write(lease.id, lease.token, serialized, inlineBytes) + } + else if (inlineBytes + (artifacts?.usage() ?? 0) + Buffer.byteLength(serialized) > REPORT_BUDGET_BYTES) { + throw new ReferenceInputError('RESOURCE_LIMIT', 'Reports and staged artifacts reached the 64 MiB storage budget') + } + return this.db.transaction((tx) => { + const job = this.owned(lease) + if (job.state !== 'running') throw new ReferenceInputError('INVALID_STATE', 'Cancelled work cannot publish a result') + const retained = tx.select({ bytes: sql`coalesce(sum(case when json_extract(${results.report}, '$.kind') = 'biometric-report-artifact-v1' then json_extract(${results.report}, '$.byteLength') else length(cast(${results.report} as blob)) end), 0)` }).from(results).get()?.bytes ?? 0 + if (retained + Buffer.byteLength(serialized) > REPORT_BUDGET_BYTES) throw new ReferenceInputError('RESOURCE_LIMIT', 'Backtest summaries have reached their 64 MiB storage budget') + const now = this.now() + tx.insert(results).values({ jobId: job.id, fencingToken: lease.token, report: stored, contentHash, completedAtMs: now }).run() + const progress = progressSchema.parse(job.progress) + tx.update(jobs).set({ state: 'succeeded', progress: { ...progress, phase: 'complete', completedNights: progress.totalNights }, + leaseOwner: null, leaseExpiresAtMs: null, updatedAtMs: now }).where(eq(jobs.id, job.id)).run() + return { jobId: job.id, contentHash } + }, { behavior: 'immediate' }) + } + return artifacts ? artifacts.withPublication(publish) : publish() + } +} diff --git a/src/biometrics/backtest/location.ts b/src/biometrics/backtest/location.ts new file mode 100644 index 000000000..ed8fdd254 --- /dev/null +++ b/src/biometrics/backtest/location.ts @@ -0,0 +1,30 @@ +import { existsSync } from 'node:fs' +import { resolve } from 'node:path' +import { ReferenceInputError } from '../references/time' + +/** Server-owned fixed package locations; no caller-supplied executable paths. + * These bundles are emitted by scripts/build/biometrics-worker.mjs AFTER Next + * tracing. Tracing their runtime probes would recursively include the build tree. */ +export function packagedWorkerPath() { + for (const relative of ['biometrics-worker/worker.cjs', '.next/standalone/biometrics-worker/worker.cjs']) { + const path = resolve(/* turbopackIgnore: true */ relative) + if (existsSync(/* turbopackIgnore: true */ path)) return path + } + throw new ReferenceInputError('WORKER_UNAVAILABLE', 'Build the packaged biometrics worker before submitting a replay') +} + +export function packagedCoordinatorPath() { + for (const relative of ['biometrics-worker/coordinator.cjs', '.next/standalone/biometrics-worker/coordinator.cjs']) { + const path = resolve(/* turbopackIgnore: true */ relative) + if (existsSync(/* turbopackIgnore: true */ path)) return path + } + throw new ReferenceInputError('WORKER_UNAVAILABLE', 'Build the packaged biometrics coordinator before accessing retained evaluation data') +} + +export function packagedShadowHostPath() { + for (const relative of ['biometrics-worker/shadow-host.cjs', '.next/standalone/biometrics-worker/shadow-host.cjs']) { + const path = resolve(/* turbopackIgnore: true */ relative) + if (existsSync(/* turbopackIgnore: true */ path)) return path + } + throw new ReferenceInputError('WORKER_UNAVAILABLE', 'Build the packaged shadow host before scheduling prospective runs') +} diff --git a/src/biometrics/backtest/night-summary-core.ts b/src/biometrics/backtest/night-summary-core.ts new file mode 100644 index 000000000..36adfa5ee --- /dev/null +++ b/src/biometrics/backtest/night-summary-core.ts @@ -0,0 +1,99 @@ +import type { NightBootstrapPolicy } from './uncertainty-policy' +import type { createNightUncertainty } from './uncertainty-core' +import type { compareStageOutputs } from './stage-comparison' + +type WindowComparison = ReturnType & { startMs: number, endMs: number } +interface NightComparison { + baseline: string + candidate: string + baselineHash: string + candidateHash: string + windows: WindowComparison[] +} +interface Night { cohortId: string, comparisons: NightComparison[] } +const stages = ['wake', 'light', 'deep', 'rem'] as const +const difference = (a: number | null, b: number | null) => a === null || b === null ? null : a - b +const mean = (values: number[]) => values.length ? values.reduce((sum, value) => sum + value, 0) / values.length : null + +function commonScore(windows: WindowComparison[], side: 'baseline' | 'candidate') { + const confusionMs = { wake: { wake: 0, light: 0, deep: 0, rem: 0 }, light: { wake: 0, light: 0, deep: 0, rem: 0 }, + deep: { wake: 0, light: 0, deep: 0, rem: 0 }, rem: { wake: 0, light: 0, deep: 0, rem: 0 } } + for (const window of windows) { + const matrix = side === 'baseline' ? window.commonSupport.baselineConfusionMs : window.commonSupport.candidateConfusionMs + for (const reference of stages) for (const predicted of stages) confusionMs[reference][predicted] += matrix[reference][predicted] + } + const perStage = stages.map((stage) => { + const referenceMs = stages.reduce((sum, predicted) => sum + confusionMs[stage][predicted], 0) + const predictedMs = stages.reduce((sum, reference) => sum + confusionMs[reference][stage], 0) + const correctMs = confusionMs[stage][stage] + return { stage, referenceMs, predictedMs, precision: predictedMs ? correctMs / predictedMs : null, + recall: referenceMs ? correctMs / referenceMs : null, f1: referenceMs + predictedMs ? 2 * correctMs / (referenceMs + predictedMs) : null } + }) + const pairedDurationMs = perStage.reduce((sum, row) => sum + row.referenceMs, 0) + const correctMs = stages.reduce((sum, stage) => sum + confusionMs[stage][stage], 0) + return { confusionMs, perStage, pairedDurationMs, agreement: pairedDurationMs ? correctMs / pairedDurationMs : null, + macroF1: mean(perStage.flatMap(row => row.f1 === null ? [] : [row.f1])) } +} + +function summarizeNight(cohortId: string, comparison: NightComparison) { + const baseline = commonScore(comparison.windows, 'baseline'), candidate = commonScore(comparison.windows, 'candidate') + const durationMs = comparison.windows.reduce((sum, window) => sum + window.endMs - window.startMs, 0) + const ownSupport = (side: 'baseline' | 'candidate') => { + const totals = comparison.windows.reduce((sum, window) => { + const score = window.ownSupport[side] + return { outputMs: sum.outputMs + score.outputDurationMs, pairedMs: sum.pairedMs + score.pairedDurationMs, + referenceMs: sum.referenceMs + score.eligibleReferenceMs, correctMs: sum.correctMs + (score.agreement ?? 0) * score.pairedDurationMs } + }, { outputMs: 0, pairedMs: 0, referenceMs: 0, correctMs: 0 }) + return { ...totals, agreement: totals.pairedMs ? totals.correctMs / totals.pairedMs : null, + outputCoverage: durationMs ? totals.outputMs / durationMs : null, + referenceCoverage: totals.referenceMs ? totals.pairedMs / totals.referenceMs : null } + } + return { cohortId, evaluationDurationMs: durationMs, commonOutputDurationMs: comparison.windows.reduce((sum, window) => sum + window.commonSupport.baseline.outputDurationMs, 0), commonSupport: { baseline, candidate }, + ownSupport: { baseline: ownSupport('baseline'), candidate: ownSupport('candidate') }, + commonAgreementDifference: difference(candidate.agreement, baseline.agreement), + commonMacroF1Difference: difference(candidate.macroF1, baseline.macroF1), + coverageReduced: comparison.windows.some(window => window.coverageReduced) } +} + +/** Aggregate disconnected windows within each whole-night cohort first. Equal + * night means and pooled duration scores are both explicit; neither silently + * substitutes epochs for independent nights. No promotion status is inferred. */ +export function createNightSummary(nightUncertainty: ReturnType) { + return function summarizeStageNights(nights: Night[], uncertaintyPolicy?: NightBootstrapPolicy) { + if (nights.length > 60 || new Set(nights.map(night => night.cohortId)).size !== nights.length) throw new Error('Invalid whole-night summary input') + const groups = new Map, nights: ReturnType[], windows: WindowComparison[] }>() + for (const night of nights) { + if (night.comparisons.length > 3) throw new Error('Too many comparisons within night') + const seen = new Set() + for (const comparison of night.comparisons) { + const { windows, ...identity } = comparison + const ordered = [...windows].sort((a, b) => a.startMs - b.startMs) + if (ordered.length > 60 || ordered.some((window, index) => !Number.isSafeInteger(window.startMs) || !Number.isSafeInteger(window.endMs) + || window.endMs <= window.startMs || window.endMs - window.startMs > 86400000 || (index > 0 && ordered[index - 1].endMs > window.startMs))) { + throw new Error('Nightly comparison windows must be bounded and nonoverlapping') + } + const key = JSON.stringify(identity) + if (seen.has(key)) throw new Error('Duplicate comparison within night') + seen.add(key) + const group = groups.get(key) ?? { identity, nights: [], windows: [] } + group.nights.push(summarizeNight(night.cohortId, comparison)) + group.windows.push(...windows) + groups.set(key, group) + } + } + return [...groups.values()].map((group) => { + const deltas = group.nights.flatMap(night => night.commonAgreementDifference === null ? [] : [night.commonAgreementDifference]) + return { ...group.identity, nights: group.nights, + totalNights: group.nights.length, scoredNights: deltas.length, unscoredNights: group.nights.length - deltas.length, + improvedAgreementNights: deltas.filter(delta => delta > 0).length, + regressedAgreementNights: deltas.filter(delta => delta < 0).length, + unchangedAgreementNights: deltas.filter(delta => delta === 0).length, + coverageReducedNights: group.nights.filter(night => night.coverageReduced).length, + equalNightMeanAgreementDifference: mean(deltas), + uncertainty: nightUncertainty(group.nights.map(night => ({ cohortId: night.cohortId, difference: night.commonAgreementDifference })), uncertaintyPolicy), + observedNightRange: deltas.length ? { minimum: Math.min(...deltas), maximum: Math.max(...deltas) } : null, + pooledCommonSupport: { baseline: commonScore(group.windows, 'baseline'), candidate: commonScore(group.windows, 'candidate') }, + interpretation: 'descriptive_night_aggregation_not_confidence_interval' as const } + }) + } +} diff --git a/src/biometrics/backtest/night-summary.ts b/src/biometrics/backtest/night-summary.ts new file mode 100644 index 000000000..108d2f579 --- /dev/null +++ b/src/biometrics/backtest/night-summary.ts @@ -0,0 +1,3 @@ +import { nightUncertainty } from './uncertainty' +import { createNightSummary } from './night-summary-core' +export const summarizeStageNights = createNightSummary(nightUncertainty) diff --git a/src/biometrics/backtest/numeric-comparison.ts b/src/biometrics/backtest/numeric-comparison.ts new file mode 100644 index 000000000..87e33ede7 --- /dev/null +++ b/src/biometrics/backtest/numeric-comparison.ts @@ -0,0 +1,59 @@ +import { alignReferences, type AlignmentPolicy, type Observation } from '../references/alignment' +import type { NormalizedMeasurement, ReferenceMetric } from '../references/contracts' + +type Label = 'default' | 'current' | 'candidate' +interface Output { label: Label, configHash: string, observations: Observation[] } +const summary = (score: ReturnType) => ({ status: score.status, metrics: score.metrics, + pairedCount: score.pairedCount, referenceCoverage: score.referenceCoverage, validObservationCount: score.validObservationCount, + unmatchedReferences: score.unmatchedReferences, unmatchedObservations: score.unmatchedObservations }) +const difference = (candidate: number | null, baseline: number | null) => candidate === null || baseline === null ? null : candidate - baseline + +/** Equal timestamps, not nearest output-to-output matching. All configurations + * contribute to the intersection before reference alignment, so dropping a hard + * sample cannot change an interval's aggregation support in only one result. + * Duplicate timestamps are ambiguous and excluded from common support. Known + * durations are deliberately not inferred from point timestamps. + */ +export function compareNumericConfigurations(metric: ReferenceMetric, windowMs: number | null, outputs: Output[], references: NormalizedMeasurement[], policy: AlignmentPolicy) { + if (outputs.length < 2 || outputs.length > 3 || new Set(outputs.map(output => output.label)).size !== outputs.length) throw new Error('Invalid numeric comparison configurations') + const own = outputs.map(output => alignReferences(metric, windowMs, output.observations, references, policy)) + const indexes = outputs.map((output) => { + const values = new Map(), seen = new Set(), ambiguous = new Set() + for (const row of output.observations) { + if (seen.has(row.timestampMs)) { + ambiguous.add(row.timestampMs) + values.delete(row.timestampMs) + } + seen.add(row.timestampMs) + if (!ambiguous.has(row.timestampMs) && row.value !== null) values.set(row.timestampMs, row.value) + } + return { values, ambiguous } + }) + const timestamps = [...indexes[0].values.keys()].filter(timestamp => indexes.every(index => index.values.has(timestamp))).sort((a, b) => a - b) + const common = indexes.map(index => alignReferences(metric, windowMs, + timestamps.map(timestampMs => ({ timestampMs, value: index.values.get(timestampMs) ?? null })), references, policy)) + return outputs.flatMap((candidate, c) => outputs.flatMap((baseline, b) => { + if (!((candidate.label === 'candidate' && baseline.label !== 'candidate') || (candidate.label === 'current' && baseline.label === 'default'))) return [] + const lost = [...indexes[b].values.keys()].filter(timestamp => !indexes[c].values.has(timestamp)).length + const added = [...indexes[c].values.keys()].filter(timestamp => !indexes[b].values.has(timestamp)).length + return [{ version: 'paired-numeric-support-v1' as const, metric, + baseline: baseline.label, candidate: candidate.label, baselineHash: baseline.configHash, candidateHash: candidate.configHash, + commonSupportScope: 'all_configurations' as const, supportDefinition: 'identical_unambiguous_observation_timestamps' as const, + commonObservationCount: timestamps.length, + ownSupport: { baseline: summary(own[b]), candidate: summary(own[c]) }, + commonSupport: { baseline: summary(common[b]), candidate: summary(common[c]) }, + differences: { + commonMae: difference(common[c].metrics?.mae ?? null, common[b].metrics?.mae ?? null), + commonMedianAbsoluteError: difference(common[c].metrics?.medianAbsoluteError ?? null, common[b].metrics?.medianAbsoluteError ?? null), + commonP95AbsoluteError: difference(common[c].metrics?.p95AbsoluteError ?? null, common[b].metrics?.p95AbsoluteError ?? null), + commonSignedBias: difference(common[c].metrics?.signedBias ?? null, common[b].metrics?.signedBias ?? null), + ownMae: difference(own[c].metrics?.mae ?? null, own[b].metrics?.mae ?? null), + referenceCoverage: difference(own[c].referenceCoverage, own[b].referenceCoverage), + }, + lostObservationTimestamps: lost, addedObservationTimestamps: added, + ambiguousObservationTimestamps: { baseline: indexes[b].ambiguous.size, candidate: indexes[c].ambiguous.size }, + coverageReduced: lost > 0 || own[c].pairedCount < own[b].pairedCount, + interpretation: 'descriptive_numeric_agreement_not_validated_improvement' as const, + }] + })) +} diff --git a/src/biometrics/backtest/numeric-night-summary.ts b/src/biometrics/backtest/numeric-night-summary.ts new file mode 100644 index 000000000..09eee3787 --- /dev/null +++ b/src/biometrics/backtest/numeric-night-summary.ts @@ -0,0 +1,83 @@ +import type { compareNumericConfigurations } from './numeric-comparison' +import type { createNightUncertainty } from './uncertainty-core' +import type { NightBootstrapPolicy } from './uncertainty-policy' + +type Comparison = ReturnType[number] +interface Window { startMs: number, endMs: number, metrics: Comparison[] } +interface Night { cohortId: string, numericComparisons?: Window[] } +const mean = (values: number[]) => values.length ? values.reduce((sum, value) => sum + value, 0) / values.length : null +function pooled(rows: Comparison[], side: 'baseline' | 'candidate') { + let pairedCount = 0, absoluteError = 0, signedError = 0 + for (const row of rows) { + const score = row.commonSupport[side] + if (score.metrics && score.pairedCount) { + pairedCount += score.pairedCount + absoluteError += score.metrics.mae * score.pairedCount + signedError += score.metrics.signedBias * score.pairedCount + } + } + // Quantiles cannot be pooled from window quantiles; deliberately omit them. + return { pairedCount, mae: pairedCount ? absoluteError / pairedCount : null, signedBias: pairedCount ? signedError / pairedCount : null } +} + +/** Weight references within each whole night, then weight nights equally. */ +export function createNumericNightSummary(uncertainty: ReturnType) { + return function summarize(nights: Night[], policy?: NightBootstrapPolicy) { + if (nights.length > 60 || new Set(nights.map(night => night.cohortId)).size !== nights.length) throw new Error('Invalid numeric night cohorts') + const groups = new Map + nights: Array<{ + cohortId: string + commonSupport: { baseline: ReturnType, candidate: ReturnType } + commonMaeDifference: number | null + coverageReduced: boolean + lostObservationTimestamps: number + addedObservationTimestamps: number + }> + rows: Comparison[] + }>() + for (const night of nights) { + const windows = [...(night.numericComparisons ?? [])].sort((a, b) => a.startMs - b.startMs) + if (windows.length > 60 || windows.some((window, i) => !Number.isSafeInteger(window.startMs) || !Number.isSafeInteger(window.endMs) + || window.endMs <= window.startMs || window.endMs - window.startMs > 86400000 || (i > 0 && windows[i - 1].endMs > window.startMs))) throw new Error('Invalid numeric night windows') + const local = new Map() + for (const window of windows) { + if (window.metrics.length > 9) throw new Error('Too many numeric comparisons') + const seen = new Set() + for (const row of window.metrics) { + const identity = { metric: row.metric, baseline: row.baseline, candidate: row.candidate, baselineHash: row.baselineHash, candidateHash: row.candidateHash } + const key = JSON.stringify(identity) + if (seen.has(key)) throw new Error('Duplicate numeric comparison') + if (row.commonSupport.baseline.pairedCount !== row.commonSupport.candidate.pairedCount) throw new Error('Unequal common reference support') + seen.add(key) + const rows = local.get(key) ?? [] + rows.push(row) + local.set(key, rows) + if (!groups.has(key)) groups.set(key, { identity, nights: [], rows: [] }) + } + } + for (const [key, rows] of local) { + const group = groups.get(key) + if (!group) throw new Error('Missing numeric comparison group') + const baseline = pooled(rows, 'baseline'), candidate = pooled(rows, 'candidate') + group.rows.push(...rows) + group.nights.push({ cohortId: night.cohortId, commonSupport: { baseline, candidate }, + commonMaeDifference: baseline.mae === null || candidate.mae === null ? null : candidate.mae - baseline.mae, + coverageReduced: rows.some(row => row.coverageReduced), + lostObservationTimestamps: rows.reduce((sum, row) => sum + row.lostObservationTimestamps, 0), + addedObservationTimestamps: rows.reduce((sum, row) => sum + row.addedObservationTimestamps, 0) }) + } + } + return [...groups.values()].map((group) => { + const values = group.nights.flatMap(night => night.commonMaeDifference === null ? [] : [night.commonMaeDifference]) + return { ...group.identity, version: 'numeric-night-summary-v1' as const, nights: group.nights, + totalNights: group.nights.length, scoredNights: values.length, unscoredNights: group.nights.length - values.length, + improvedErrorNights: values.filter(value => value < 0).length, regressedErrorNights: values.filter(value => value > 0).length, + unchangedErrorNights: values.filter(value => value === 0).length, coverageReducedNights: group.nights.filter(night => night.coverageReduced).length, + equalNightMeanMaeDifference: mean(values), + pooledCommonSupport: { baseline: pooled(group.rows, 'baseline'), candidate: pooled(group.rows, 'candidate') }, + uncertainty: uncertainty(group.nights.map(night => ({ cohortId: night.cohortId, difference: night.commonMaeDifference })), policy), + interpretation: 'descriptive_numeric_agreement_not_validated_improvement' as const } + }) + } +} diff --git a/src/biometrics/backtest/numeric-subgroups.ts b/src/biometrics/backtest/numeric-subgroups.ts new file mode 100644 index 000000000..a2637b770 --- /dev/null +++ b/src/biometrics/backtest/numeric-subgroups.ts @@ -0,0 +1,44 @@ +import type { NormalizedMeasurement } from '../references/contracts' + +/** Fixed reference-defined groups; never select membership using candidate + * output, error, quality, or coverage. Interval records are never split. */ +export const numericSubgroupNames = ['reference_points', 'reference_intervals', 'window_first_half', 'window_second_half'] as const +export type NumericSubgroupName = typeof numericSubgroupNames[number] +export function subgroupReferences(name: NumericSubgroupName, rows: NormalizedMeasurement[], window: { startMs: number, endMs: number }) { + const middle = window.startMs + (window.endMs - window.startMs) / 2 + return rows.filter(row => name === 'reference_points' + ? row.temporalKind === 'point' + : name === 'reference_intervals' + ? row.temporalKind === 'interval' + : name === 'window_first_half' + ? (row.temporalKind === 'point' ? row.startMs < middle : row.endMs <= middle) + : row.startMs >= middle) +} + +export function subgroupBoundaryExclusions(name: NumericSubgroupName, rows: NormalizedMeasurement[], window: { startMs: number, endMs: number }) { + if (name !== 'window_first_half' && name !== 'window_second_half') return 0 + const middle = window.startMs + (window.endMs - window.startMs) / 2 + return rows.filter(row => row.temporalKind === 'interval' && row.startMs < middle && row.endMs > middle).length +} + +interface Identity { + baseline: string + candidate: string + metric: string + baselineHash: string + candidateHash: string +} +interface SubgroupSummary extends Identity { + name: string + nights: Array<{ commonMaeDifference: number | null, commonSupport: { candidate: { pairedCount: number } } }> +} +export function subgroupObjectiveEvidence(summaries: SubgroupSummary[], identity: Identity, minimumPairs: number) { + return summaries.filter(row => row.baseline === identity.baseline && row.candidate === identity.candidate && row.metric === identity.metric + && row.baselineHash === identity.baselineHash && row.candidateHash === identity.candidateHash).map((row) => { + const supported = row.nights.filter(night => night.commonMaeDifference !== null && night.commonSupport.candidate.pairedCount >= minimumPairs) + return { name: row.name, pairedNights: supported.length, + maeDifference: supported.length > 0 && supported.length === row.nights.length + ? supported.reduce((sum, night) => sum + (night.commonMaeDifference ?? 0), 0) / supported.length + : null } + }) +} diff --git a/src/biometrics/backtest/numeric-trace-page.ts b/src/biometrics/backtest/numeric-trace-page.ts new file mode 100644 index 000000000..7530a7a2c --- /dev/null +++ b/src/biometrics/backtest/numeric-trace-page.ts @@ -0,0 +1,57 @@ +import { z } from 'zod' +import { ReferenceInputError } from '../references/time' +import { traceRequestSchema } from './trace-contracts' +import { waveformReadingSchema } from './waveform-output' + +const reading = waveformReadingSchema.pick({ timestampMs: true, heartRate: true, breathingRate: true, hrvIndex: true, + hrvMetric: true, qualityScore: true, flags: true, sourceSide: true, configHash: true, side: true, limitations: true }).strip() +export const numericTracePageSchema = z.object({ jobId: z.string().uuid(), contentHash: z.string(), + configuration: z.enum(['default', 'current', 'candidate']), side: z.enum(['left', 'right']), + provenance: z.literal('retained_waveform_recomputation'), timestampBasis: z.literal('original_trace_before_reference_clock_offset'), clockOffsetMs: z.number(), + totalItems: z.number().int(), offset: z.number().int(), limit: z.number().int(), downsampled: z.literal(false), + items: z.array(reading.extend({ cohortId: z.string(), revisionId: z.string().uuid().nullable() })).max(500), + nextCursor: z.object({ offset: z.number().int(), contentHash: z.string() }).nullable(), +}) +const reportSchema = z.object({ version: z.literal('waveform-backtest-report-v1'), side: z.enum(['left', 'right']), + protocol: z.object({ mode: z.literal('signal_replay'), clockOffsetMs: z.number().int() }), + waveform: z.object({ outputs: z.array(z.object({ cohortId: z.string(), configHash: z.string(), readings: z.array(reading).max(20000) })).max(180) }), + nights: z.array(z.object({ cohortId: z.string(), evaluations: z.array(z.object({ label: z.enum(['default', 'current', 'candidate']), + configHash: z.string(), revisionId: z.string().uuid().nullable(), + windows: z.array(z.object({ startMs: z.number().int(), endMs: z.number().int() })).max(60), + })).max(3) })).max(60), +}).refine(report => report.waveform.outputs.reduce((sum, output) => sum + output.readings.length, 0) <= 20000 + && new Set(report.nights.map(night => night.cohortId)).size === report.nights.length) + +/** Exact retained numeric rows for the dataset side only. Cross-side routing + * provenance is retained, but opposite-side observations are never returned. */ +export function pageNumericTraces(report: unknown, raw: z.input) { + const input = traceRequestSchema.parse(raw) + const parsed = reportSchema.safeParse(report) + if (!parsed.success) throw new ReferenceInputError('RESULT_CORRUPT', 'Numeric trace format or side provenance is unavailable') + const data = parsed.data + const nights = data.nights.filter(night => input.cohortId === undefined || night.cohortId === input.cohortId) + const items: z.infer['items'] = [] + let totalItems = 0, selections = 0 + for (const night of nights) { + const evaluations = night.evaluations.filter(row => row.label === input.configuration) + if (evaluations.length > 1) throw new ReferenceInputError('RESULT_CORRUPT', 'Duplicate numeric configuration') + for (const evaluation of evaluations) { + selections++ + const outputs = data.waveform.outputs.filter(output => output.cohortId === night.cohortId && output.configHash === evaluation.configHash) + if (outputs.length !== 1) throw new ReferenceInputError('RESULT_CORRUPT', 'Numeric trace output identity is ambiguous or missing') + for (const row of outputs[0].readings) { + if (row.configHash !== evaluation.configHash && row.side === data.side) throw new ReferenceInputError('RESULT_CORRUPT', 'Numeric reading configuration mismatch') + if (row.side !== data.side || !evaluation.windows.some(window => row.timestampMs >= window.startMs && row.timestampMs < window.endMs)) continue + if (totalItems >= input.offset && items.length < input.limit) items.push({ ...row, cohortId: night.cohortId, revisionId: evaluation.revisionId }) + totalItems++ + } + } + } + if (!selections) throw new ReferenceInputError('NOT_FOUND', 'Requested cohort or configuration was not evaluated') + if (input.offset > totalItems) throw new ReferenceInputError('INVALID_CURSOR', 'Trace cursor exceeds the selected report') + const next = input.offset + items.length + return numericTracePageSchema.parse({ jobId: input.id, contentHash: input.expectedContentHash, configuration: input.configuration, side: data.side, + provenance: 'retained_waveform_recomputation', timestampBasis: 'original_trace_before_reference_clock_offset', clockOffsetMs: data.protocol.clockOffsetMs, + totalItems, offset: input.offset, limit: input.limit, items, downsampled: false, + nextCursor: next < totalItems ? { offset: next, contentHash: input.expectedContentHash } : null }) +} diff --git a/src/biometrics/backtest/objective-finalization.ts b/src/biometrics/backtest/objective-finalization.ts new file mode 100644 index 000000000..0ec983035 --- /dev/null +++ b/src/biometrics/backtest/objective-finalization.ts @@ -0,0 +1,62 @@ +import { assessStageObjective, stageObjectivePolicySchema, stageObjectiveEvidenceSchema, type StageObjectivePolicy } from './stage-objective' +import { z } from 'zod' +import { jobOperationsSchema } from './job-operations' +import { assessNumericObjective, objectiveEvidenceSchema, objectivePolicySchema, type ObjectivePolicy, type ObjectiveEvidence } from './objective-policy' + +/** Convert supervisor measurements only when this frozen policy explicitly + * permits sampled RSS. Legacy peak policies do not silently change meaning. */ +export function objectiveOperations(raw: unknown, policy: Pick): ObjectiveEvidence['operational'] { + if (!policy.operational.observations) return null + const parsed = jobOperationsSchema.safeParse(raw) + if (!parsed.success) return null + const { wallMs, ingestion, memory } = parsed.data + if (!memory || !memory.allSamplesAvailable || memory.samples < 2 || memory.samples !== memory.attempts || memory.peakObservedRssBytes === null + || !ingestion.allChecksMeasured || ingestion.samples < 2 || ingestion.samples !== ingestion.checks + || ingestion.maximumObservedAgeMs === null || ingestion.qualificationId === null) return null + return { wallMs, peakRssBytes: memory.peakObservedRssBytes, maximumIngestionAgeMs: ingestion.maximumObservedAgeMs, + observations: { version: 'sampled-process-group-v1', qualificationId: ingestion.qualificationId, + maximumMemoryGapMs: memory.maximumObservationGapMs, maximumIngestionGapMs: ingestion.maximumObservationGapMs } } +} +const assessment = z.object({ version: z.literal('numeric-objective-assessment-v1'), policy: objectivePolicySchema, + evidence: objectiveEvidenceSchema, baseline: z.enum(['default', 'current']), candidate: z.literal('candidate'), + baselineHash: z.string(), candidateHash: z.string() }) + +/** Called after final admission, before immutable report publication. Operational + * evidence from the scoring child is replaced, never trusted as host telemetry. */ +export function finalizeObjectiveReport(raw: unknown, policy: ObjectivePolicy | readonly ObjectivePolicy[] | undefined, operations: unknown, designatedPolicy: ObjectivePolicy | readonly ObjectivePolicy[] | null) { + const report = z.record(z.string(), z.unknown()).parse(raw) + const policies = policy === undefined ? [] : Array.isArray(policy) ? policy : [policy] + if (!policies.length) return report + const parsedPolicies = policies.map(item => objectivePolicySchema.parse(item)) + if (new Set(parsedPolicies.map(item => item.metric)).size !== parsedPolicies.length) throw new Error('Duplicate numeric objective metric') + const designations = designatedPolicy === null ? [] : Array.isArray(designatedPolicy) ? designatedPolicy : [designatedPolicy] + const parsedDesignations = designations.map(item => objectivePolicySchema.parse(item)) + if (new Set(parsedDesignations.map(item => item.metric)).size !== parsedDesignations.length) throw new Error('Duplicate numeric policy designation') + const assessments = z.array(assessment).length(parsedPolicies.length * 2).parse(report.objectiveAssessments) + const identities = new Set() + const finalized = assessments.map((row) => { + const selected = parsedPolicies.find(item => item.metric === row.policy.metric) + const identity = `${row.policy.metric}:${row.baseline}` + if (!selected || identities.has(identity) || JSON.stringify(row.policy) !== JSON.stringify(selected)) throw new Error('Objective report identity mismatch') + identities.add(identity) + return { baseline: row.baseline, candidate: row.candidate, baselineHash: row.baselineHash, candidateHash: row.candidateHash, + ...assessNumericObjective(selected, { ...row.evidence, operational: objectiveOperations(operations, selected) }, + parsedDesignations.find(item => item.metric === selected.metric) ?? null) } + }) + return { ...report, objectiveAssessments: finalized } +} + +const stageAssessment = z.object({ version: z.literal('stage-objective-assessment-v1'), policy: stageObjectivePolicySchema, + evidence: stageObjectiveEvidenceSchema, baseline: z.enum(['default', 'current']), candidate: z.literal('candidate'), baselineHash: z.string(), candidateHash: z.string() }) +export function finalizeStageObjectiveReport(raw: unknown, policy: StageObjectivePolicy | undefined, operations: unknown, designation: StageObjectivePolicy | null) { + const report = z.record(z.string(), z.unknown()).parse(raw) + if (!policy) return report + const assessments = z.array(stageAssessment).length(2).parse(report.stageObjectiveAssessments) + if (new Set(assessments.map(row => row.baseline)).size !== 2 + || assessments.some(row => JSON.stringify(row.policy) !== JSON.stringify(stageObjectivePolicySchema.parse(policy)))) throw new Error('Stage objective report identity mismatch') + const operational = objectiveOperations(operations, policy) + return { ...report, stageObjectiveAssessments: assessments.map(row => ({ baseline: row.baseline, candidate: row.candidate, + baselineHash: row.baselineHash, candidateHash: row.candidateHash, + ...assessStageObjective(policy, { ...row.evidence, operational }, designation), + })) } +} diff --git a/src/biometrics/backtest/objective-policy.ts b/src/biometrics/backtest/objective-policy.ts new file mode 100644 index 000000000..db7b26746 --- /dev/null +++ b/src/biometrics/backtest/objective-policy.ts @@ -0,0 +1,106 @@ +import { z } from 'zod' + +const nonnegative = z.number().finite().nonnegative() +export const operationalPolicySchema = z.object({ maximumWallMs: z.number().int().positive(), maximumPeakRssBytes: z.number().int().positive(), + maximumIngestionAgeMs: z.number().int().positive(), + observations: z.object({ version: z.literal('sampled-process-group-v1'), + maximumMemoryGapMs: z.number().int().positive(), maximumIngestionGapMs: z.number().int().positive(), + }).strict().optional(), +}).strict() + +/** Frozen with the experiment. Supplying thresholds never designates this as a + * release promotion policy; that authority belongs to the qualified build. */ +export const objectivePolicySchema = z.object({ + version: z.literal('numeric-objective-v1'), + qualificationId: z.string().min(1).max(200), + metric: z.enum(['heart_rate', 'breathing_rate', 'window_ibi_variability_v1']), + minimumNights: z.number().int().min(3).max(60), + minimumPairedReferencesPerNight: z.number().int().min(1).max(100000), + maximumLostObservationFraction: z.number().min(0).max(1), + minimumMaeImprovement: z.number().finite().positive(), + maximumNightMaeRegression: nonnegative, + maximumRegressedNightFraction: z.number().min(0).max(1), + maximumSubgroupMaeRegression: nonnegative, + requiredSubgroups: z.array(z.string().min(1).max(100)).max(20), + operational: operationalPolicySchema, +}).strict().refine(policy => new Set(policy.requiredSubgroups).size === policy.requiredSubgroups.length, 'Duplicate objective subgroups') +export type ObjectivePolicy = z.infer + +export const operationalEvidenceSchema = z.object({ wallMs: nonnegative, peakRssBytes: nonnegative, maximumIngestionAgeMs: nonnegative, + observations: z.object({ version: z.literal('sampled-process-group-v1'), qualificationId: z.string(), + maximumMemoryGapMs: nonnegative, maximumIngestionGapMs: nonnegative, + }).strict().optional(), +}).strict() + +const nightSchema = z.object({ cohortId: z.string(), pairedReferences: z.number().int().nonnegative(), + maeDifference: z.number().finite().nullable(), baselineObservationCount: z.number().int().nonnegative(), lostObservations: z.number().int().nonnegative() }).strict() +export const objectiveEvidenceSchema = z.object({ + metricCompatible: z.boolean(), split: z.enum(['tuning', 'validation', 'holdout']), holdoutUse: z.number().int().nonnegative(), + nights: z.array(nightSchema).max(60), + intervalUpper: z.number().finite().nullable(), + subgroups: z.array(z.object({ name: z.string(), pairedNights: z.number().int().nonnegative(), maeDifference: z.number().finite().nullable() }).strict()).max(20), + operational: operationalEvidenceSchema.nullable(), +}).strict() +export type ObjectiveEvidence = z.infer + +/** Pure assessment. Designation is supplied by a trusted build dependency, not + * experiment JSON. No clinical accuracy or activation is inferred. */ +export function assessNumericObjective(rawPolicy: ObjectivePolicy, rawEvidence: ObjectiveEvidence, designatedPolicy: ObjectivePolicy | null) { + const policy = objectivePolicySchema.parse(rawPolicy), evidence = objectiveEvidenceSchema.parse(rawEvidence) + if (new Set(evidence.nights.map(night => night.cohortId)).size !== evidence.nights.length + || new Set(evidence.subgroups.map(group => group.name)).size !== evidence.subgroups.length + || evidence.nights.some(night => night.lostObservations > night.baselineObservationCount)) throw new Error('Invalid objective evidence identity or coverage') + const designated = designatedPolicy !== null && JSON.stringify(objectivePolicySchema.parse(designatedPolicy)) === JSON.stringify(policy) + const missing: string[] = [], failures: string[] = [] + if (!designated) missing.push('policy_not_designated') + if (evidence.split !== 'holdout' || evidence.holdoutUse !== 1) missing.push('independent_holdout_required') + const scored = evidence.nights.filter(night => night.maeDifference !== null && night.pairedReferences >= policy.minimumPairedReferencesPerNight) + if (scored.length < policy.minimumNights) missing.push('insufficient_supported_nights') + if (scored.length !== evidence.nights.length) missing.push('unsupported_nights') + if (evidence.intervalUpper === null) missing.push('uncertainty_unavailable') + const operational = assessOperationalEvidence(policy.qualificationId, policy.operational, evidence.operational) + missing.push(...operational.missing) + failures.push(...operational.failures) + if (evidence.nights.some(night => night.baselineObservationCount === 0)) missing.push('coverage_unavailable') + if (evidence.nights.some(night => night.baselineObservationCount > 0 && night.lostObservations / night.baselineObservationCount > policy.maximumLostObservationFraction)) failures.push('coverage_loss') + const deltas = scored.map(night => night.maeDifference ?? 0) + if (deltas.some(delta => delta > policy.maximumNightMaeRegression)) failures.push('night_regression') + if (deltas.length && deltas.filter(delta => delta > 0).length / deltas.length > policy.maximumRegressedNightFraction) failures.push('regressed_night_fraction') + for (const name of policy.requiredSubgroups) { + const group = evidence.subgroups.find(row => row.name === name) + if (!group || group.maeDifference === null || group.pairedNights < policy.minimumNights) missing.push(`subgroup_support:${name}`) + else if (group.maeDifference > policy.maximumSubgroupMaeRegression) failures.push(`subgroup_regression:${name}`) + } + const estimate = deltas.length ? deltas.reduce((sum, delta) => sum + delta, 0) / deltas.length : null + const status = !evidence.metricCompatible + ? 'incompatible' as const + : missing.length + ? 'insufficient_evidence' as const + : failures.length + ? 'regressed' as const + : estimate !== null && estimate <= -policy.minimumMaeImprovement && evidence.intervalUpper !== null && evidence.intervalUpper <= -policy.minimumMaeImprovement + ? 'improved' as const + : 'no_material_change' as const + return { version: 'numeric-objective-assessment-v1' as const, policy, evidence, designated, status, missingEvidence: missing, failedThresholds: failures, + equalNightMeanMaeDifference: estimate, intervalUpper: evidence.intervalUpper, + interpretation: 'agreement_with_named_reference_not_clinical_validation' as const } +} + +export function assessOperationalEvidence(qualificationId: string, policy: z.infer, evidence: z.infer | null) { + const missing: string[] = [], failures: string[] = [] + if (!evidence) missing.push('operational_evidence_unavailable') + else { + const required = policy.observations, observed = evidence.observations + if (required && !observed) missing.push('operational_sampling_evidence_unavailable') + if (!required && observed) missing.push('sampled_memory_not_authorized_by_policy') + if (required && observed) { + if (observed.qualificationId !== qualificationId) missing.push('operational_qualification_mismatch') + if (observed.maximumMemoryGapMs > required.maximumMemoryGapMs) missing.push('memory_sampling_gap') + if (observed.maximumIngestionGapMs > required.maximumIngestionGapMs) missing.push('ingestion_sampling_gap') + } + if (evidence.wallMs > policy.maximumWallMs) failures.push('wall_time_limit') + if (evidence.peakRssBytes > policy.maximumPeakRssBytes) failures.push('memory_limit') + if (evidence.maximumIngestionAgeMs > policy.maximumIngestionAgeMs) failures.push('ingestion_limit') + } + return { missing, failures } +} diff --git a/src/biometrics/backtest/process-group-memory-contract.ts b/src/biometrics/backtest/process-group-memory-contract.ts new file mode 100644 index 000000000..bf4ead7a9 --- /dev/null +++ b/src/biometrics/backtest/process-group-memory-contract.ts @@ -0,0 +1,12 @@ +import { z } from 'zod' + +export const processGroupMemorySchema = z.object({ + version: z.literal('process-group-rss-observations-v1'), + scope: z.literal('sum_rss_of_replay_process_group'), + attempts: z.number().int().nonnegative(), samples: z.number().int().nonnegative(), + allSamplesAvailable: z.boolean(), peakObservedRssBytes: z.number().int().positive().nullable(), + maximumObservationGapMs: z.number().finite().nonnegative(), intervalMs: z.literal(1000), + exactKernelPeak: z.literal(false), +}).strict() +export type ProcessGroupMemory = z.infer + diff --git a/src/biometrics/backtest/process-group-memory.ts b/src/biometrics/backtest/process-group-memory.ts new file mode 100644 index 000000000..71de1b61e --- /dev/null +++ b/src/biometrics/backtest/process-group-memory.ts @@ -0,0 +1,82 @@ +import { readdir, readFile } from 'node:fs/promises' +import { join } from 'node:path' +import type { ProcessGroupMemory } from './process-group-memory-contract' + +/** Fixed procfs reads only; never inspect argv, environment, or other exports. + * Shared pages may be counted in more than one process's RSS. */ +export async function sampleProcessGroupRss(group: number, root = '/proc'): Promise { + if (!Number.isSafeInteger(group) || group < 1) return null + try { + const pids = (await readdir(/* turbopackIgnore: true */ root)).filter(name => /^[1-9]\d*$/.test(name)) + if (pids.length > 4096) return null + let total = 0, members = 0 + for (let i = 0; i < pids.length; i += 16) { + const values = await Promise.all(pids.slice(i, i + 16).map(async (pid) => { + try { + const stat = await readFile(join(root, pid, 'stat'), 'utf8') + if (stat.length > 16384) throw new Error('Invalid proc stat') + const end = stat.lastIndexOf(')') + if (end < 0) throw new Error('Invalid proc stat') + const fields = stat.slice(end + 1).trim().split(/\s+/) + if (!/^\d+$/.test(fields[2] ?? '')) throw new Error('Missing process group') + if (Number(fields[2]) !== group) return 0 + const status = await readFile(join(root, pid, 'status'), 'utf8') + if (status.length > 16384) throw new Error('Invalid proc status') + const rss = /^VmRSS:\s+(\d+)\s+kB\s*$/m.exec(status) + // Zombies have no address space and no VmRSS entry. + if (!rss && fields[0] === 'Z') return 0 + if (!rss) throw new Error('Missing process RSS') + members++ + const bytes = Number(rss[1]) * 1024 + if (!Number.isSafeInteger(bytes) || bytes < 0) throw new Error('Invalid process RSS') + return bytes + } + catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT' || (error as NodeJS.ErrnoException).code === 'ESRCH') return 0 + throw error + } + })) + total += values.reduce((sum, value) => sum + value, 0) + if (members > 64 || !Number.isSafeInteger(total)) return null + } + return members > 0 && total > 0 ? total : null + } + catch { return null } +} + +export function monitorProcessGroupMemory(group: number, options: { sample?: () => Promise, now?: () => number } = {}) { + const now = options.now ?? (() => performance.now()) + const read = options.sample ?? (() => process.platform === 'linux' ? sampleProcessGroupRss(group) : Promise.resolve(null)) + const started = now() + let last = started, gap = 0, attempts = 0, samples = 0, peak = 0, stopped = false + let active: Promise | null = null + const sample = () => { + if (stopped) return Promise.resolve() + active ??= Promise.resolve().then(read).catch(() => null).then((rss) => { + const at = now() + gap = Math.max(gap, at - last) + last = at + attempts++ + if (rss !== null && Number.isSafeInteger(rss) && rss > 0) { + samples++ + peak = Math.max(peak, rss) + } + }).finally(() => { active = null }) + return active + } + const timer = setInterval(() => { + void sample() + }, 1000) + timer.unref() + void sample() + return { sample, + snapshot: (): ProcessGroupMemory => ({ version: 'process-group-rss-observations-v1', scope: 'sum_rss_of_replay_process_group', + attempts, samples, allSamplesAvailable: samples > 0 && samples === attempts, peakObservedRssBytes: samples ? peak : null, + maximumObservationGapMs: Math.max(gap, now() - last), intervalMs: 1000, exactKernelPeak: false }), + async stop() { + stopped = true + clearInterval(timer) + await active + }, + } +} diff --git a/src/biometrics/backtest/raw-catalog.ts b/src/biometrics/backtest/raw-catalog.ts new file mode 100644 index 000000000..fb8051755 --- /dev/null +++ b/src/biometrics/backtest/raw-catalog.ts @@ -0,0 +1,106 @@ +import { and, asc, eq, gt, gte, isNull, lt, ne } from 'drizzle-orm' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import { z } from 'zod' +import * as schema from '@/src/db/biometrics-schema' +import { configHash } from '../algorithms/canonical' +import { ReferenceInputError } from '../references/time' + +const count = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) +const hash = z.string().regex(/^[a-f0-9]{64}$/) +export { rawArtifactDescriptorSchema } from './waveform-contracts' +import { rawArtifactDescriptorSchema } from './waveform-contracts' +const channel = z.object({ samples: count, lost_markers: count, truncated_tail_bytes: count }).strict() +export const rawInventorySchema = z.object({ + version: z.literal('retained-raw-inventory-v1'), artifact_sha256: hash, + decoded_records: count, piezo_records: count, pump_records: count, + usable_dual_frames: count, empty_channel_frames: count, oversize_channel_frames: count, + invalid_timestamps: count, timestamp_reversals: count, duplicate_timestamps: count, + first_event_ms: count.nullable(), last_event_ms: count.nullable(), + channels: z.object({ left1: channel, right1: channel }).strict(), + ordered_waveform_available: z.boolean(), limitations: z.array(z.string().max(500)).max(10), +}).strict().superRefine((value, context) => { + const invalid = (message: string) => context.addIssue({ code: 'custom', message }) + if ((value.first_event_ms === null) !== (value.last_event_ms === null) + || (value.first_event_ms !== null && value.last_event_ms !== null && value.first_event_ms > value.last_event_ms)) invalid('Invalid event bounds') + if (value.piezo_records + value.pump_records > value.decoded_records + || value.usable_dual_frames + value.empty_channel_frames + value.oversize_channel_frames !== value.piezo_records) invalid('Inconsistent record counts') + for (const totals of Object.values(value.channels)) { + if (totals.lost_markers > totals.samples) invalid('Lost markers exceed sample count') + } + const ordered = value.usable_dual_frames > 0 && value.invalid_timestamps === 0 + && value.timestamp_reversals === 0 && value.oversize_channel_frames === 0 + if (value.ordered_waveform_available !== ordered || (ordered && value.first_event_ms === null)) invalid('Inconsistent waveform availability') +}) +export const rawInspectionSchema = z.object({ + source_name: z.string().max(255).regex(/^[A-Za-z0-9_.-]+\.RAW\.gz$/), + retained: z.literal(false), artifact: rawArtifactDescriptorSchema, inventory: rawInventorySchema, +}).strict().refine(value => value.artifact.sha256 === value.inventory.artifact_sha256, 'Inventory must identify the inspected bytes') + +export const rawSourceDescriptorSchema = z.object({ + id: hash, inspection: rawInspectionSchema, inspectedAtMs: count, unavailableAtMs: count.nullable(), + availability: z.enum(['requires_reverification', 'unavailable']), +}).strict() +export const rawSourcePageResultSchema = z.object({ items: z.array(rawSourceDescriptorSchema).max(50), nextCursor: hash.nullable() }).strict() + +export const rawSourcePageSchema = z.object({ startMs: count, endMs: count, afterId: hash.optional() }).strict() + .refine(value => value.endMs > value.startMs, 'Expected a nonempty time window') + +/** Internal scan evidence only. This catalog neither retains bytes nor grants replay capability. */ +export class RawSourceCatalog { + constructor(private readonly db: BetterSQLite3Database, private readonly now = Date.now) {} + + record(raw: unknown) { + const input = rawInspectionSchema.parse(raw) + const id = configHash({ sourceName: input.source_name, artifactHash: input.artifact.sha256, inventoryVersion: input.inventory.version }) + const inspectedAtMs = count.parse(this.now()) + return this.db.transaction((tx) => { + const existing = tx.select().from(schema.biometricRawSources).where(eq(schema.biometricRawSources.id, id)).get() + if (existing && (configHash(existing.descriptor) !== configHash(input.artifact) + || configHash(existing.inventory) !== configHash(input.inventory))) { + throw new ReferenceInputError('SOURCE_CONFLICT', 'Identical source bytes and inventory version produced different inspection evidence') + } + // Replacement invalidates only previous identities of this filename. It + // never changes their frozen descriptor or any dataset's retained bytes. + tx.update(schema.biometricRawSources).set({ unavailableAtMs: inspectedAtMs }) + .where(and(eq(schema.biometricRawSources.sourceName, input.source_name), ne(schema.biometricRawSources.id, id), isNull(schema.biometricRawSources.unavailableAtMs))).run() + tx.insert(schema.biometricRawSources).values({ id, sourceName: input.source_name, + artifactHash: input.artifact.sha256, descriptor: input.artifact, inventoryVersion: input.inventory.version, + inventory: input.inventory, firstEventMs: input.inventory.first_event_ms, lastEventMs: input.inventory.last_event_ms, + inspectedAtMs, unavailableAtMs: null, + }).onConflictDoUpdate({ target: schema.biometricRawSources.id, set: { inspectedAtMs, unavailableAtMs: null } }).run() + return id + }, { behavior: 'immediate' }) + } + + markUnavailable(id: string) { + // Mark the failed inspected identity, not a filename that may now point at + // a newer successful scan. Existing tombstone timestamps remain stable. + this.db.update(schema.biometricRawSources).set({ unavailableAtMs: count.parse(this.now()) }) + .where(and(eq(schema.biometricRawSources.id, hash.parse(id)), isNull(schema.biometricRawSources.unavailableAtMs))).run() + } + + read(id: string) { + const row = this.db.select().from(schema.biometricRawSources).where(eq(schema.biometricRawSources.id, hash.parse(id))).get() + if (!row) throw new ReferenceInputError('NOT_FOUND', 'Inspected RAW source not found') + const inspection = rawInspectionSchema.parse({ source_name: row.sourceName, retained: false, artifact: row.descriptor, inventory: row.inventory }) + if (row.artifactHash !== inspection.artifact.sha256 || row.inventoryVersion !== inspection.inventory.version + || row.firstEventMs !== inspection.inventory.first_event_ms || row.lastEventMs !== inspection.inventory.last_event_ms + || row.id !== configHash({ sourceName: row.sourceName, artifactHash: row.artifactHash, inventoryVersion: row.inventoryVersion })) { + throw new ReferenceInputError('SOURCE_CORRUPT', 'Inspected RAW source integrity check failed') + } + return { id: row.id, inspection, inspectedAtMs: row.inspectedAtMs, unavailableAtMs: row.unavailableAtMs, + availability: row.unavailableAtMs === null ? 'requires_reverification' as const : 'unavailable' as const } + } + + page(raw: { startMs: number, endMs: number, afterId?: string }) { + const input = rawSourcePageSchema.parse(raw) + // Event bounds are inclusive; requested windows are half-open. Inclusion + // only indicates potential overlap, never continuous or complete coverage. + const rows = this.db.select({ id: schema.biometricRawSources.id }).from(schema.biometricRawSources) + .where(and(isNull(schema.biometricRawSources.unavailableAtMs), lt(schema.biometricRawSources.firstEventMs, input.endMs), + gte(schema.biometricRawSources.lastEventMs, input.startMs), input.afterId ? gt(schema.biometricRawSources.id, input.afterId) : undefined)) + .orderBy(asc(schema.biometricRawSources.id)).limit(51).all() + const items = rows.slice(0, 50).map(row => this.read(row.id)) + return { items, nextCursor: rows.length > 50 ? items[49].id : null } + } +} diff --git a/src/biometrics/backtest/raw-collection-child.ts b/src/biometrics/backtest/raw-collection-child.ts new file mode 100644 index 000000000..4941a17fa --- /dev/null +++ b/src/biometrics/backtest/raw-collection-child.ts @@ -0,0 +1,61 @@ +import { spawn } from 'node:child_process' +import { tmpdir } from 'node:os' +import { z } from 'zod' + +const resultSchema = z.object({ version: z.literal('raw-collection-result-v1'), released: z.number().int().min(0).max(10000), orphans: z.number().int().min(0).max(10000) }).strict() +interface RawCollectionLocation { pythonPath: string, modulesPath: string, databasePath: string, retainedDirectory: string } + +/** Trusted deployment paths only; no RPC arguments select files or alter retention grace. */ +export function runRawCollectionChild(location: RawCollectionLocation, options: { + signal?: AbortSignal + timeoutMs?: number +} = {}): Promise> { + const timeoutMs = options.timeoutMs ?? 120000 + if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 10 || timeoutMs > 120000) throw new Error('Invalid collection time budget') + if (options.signal?.aborted) return Promise.reject(new Error('RAW collection cancelled')) + return new Promise((resolve, reject) => { + const child = spawn(location.pythonPath, ['-B', '-m', 'common.biometrics.collection_worker', '--database-path', location.databasePath, '--retained-directory', location.retainedDirectory], { + cwd: tmpdir(), env: { NODE_ENV: 'production', PYTHONPATH: location.modulesPath, LANG: 'C.UTF-8', TZ: 'UTC', + OPENBLAS_NUM_THREADS: '1', OMP_NUM_THREADS: '1', MKL_NUM_THREADS: '1', NUMEXPR_NUM_THREADS: '1' }, + stdio: ['pipe', 'pipe', 'pipe'], + }) + const chunks: Buffer[] = [] + let outputBytes = 0, diagnosticBytes = 0 + let failure: Error | undefined, killTimer: ReturnType | undefined + const stop = (error: Error) => { + failure ??= error + child.kill('SIGTERM') + killTimer ??= setTimeout(() => child.kill('SIGKILL'), 500) + } + const timeout = setTimeout(() => stop(new Error('RAW collection exceeded its time budget')), timeoutMs) + const abort = () => stop(new Error('RAW collection cancelled')) + options.signal?.addEventListener('abort', abort, { once: true }) + if (options.signal?.aborted) abort() + child.on('error', () => stop(new Error('RAW collection worker could not start'))) + child.stdin.on('error', () => stop(new Error('RAW collection input pipe closed'))) + child.stderr.on('data', (chunk: Buffer) => { + diagnosticBytes += chunk.length + if (diagnosticBytes > 16384) stop(new Error('RAW collection diagnostics exceeded their bound')) + }) + child.stdout.on('data', (chunk: Buffer) => { + if (failure) return + outputBytes += chunk.length + if (outputBytes > 1024) return stop(new Error('RAW collection output exceeded its bound')) + chunks.push(chunk) + }) + child.on('close', (code) => { + clearTimeout(timeout) + if (killTimer) clearTimeout(killTimer) + options.signal?.removeEventListener('abort', abort) + if (failure) return reject(failure) + try { + if (code !== 0) throw new Error('Worker failed') + const text = new TextDecoder('utf-8', { fatal: true }).decode(Buffer.concat(chunks)) + const result = resultSchema.parse(JSON.parse(text)) + resolve(result) + } + catch { reject(new Error('RAW collection worker returned an invalid or incomplete result')) } + }) + child.stdin.end(JSON.stringify({ version: 'raw-collection-work-v1' })) + }) +} diff --git a/src/biometrics/backtest/raw-inspection-child.ts b/src/biometrics/backtest/raw-inspection-child.ts new file mode 100644 index 000000000..be58ba386 --- /dev/null +++ b/src/biometrics/backtest/raw-inspection-child.ts @@ -0,0 +1,81 @@ +import { spawn } from 'node:child_process' +import { tmpdir } from 'node:os' +import { z } from 'zod' +import { rawInspectionSchema } from './raw-catalog' + +export const rawSourceNameSchema = z.string().max(255).regex(/^[A-Za-z0-9_.-]+\.RAW\.gz$/) +export const rawScanInputSchema = z.object({ afterName: rawSourceNameSchema.nullable() }).strict() +const inspectionResultSchema = z.discriminatedUnion('status', [ + z.object({ status: z.literal('inspected'), nextCursor: rawSourceNameSchema, inspection: rawInspectionSchema }).strict(), + z.object({ status: z.literal('source_unavailable'), nextCursor: rawSourceNameSchema }).strict(), + z.object({ status: z.literal('archive_unavailable'), nextCursor: z.null() }).strict(), + z.object({ status: z.literal('complete'), nextCursor: z.null() }).strict(), +]) +export interface RawInspectionLocation { + pythonPath: string + modulesPath: string + archiveDirectory: string +} +export const podRawInspectionLocation: RawInspectionLocation = { + pythonPath: '/opt/sleepypod/modules/piezo-processor/.venv/bin/python', + modulesPath: '/opt/sleepypod/modules', + archiveDirectory: '/persistent/biometrics-archive', +} + +/** Trusted deployment paths only; callers supply a filename cursor, never code or paths. */ +export function runRawInspectionChild(location: RawInspectionLocation, raw: z.input, options: { + signal?: AbortSignal + timeoutMs?: number +} = {}): Promise> { + const input = rawScanInputSchema.parse(raw) + const timeoutMs = options.timeoutMs ?? 120000 + if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 10 || timeoutMs > 120000) throw new Error('Invalid inspection time budget') + if (options.signal?.aborted) return Promise.reject(new Error('RAW inspection cancelled')) + return new Promise((resolve, reject) => { + const child = spawn(location.pythonPath, ['-B', '-m', 'common.biometrics.inspection_worker', '--archive-directory', location.archiveDirectory], { + cwd: tmpdir(), env: { NODE_ENV: 'production', PYTHONPATH: location.modulesPath, LANG: 'C.UTF-8', TZ: 'UTC', + OPENBLAS_NUM_THREADS: '1', OMP_NUM_THREADS: '1', MKL_NUM_THREADS: '1', NUMEXPR_NUM_THREADS: '1' }, + stdio: ['pipe', 'pipe', 'pipe'], + }) + const chunks: Buffer[] = [] + let outputBytes = 0, diagnosticBytes = 0 + let failure: Error | undefined, killTimer: ReturnType | undefined + const stop = (error: Error) => { + failure ??= error + child.kill('SIGTERM') + killTimer ??= setTimeout(() => child.kill('SIGKILL'), 500) + } + const timeout = setTimeout(() => stop(new Error('RAW inspection exceeded its time budget')), timeoutMs) + const abort = () => stop(new Error('RAW inspection cancelled')) + options.signal?.addEventListener('abort', abort, { once: true }) + if (options.signal?.aborted) abort() + child.on('error', () => stop(new Error('RAW inspection worker could not start'))) + child.stdin.on('error', () => stop(new Error('RAW inspection input pipe closed'))) + child.stderr.on('data', (chunk: Buffer) => { + diagnosticBytes += chunk.length + if (diagnosticBytes > 16384) stop(new Error('RAW inspection diagnostics exceeded their bound')) + }) + child.stdout.on('data', (chunk: Buffer) => { + if (failure) return + outputBytes += chunk.length + if (outputBytes > 65536) return stop(new Error('RAW inspection output exceeded its bound')) + chunks.push(chunk) + }) + child.on('close', (code) => { + clearTimeout(timeout) + if (killTimer) clearTimeout(killTimer) + options.signal?.removeEventListener('abort', abort) + if (failure) return reject(failure) + try { + if (code !== 0) throw new Error('Worker failed') + const text = new TextDecoder('utf-8', { fatal: true }).decode(Buffer.concat(chunks)) + const result = inspectionResultSchema.parse(JSON.parse(text)) + if ((result.nextCursor !== null && input.afterName !== null && result.nextCursor <= input.afterName) + || (result.status === 'inspected' && result.nextCursor !== result.inspection.source_name)) throw new Error('Invalid cursor') + resolve(result) + } + catch { reject(new Error('RAW inspection worker returned an invalid or incomplete result')) } + }) + child.stdin.end(JSON.stringify({ version: 'raw-inspection-work-v1', after_name: input.afterName })) + }) +} diff --git a/src/biometrics/backtest/raw-publication-child.ts b/src/biometrics/backtest/raw-publication-child.ts new file mode 100644 index 000000000..0f6b6f649 --- /dev/null +++ b/src/biometrics/backtest/raw-publication-child.ts @@ -0,0 +1,70 @@ +import { spawn } from 'node:child_process' +import { tmpdir } from 'node:os' +import { z } from 'zod' +import type { RawInspectionLocation } from './raw-inspection-child' +import type { prepareSelectedWaveformDataset } from './waveform-selection' + +const publicationResultSchema = z.object({ type: z.literal('result'), id: z.string().uuid(), + contentHash: z.string().regex(/^[a-f0-9]{64}$/), available: z.boolean() }).strict() +export interface RawPublicationLocation extends RawInspectionLocation { + databasePath: string + retainedDirectory: string +} + +/** Only coordinator-prepared rows enter this fixed shipped module. No RPC paths or code. */ +export function runRawPublicationChild(location: RawPublicationLocation, work: ReturnType, options: { + signal?: AbortSignal + timeoutMs?: number +} = {}): Promise> { + const serialized = JSON.stringify({ version: 'waveform-publication-work-v1', ...work }) + if (Buffer.byteLength(serialized) > 10 * 1024 * 1024) return Promise.reject(new Error('Waveform publication input exceeds 10 MiB')) + const timeoutMs = options.timeoutMs ?? 120000 + if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 10 || timeoutMs > 120000) throw new Error('Invalid publication time budget') + if (options.signal?.aborted) return Promise.reject(new Error('Waveform publication cancelled')) + return new Promise((resolve, reject) => { + const child = spawn(location.pythonPath, ['-B', '-m', 'common.biometrics.publication_worker', '--database', location.databasePath, '--archive-directory', location.archiveDirectory, '--retained-directory', location.retainedDirectory], { + cwd: tmpdir(), env: { NODE_ENV: 'production', PYTHONPATH: location.modulesPath, LANG: 'C.UTF-8', TZ: 'UTC', + OPENBLAS_NUM_THREADS: '1', OMP_NUM_THREADS: '1', MKL_NUM_THREADS: '1', NUMEXPR_NUM_THREADS: '1' }, + stdio: ['pipe', 'pipe', 'pipe'], + }) + const chunks: Buffer[] = [] + let outputBytes = 0, diagnosticBytes = 0 + let failure: Error | undefined, killTimer: ReturnType | undefined + const stop = (error: Error) => { + failure ??= error + child.kill('SIGTERM') + killTimer ??= setTimeout(() => child.kill('SIGKILL'), 500) + } + const timeout = setTimeout(() => stop(new Error('Waveform publication exceeded its time budget')), timeoutMs) + const abort = () => stop(new Error('Waveform publication cancelled')) + options.signal?.addEventListener('abort', abort, { once: true }) + if (options.signal?.aborted) abort() + child.on('error', () => stop(new Error('Waveform publication worker could not start'))) + child.stdin.on('error', () => stop(new Error('Waveform publication input pipe closed'))) + child.stderr.on('data', (chunk: Buffer) => { + diagnosticBytes += chunk.length + if (diagnosticBytes > 16384) stop(new Error('Waveform publication diagnostics exceeded their bound')) + }) + child.stdout.on('data', (chunk: Buffer) => { + if (failure) return + outputBytes += chunk.length + if (outputBytes > 65536) return stop(new Error('Waveform publication output exceeded its bound')) + chunks.push(chunk) + }) + child.on('close', (code) => { + clearTimeout(timeout) + if (killTimer) clearTimeout(killTimer) + options.signal?.removeEventListener('abort', abort) + if (failure) return reject(failure) + try { + if (code !== 0) throw new Error('Worker failed') + const text = new TextDecoder('utf-8', { fatal: true }).decode(Buffer.concat(chunks)) + const result = publicationResultSchema.parse(JSON.parse(text)) + if (result.contentHash !== work.prepared.contentHash) throw new Error('Mismatched published dataset') + resolve(result) + } + catch { reject(new Error('Waveform publication worker returned an invalid or incomplete result')) } + }) + child.stdin.end(serialized) + }) +} diff --git a/src/biometrics/backtest/reference-trace-page.ts b/src/biometrics/backtest/reference-trace-page.ts new file mode 100644 index 000000000..9987a848f --- /dev/null +++ b/src/biometrics/backtest/reference-trace-page.ts @@ -0,0 +1,66 @@ +import { z } from 'zod' +import { traceRequestSchema } from './trace-contracts' +import { normalizedMeasurementSchema } from '../references/contracts' +import { referenceCompatibility } from '../references/alignment' +import { ReferenceInputError } from '../references/time' +import type { DatasetSnapshot } from './contracts' + +export const referenceTraceRequestSchema = traceRequestSchema.omit({ configuration: true }).extend({ + cohortId: z.string().min(1).max(200), metric: z.enum(['heart_rate', 'breathing_rate', 'window_ibi_variability_v1', 'sleep_stages']), +}) +const projected = normalizedMeasurementSchema.omit({ original: true }).strip().extend({ + importId: z.string(), importRevision: z.number().int(), importContentHash: z.string(), source: z.string(), + compatibility: z.string().nullable(), +}) +export const referenceTracePageSchema = z.object({ jobId: z.string().uuid(), contentHash: z.string(), datasetHash: z.string(), + cohortId: z.string(), metric: referenceTraceRequestSchema.shape.metric, side: z.enum(['left', 'right']), + provenance: z.literal('frozen_dataset_reference'), timestampBasis: z.literal('reference_source_time'), + clockOffsetMs: z.number().int(), pointToleranceMs: z.number().int(), + totalItems: z.number().int(), offset: z.number().int(), limit: z.number().int(), downsampled: z.literal(false), + items: z.array(projected).max(500), nextCursor: z.object({ offset: z.number().int(), contentHash: z.string() }).nullable(), +}) +const reportSchema = z.object({ version: z.enum(['processed-backtest-report-v1', 'waveform-backtest-report-v1']), + datasetId: z.string().uuid(), datasetHash: z.string(), + protocol: z.object({ clockOffsetMs: z.number().int(), pointToleranceMs: z.number().int() }), + nights: z.array(z.object({ cohortId: z.string(), evaluations: z.array(z.object({ + windows: z.array(z.object({ startMs: z.number().int(), endMs: z.number().int() })).max(60), + })).max(3) })).max(60), +}) +export function referenceTraceIdentity(report: unknown) { + const parsed = reportSchema.safeParse(report) + if (!parsed.success) throw new ReferenceInputError('RESULT_CORRUPT', 'Reference trace report identity is unavailable') + return parsed.data +} + +/** Read only frozen imports, never the current reference library. Projection + * excludes submitted payloads, annotations and subject identity. No score is + * recomputed, and display records are not claims of independently paired data. */ +export function pageReferenceTraces(report: unknown, dataset: { contentHash: string, snapshot: DatasetSnapshot }, raw: z.input) { + const input = referenceTraceRequestSchema.parse(raw), identity = referenceTraceIdentity(report) + if (identity.datasetHash !== dataset.contentHash) throw new ReferenceInputError('REVISION_CONFLICT', 'Reference dataset hash changed') + const nights = identity.nights.filter(night => night.cohortId === input.cohortId) + if (nights.length !== 1) throw new ReferenceInputError('NOT_FOUND', 'Requested reference cohort was not evaluated') + const windows = nights[0].evaluations[0]?.windows + if (!windows) throw new ReferenceInputError('RESULT_CORRUPT', 'Reference scoring windows unavailable') + const items: z.infer['items'] = [] + let totalItems = 0 + for (const reference of dataset.snapshot.references) { + for (const row of reference.measurements) { + const sameFamily = row.metric === input.metric || (input.metric === 'window_ibi_variability_v1' && row.metric.startsWith('hrv_')) + if (!sameFamily || !windows.some(window => row.startMs >= window.startMs && (row.temporalKind === 'point' ? row.startMs < window.endMs : row.endMs <= window.endMs))) continue + if (totalItems >= input.offset && items.length < input.limit) { + items.push(projected.parse({ ...row, importId: reference.importId, importRevision: reference.revision, importContentHash: reference.contentHash, source: reference.source, + compatibility: input.metric === 'sleep_stages' + ? (row.scorable ? null : row.exclusions[0] ?? 'unscorable') + : referenceCompatibility(input.metric, input.metric === 'window_ibi_variability_v1' ? 300000 : null, row) })) + } + totalItems++ + } + } + if (input.offset > totalItems) throw new ReferenceInputError('INVALID_CURSOR', 'Reference cursor exceeds the selected records') + const next = input.offset + items.length + return referenceTracePageSchema.parse({ jobId: input.id, contentHash: input.expectedContentHash, datasetHash: identity.datasetHash, + cohortId: input.cohortId, metric: input.metric, side: dataset.snapshot.side, provenance: 'frozen_dataset_reference', timestampBasis: 'reference_source_time', + ...identity.protocol, totalItems, offset: input.offset, limit: input.limit, items, downsampled: false, + nextCursor: next < totalItems ? { offset: next, contentHash: input.expectedContentHash } : null }) +} diff --git a/src/biometrics/backtest/report-summary.ts b/src/biometrics/backtest/report-summary.ts new file mode 100644 index 000000000..55f78d1c0 --- /dev/null +++ b/src/biometrics/backtest/report-summary.ts @@ -0,0 +1,40 @@ +import { z } from 'zod' +import { ReferenceInputError } from '../references/time' + +const report = z.looseObject({ waveform: z.looseObject({ outputs: z.array(z.looseObject({ readings: z.unknown() })).max(180) }).optional(), version: z.enum(['processed-backtest-report-v1', 'waveform-backtest-report-v1']), + nights: z.array(z.looseObject({ evaluations: z.array(z.looseObject({ + windows: z.array(z.looseObject({ blocks: z.unknown() })).max(60), + })).max(3) })).max(60), +}) + +/** Project only after verifying the complete retained report. The content hash + * still identifies that complete artifact; exact traces remain separately paged. */ +export function summarizeBacktestResult(result: T | null) { + if (result === null) return null + let summary: unknown = null + let waveformReport = false + if (result.available) { + const parsed = report.safeParse(result.report) + if (!parsed.success) throw new ReferenceInputError('RESULT_CORRUPT', 'Report summary format is unavailable') + waveformReport = parsed.data.version === 'waveform-backtest-report-v1' + summary = { ...parsed.data, ...(parsed.data.waveform + ? { waveform: { ...parsed.data.waveform, outputs: parsed.data.waveform.outputs.map((output) => { + const metadata = { ...output } + delete metadata.readings + return metadata + }) } } + : {}), nights: parsed.data.nights.map(night => ({ ...night, + evaluations: night.evaluations.map(evaluation => ({ ...evaluation, windows: evaluation.windows.map((window) => { + const scores = { ...window } + delete scores.blocks + return scores + }) })), + })) } + // Bound the response without truncating scores or silently dropping nights. + if (new TextEncoder().encode(JSON.stringify(summary)).byteLength > 2 * 1024 * 1024) { + throw new ReferenceInputError('RESOURCE_LIMIT', 'Report summary exceeds its 2 MiB transport limit') + } + } + return { ...result, report: summary, projection: waveformReport ? 'scores_without_replay_trace_rows_v1' as const : 'scores_without_stage_trace_rows_v1' as const, + contentHashScope: 'complete_retained_report' as const } +} diff --git a/src/biometrics/backtest/runner.ts b/src/biometrics/backtest/runner.ts new file mode 100644 index 000000000..de48c293e --- /dev/null +++ b/src/biometrics/backtest/runner.ts @@ -0,0 +1,157 @@ +import { assessEvaluation } from '../profiles/evaluation-assessment' +import { dataHash } from '../algorithms/canonical' +import { verifyReplayContext } from '../profiles/replay-context' +import { datasetManifestSchema } from './contracts' +import { installedPromotionPolicies, unqualifiedPolicies } from '../profiles/qualification-registry' +import { numericObjectivePolicies } from './contracts' +import { finalizeObjectiveReport, finalizeStageObjectiveReport } from './objective-finalization' +import { createJobOperations, type AdmissionReceipt } from './job-operations' +import type { ReplayBudget } from './execution-budget' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import { z } from 'zod' +import type * as schema from '@/src/db/biometrics-schema' +import { DatasetStore } from './datasets' +import { BacktestJobStore } from './jobs' +import { backtestWorkSchema } from './evaluator' +import { executeWaveformInputs, verifyWaveformJobBuild } from './waveform-execution' +import type { RawInspectionLocation } from './raw-inspection-child' +import { workerBuildHash, runBacktestChild } from './child' + +/** One leased attempt. Call only after the host's resource/ingest gate admits work. */ +export async function executeNextBacktest(db: BetterSQLite3Database, options: { + owner: string + workerPath: string + rawLocation?: RawInspectionLocation + replayBudget?: ReplayBudget + signal?: AbortSignal + timeoutMs?: number + admit: () => Promise + admissionReceipt?: () => AdmissionReceipt | null +}) { + const operations = createJobOperations() + const admit = async () => { + const allowed = await options.admit() + operations.record(options.admissionReceipt?.() ?? null) + return allowed + } + if (options.signal?.aborted || !await admit() || options.signal?.aborted) return { state: 'deferred' as const } + const jobs = new BacktestJobStore(db) + const claimed = jobs.claim(options.owner) + if (!claimed) return { state: 'idle' as const } + const lease = { id: claimed.id, owner: options.owner, token: claimed.fencingToken } + const controller = new AbortController() + const abort = () => controller.abort() + options.signal?.addEventListener('abort', abort, { once: true }) + if (options.signal?.aborted) controller.abort() + let progress = z.object({ completedNights: z.number(), totalNights: z.number() }).parse(claimed.progress) + let heartbeatFailure: unknown + let resourceFailure = false + let resourceCheck: Promise | null = null + const heartbeat = () => { + try { + const result = jobs.heartbeat(lease, { phase: 'evaluating', ...progress }) + if (result.cancelRequested) controller.abort() + } + catch (error) { + heartbeatFailure = error + controller.abort() + } + } + const checkResources = async () => { + try { + if (await admit()) return + } + catch { /* Missing health evidence must stop background work. */ } + resourceFailure = true + controller.abort() + } + const timer = setInterval(() => { + heartbeat() + if (!resourceCheck && !controller.signal.aborted) { + resourceCheck = checkResources().finally(() => { + resourceCheck = null + }) + } + }, 5000) + try { + const input = z.object({ datasetId: z.string() }).parse(claimed.input) + const retained = new DatasetStore(db).read(input.datasetId) + const work = backtestWorkSchema.parse({ job: claimed.input, dataset: { manifest: retained.manifest, snapshot: retained.snapshot } }) + if (work.job.protocol.mode === 'signal_replay') { + work.waveform = await executeWaveformInputs(db, work, { ...options, signal: controller.signal }) + } + else if (await workerBuildHash(options.workerPath) !== work.job.evaluatorBuildHash) throw new Error('Evaluator bundle changed since enqueue') + heartbeat() + const report = await runBacktestChild(options.workerPath, work, { signal: controller.signal, timeoutMs: options.timeoutMs, + onProgress: (completedNights, totalNights) => { + if (totalNights !== progress.totalNights || completedNights < progress.completedNights || completedNights > totalNights) throw new Error('Invalid worker progress') + progress = { completedNights, totalNights } + heartbeat() + }, + }) + if (work.job.protocol.mode === 'signal_replay') await verifyWaveformJobBuild(work, { ...options, signal: controller.signal }) + let policies = await installedPromotionPolicies(options.workerPath, { evaluatorBuildHash: work.job.evaluatorBuildHash, + pythonBuildHash: work.job.waveform?.pythonBuildHash, protocolHash: work.job.protocolHash }) + // Quiesce the periodic check before the final fresh check and publication. + clearInterval(timer) + await resourceCheck + await checkResources() + if (heartbeatFailure) throw heartbeatFailure + const identity = z.object({ datasetHash: z.string(), protocolHash: z.string(), evaluatorBuildHash: z.string(), split: z.string(), nights: z.array(z.unknown()) }).parse(report) + if (identity.evaluatorBuildHash !== work.job.evaluatorBuildHash || identity.datasetHash !== work.job.datasetHash || identity.protocolHash !== work.job.protocolHash || identity.split !== work.job.split + || identity.nights.length !== progress.totalNights || progress.completedNights !== progress.totalNights) throw new Error('Worker report identity/progress mismatch') + if (controller.signal.aborted) throw new Error('Replay cancelled') + const jobOperations = operations.finish() + let qualificationWarning: string | null = null + if (policies.protocolHash && work.job.waveform) { + try { + verifyReplayContext(z.object({ waveform: z.unknown() }).parse(report).waveform, { + cohortIds: datasetManifestSchema.parse(retained.manifest).splitProtocol.assignments.filter(row => row.split === work.job.split).map(row => row.cohortId), + configHashes: work.job.configs.map(config => config.resolvedHash), side: retained.side, + pythonBuildHash: work.job.waveform.pythonBuildHash, otherSideConfigHash: work.job.waveform.otherSide.resolvedHash, + }, policies.waveformContext) + } + catch { + policies = unqualifiedPolicies() + qualificationWarning = 'Replay context is outside the installed qualification; this report is descriptive.' + } + } + const finalized = finalizeObjectiveReport(report, numericObjectivePolicies(work.job.protocol), jobOperations, policies.numeric) + const measured = { ...finalizeStageObjectiveReport(finalized, work.job.protocol.stageObjectivePolicy, jobOperations, policies.stages), jobOperations, qualificationWarning } + let status: ReturnType['status'] = 'insufficient_evidence' + const candidate = work.job.configs.find(config => config.label === 'candidate') + const current = work.job.configs.find(config => config.label === 'current') + if (candidate?.revisionId && current) { + try { + status = assessEvaluation(work.job, { report: measured, contentHash: dataHash(measured), available: true, + currentHoldoutUseCount: jobs.holdoutUseCount(work.job.datasetId) }, { + side: retained.side, revisionId: candidate.revisionId, resolvedHash: candidate.resolvedHash, currentResolvedHash: current.resolvedHash, + evaluatorBuildHash: work.job.evaluatorBuildHash, dataset: { id: retained.id, contentHash: retained.contentHash, side: retained.side, + available: retained.available, holdoutCohortIds: datasetManifestSchema.parse(retained.manifest).splitProtocol.assignments.filter(row => row.split === 'holdout').map(row => row.cohortId) }, + ...(work.job.waveform ? { waveform: { pythonBuildHash: work.job.waveform.pythonBuildHash, otherSideConfigHash: work.job.waveform.otherSide.resolvedHash } } : {}), + }, policies).status + } + catch { /* Exploratory, incomplete or incompatible evidence cannot claim improvement. */ } + } + if (candidate && current && candidate.resolvedHash === current.resolvedHash) status = 'no_material_change' + jobs.complete(lease, { ...measured, status, evidence: status === 'improved' ? 'qualified_reference_agreement' : work.job.evidence }) + return { state: 'succeeded' as const, jobId: claimed.id } + } + catch { + // Only the authoritative lease may update state. Expired work is recovered + // by the next claimant; it must not overwrite a replacement's result. + try { + const latest = jobs.get(claimed.id) + if (latest.state === 'cancel_requested') jobs.acknowledgeCancellation(lease) + else if (options.signal?.aborted || resourceFailure) jobs.interrupt(lease, resourceFailure ? 'Resource admission withdrawn' : 'Worker host stopped') + else jobs.fail(lease, 'Replay failed validation, resources, or execution') + } + catch { /* Lost lease: recovery owns the state transition. */ } + return { state: 'stopped' as const, jobId: claimed.id } + } + finally { + clearInterval(timer) + await resourceCheck + options.signal?.removeEventListener('abort', abort) + } +} diff --git a/src/biometrics/backtest/splits.ts b/src/biometrics/backtest/splits.ts new file mode 100644 index 000000000..986b035a0 --- /dev/null +++ b/src/biometrics/backtest/splits.ts @@ -0,0 +1,74 @@ +import { z } from 'zod' + +export const sessionWindowSchema = z.object({ + id: z.number().int().positive(), + startMs: z.number().int().min(1e12).max(4102444800000), + endMs: z.number().int().min(1e12).max(4102444800000), +}).strict().refine(row => row.endMs > row.startMs && row.endMs - row.startMs <= 36 * 3600000, + { message: 'Sessions must be positive and at most 36 hours long' }) +export type SessionWindow = z.infer +export type EvaluationSplit = 'tuning' | 'validation' | 'holdout' + +/** + * Local-noon night grouping. Sessions touching the same night, or overlapping + * sessions crossing noon, are one indivisible cohort. Snapshot the returned + * assignment; never recalculate an existing dataset after timezone changes. + */ +export function assignWholeNightSplits(raw: SessionWindow[], timezone: string) { + const sessions = z.array(sessionWindowSchema).min(1).max(60).parse(raw).sort((a, b) => a.startMs - b.startMs || a.id - b.id) + if (new Set(sessions.map(row => row.id)).size !== sessions.length) throw new Error('Duplicate session selection') + const formatter = new Intl.DateTimeFormat('en-CA', { timeZone: timezone, year: 'numeric', month: '2-digit', day: '2-digit', hour: '2-digit', hourCycle: 'h23' }) + const nightAt = (timestamp: number) => { + const parts = Object.fromEntries(formatter.formatToParts(timestamp).map(part => [part.type, part.value])) + const day = new Date(Date.UTC(Number(parts.year), Number(parts.month) - 1, Number(parts.day))) + if (Number(parts.hour) < 12) day.setUTCDate(day.getUTCDate() - 1) + return day.toISOString().slice(0, 10) + } + const labels = sessions.map((session) => { + const keys = new Set([nightAt(session.startMs), nightAt(session.endMs - 1)]) + // Less than any local night length, including DST transitions. Endpoints + // preserve short naps; intermediate samples cover sessions spanning noon. + for (let time = session.startMs + 3600000; time < session.endMs; time += 3600000) keys.add(nightAt(time)) + return [...keys].sort() + }) + const parent = sessions.map((_, i) => i) + const root = (index: number): number => { + while (parent[index] !== index) index = parent[index] + return index + } + const owner = new Map() + for (let index = 0; index < sessions.length; index++) { + for (const label of labels[index]) { + const previous = owner.get(label) + if (previous !== undefined) parent[root(index)] = root(previous) + else owner.set(label, index) + } + } + const grouped = new Map() + for (let index = 0; index < sessions.length; index++) { + const key = root(index) + if (!grouped.has(key)) grouped.set(key, []) + grouped.get(key)?.push(index) + } + const groups = [...grouped.values()].sort((a, b) => sessions[a[0]].startMs - sessions[b[0]].startMs) + // This partition establishes independence, not statistical sufficiency. + // With fewer than three cohorts, all work is descriptive tuning only. + const hasThreeSplits = groups.length >= 3 + const tuningCount = hasThreeSplits ? Math.max(1, Math.min(groups.length - 2, Math.floor(groups.length * 0.6))) : groups.length + const validationCount = hasThreeSplits ? Math.max(1, Math.floor(groups.length * 0.2)) : 0 + const assignments = groups.map((indices, index) => ({ + cohortId: labels[indices[0]][0], + nightLabels: [...new Set(indices.flatMap(i => labels[i]))].sort(), + sessionIds: indices.map(i => sessions[i].id), + startMs: Math.min(...indices.map(i => sessions[i].startMs)), + endMs: Math.max(...indices.map(i => sessions[i].endMs)), + split: (index < tuningCount ? 'tuning' : index < tuningCount + validationCount ? 'validation' : 'holdout') as EvaluationSplit, + })) + return { version: 'chronological-whole-night-v1' as const, timezone, + timezoneRuntime: { icu: typeof process !== 'undefined' ? process.versions?.icu ?? 'unknown' : 'unknown', tz: typeof process !== 'undefined' ? process.versions?.tz ?? 'unknown' : 'unknown' }, + grouping: 'local_noon_connected_sessions' as const, assignments, + evidence: hasThreeSplits ? 'partitioned_not_validated' as const : 'descriptive_only' as const, + limitations: ['Whole-night separation does not by itself establish sufficient reference support or improvement.', + ...(hasThreeSplits ? [] : ['Fewer than three independent night cohorts; no validation or holdout claim is available.'])], + } +} diff --git a/src/biometrics/backtest/stage-comparison.ts b/src/biometrics/backtest/stage-comparison.ts new file mode 100644 index 000000000..608facbbd --- /dev/null +++ b/src/biometrics/backtest/stage-comparison.ts @@ -0,0 +1,43 @@ +import { intersectOutputSupport, restrictToSupport, type SupportInterval } from './common-support' +import type { SleepStageBlock } from '@/src/lib/sleep-stages' +import type { NormalizedMeasurement } from '../references/contracts' +import { scoreStageDurations } from '../references/stage-scoring' + +const summarize = (score: ReturnType) => ({ + agreement: score.agreement, macroF1: score.macroF1, pairedDurationMs: score.pairedDurationMs, + eligibleReferenceMs: score.eligibleReferenceMs, outputDurationMs: score.outputDurationMs, + referenceCoverage: score.referenceCoverage, outputCoverage: score.outputCoverage, +}) +const difference = (candidate: number | null, baseline: number | null) => candidate === null || baseline === null ? null : candidate - baseline + +/** Compare accuracy on exactly the same intervals, while preserving each + * output's own support. Dropping hard intervals cannot manufacture an accuracy + * gain on common support. This is descriptive evidence, not a promotion gate. */ +export function compareStageOutputs(baseline: SleepStageBlock[], candidate: SleepStageBlock[], references: NormalizedMeasurement[], startMs: number, endMs: number, sharedSupport?: SupportInterval[]) { + // The shared scorer validates ordering/overlap and normalizes conflicts across + // reference imports identically for both outputs. + const baselineOwn = scoreStageDurations(baseline, references, startMs, endMs) + const candidateOwn = scoreStageDurations(candidate, references, startMs, endMs) + const pairSupport = intersectOutputSupport([baseline, candidate], startMs, endMs) + const support = sharedSupport === undefined ? pairSupport : restrictToSupport(pairSupport, sharedSupport) + const commonBaseline = restrictToSupport(baseline, support) + const commonCandidate = restrictToSupport(candidate, support) + const baselineCommon = scoreStageDurations(commonBaseline, references, startMs, endMs) + const candidateCommon = scoreStageDurations(commonCandidate, references, startMs, endMs) + return { version: 'paired-stage-support-v1' as const, + commonSupportScope: sharedSupport === undefined ? 'pair' as const : 'all_configurations' as const, + ownSupport: { baseline: summarize(baselineOwn), candidate: summarize(candidateOwn) }, + commonSupport: { baseline: summarize(baselineCommon), candidate: summarize(candidateCommon), + baselineConfusionMs: baselineCommon.confusionMs, candidateConfusionMs: candidateCommon.confusionMs }, + differences: { + commonAgreement: difference(candidateCommon.agreement, baselineCommon.agreement), + commonMacroF1: difference(candidateCommon.macroF1, baselineCommon.macroF1), + ownAgreement: difference(candidateOwn.agreement, baselineOwn.agreement), + referenceCoverage: difference(candidateOwn.referenceCoverage, baselineOwn.referenceCoverage), + outputCoverage: candidateOwn.outputCoverage - baselineOwn.outputCoverage, + pairedDurationMs: candidateOwn.pairedDurationMs - baselineOwn.pairedDurationMs, + }, + coverageReduced: candidateOwn.pairedDurationMs < baselineOwn.pairedDurationMs || candidateOwn.outputDurationMs < baselineOwn.outputDurationMs, + interpretation: 'descriptive_agreement_not_validated_improvement' as const, + } +} diff --git a/src/biometrics/backtest/stage-objective.ts b/src/biometrics/backtest/stage-objective.ts new file mode 100644 index 000000000..476c7c723 --- /dev/null +++ b/src/biometrics/backtest/stage-objective.ts @@ -0,0 +1,70 @@ +import { z } from 'zod' +import { operationalPolicySchema, operationalEvidenceSchema, assessOperationalEvidence } from './objective-policy' +const fraction = z.number().min(0).max(1) +const delta = z.number().min(-1).max(1) +const stage = z.enum(['wake', 'light', 'deep', 'rem']) +export const stageObjectivePolicySchema = z.object({ + version: z.literal('stage-objective-v1'), qualificationId: z.string().min(1).max(200), + minimumNights: z.number().int().min(3).max(60), + minimumPairedDurationMsPerNight: z.number().int().positive().max(86400000), + minimumOutputCoverage: fraction, maximumLostOutputFraction: fraction, + minimumAgreementImprovement: z.number().positive().max(1), + maximumNightAgreementRegression: fraction, maximumRegressedNightFraction: fraction, + maximumMacroF1Regression: fraction, maximumStageF1Regression: fraction, + minimumStageReferenceMsPerNight: z.number().int().positive().max(86400000), + requiredStages: z.array(stage).min(1).max(4).refine(rows => new Set(rows).size === rows.length, 'Duplicate required stages'), + operational: operationalPolicySchema, +}).strict() +export type StageObjectivePolicy = z.infer +export const stageObjectiveEvidenceSchema = z.object({ + split: z.enum(['tuning', 'validation', 'holdout']), holdoutUse: z.number().int().nonnegative(), + intervalLower: delta.nullable(), + nights: z.array(z.object({ cohortId: z.string(), pairedDurationMs: z.number().finite().nonnegative(), + agreementDifference: delta.nullable(), macroF1Difference: delta.nullable(), candidateOutputCoverage: fraction.nullable(), + baselineOutputMs: z.number().finite().nonnegative(), commonOutputMs: z.number().finite().nonnegative(), + perStage: z.array(z.object({ stage, referenceMs: z.number().finite().nonnegative(), f1Difference: delta.nullable() }).strict()).max(4), + }).strict()).max(60), + operational: operationalEvidenceSchema.nullable(), +}).strict() +export type StageObjectiveEvidence = z.infer +export function assessStageObjective(rawPolicy: StageObjectivePolicy, rawEvidence: StageObjectiveEvidence, designation: StageObjectivePolicy | null) { + const policy = stageObjectivePolicySchema.parse(rawPolicy), evidence = stageObjectiveEvidenceSchema.parse(rawEvidence) + if (new Set(evidence.nights.map(row => row.cohortId)).size !== evidence.nights.length + || evidence.nights.some(row => row.commonOutputMs > row.baselineOutputMs || row.pairedDurationMs > row.commonOutputMs + || row.perStage.reduce((sum, item) => sum + item.referenceMs, 0) !== row.pairedDurationMs || new Set(row.perStage.map(item => item.stage)).size !== row.perStage.length)) throw new Error('Invalid stage objective evidence') + const designated = designation !== null && JSON.stringify(stageObjectivePolicySchema.parse(designation)) === JSON.stringify(policy) + const missing: string[] = [], failures: string[] = [] + if (!designated) missing.push('policy_not_designated') + if (evidence.split !== 'holdout' || evidence.holdoutUse !== 1) missing.push('independent_holdout_required') + const supported = evidence.nights.filter(row => row.agreementDifference !== null && row.pairedDurationMs >= policy.minimumPairedDurationMsPerNight) + if (supported.length < policy.minimumNights) missing.push('insufficient_supported_nights') + if (supported.length !== evidence.nights.length) missing.push('unsupported_nights') + if (evidence.intervalLower === null) missing.push('uncertainty_unavailable') + if (evidence.nights.some(row => row.candidateOutputCoverage === null || row.baselineOutputMs === 0)) missing.push('coverage_unavailable') + if (evidence.nights.some(row => row.candidateOutputCoverage !== null && row.candidateOutputCoverage < policy.minimumOutputCoverage)) failures.push('output_coverage') + if (evidence.nights.some(row => row.baselineOutputMs > 0 && (row.baselineOutputMs - row.commonOutputMs) / row.baselineOutputMs > policy.maximumLostOutputFraction)) failures.push('coverage_loss') + const differences = supported.map(row => row.agreementDifference ?? 0) + if (differences.some(value => value < -policy.maximumNightAgreementRegression)) failures.push('night_regression') + if (differences.length && differences.filter(value => value < 0).length / differences.length > policy.maximumRegressedNightFraction) failures.push('regressed_night_fraction') + if (evidence.nights.some(row => row.macroF1Difference === null)) missing.push('macro_f1_unavailable') + if (evidence.nights.some(row => row.macroF1Difference !== null && row.macroF1Difference < -policy.maximumMacroF1Regression)) failures.push('macro_f1_regression') + for (const required of policy.requiredStages) { + const rows = evidence.nights.map(row => row.perStage.find(item => item.stage === required)) + if (rows.some(row => !row || row.referenceMs < policy.minimumStageReferenceMsPerNight || row.f1Difference === null)) missing.push(`stage_support:${required}`) + if (rows.some(row => row?.f1Difference !== null && row?.f1Difference !== undefined && row.f1Difference < -policy.maximumStageF1Regression)) failures.push(`stage_regression:${required}`) + } + const operations = assessOperationalEvidence(policy.qualificationId, policy.operational, evidence.operational) + missing.push(...operations.missing) + failures.push(...operations.failures) + const estimate = differences.length ? differences.reduce((sum, value) => sum + value, 0) / differences.length : null + const status = missing.length + ? 'insufficient_evidence' as const + : failures.length + ? 'regressed' as const + : estimate !== null && estimate >= policy.minimumAgreementImprovement && evidence.intervalLower !== null && evidence.intervalLower >= policy.minimumAgreementImprovement + ? 'improved' as const + : 'no_material_change' as const + return { version: 'stage-objective-assessment-v1' as const, policy, evidence, designated, status, missingEvidence: missing, failedThresholds: failures, + equalNightMeanAgreementDifference: estimate, intervalLower: evidence.intervalLower, + interpretation: 'agreement_with_named_stage_reference_not_clinical_validation' as const } +} diff --git a/src/biometrics/backtest/submission.ts b/src/biometrics/backtest/submission.ts new file mode 100644 index 000000000..3d1c2d059 --- /dev/null +++ b/src/biometrics/backtest/submission.ts @@ -0,0 +1,107 @@ +import { eq } from 'drizzle-orm' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import { z } from 'zod' +import type * as configSchema from '@/src/db/schema' +import * as bioSchema from '@/src/db/biometrics-schema' +import { resolveReplayBaseline } from '../profiles/replay-baseline' +import { experimentProtocolSchema } from './contracts' +import { workerBuildHash } from './child' +import { dirname, join } from 'node:path' +import { packagedWorkerPath } from './location' +import { waveformEvaluatorBuildIdentity } from './waveform-build' +import { podRawInspectionLocation } from './raw-inspection-child' +import { configHash } from '../algorithms/canonical' +import { defaultConfig, manifestHash } from '../algorithms/registry' +import { ProfileStore } from '../profiles/store' +import { ReferenceInputError } from '../references/time' +import { BacktestJobStore } from './jobs' + +export const submitBacktestSchema = z.object({ experimentId: z.string().uuid(), candidateRevisionId: z.string().uuid(), + expectedBindingGeneration: z.number().int().nonnegative(), split: z.enum(['tuning', 'validation', 'holdout']), + idempotencyKey: z.string().min(1).max(200) }).strict() + +export function retryBacktestSubmission(bioDb: BetterSQLite3Database, raw: z.input) { + const input = submitBacktestSchema.parse(raw) + const submissionHash = configHash(input) + const existing = bioDb.select().from(bioSchema.biometricBacktestJobs).where(eq(bioSchema.biometricBacktestJobs.idempotencyKey, input.idempotencyKey)).get() + if (existing) { + const saved = z.object({ submissionHash: z.string().optional() }).parse(existing.input) + if (saved.submissionHash !== submissionHash) throw new ReferenceInputError('REVISION_CONFLICT', 'Backtest idempotency key belongs to a different submission') + return existing + } + return null +} + +const buildProbes = globalThis as typeof globalThis & { + __waveformSubmissionBuild?: { key: string, promise: ReturnType } +} +function sharedWaveformBuild(workerPath: string) { + // Pin the coordinator that ships beside this worker; execution hashes the same sibling. + const coordinatorPath = join(dirname(workerPath), 'coordinator.cjs') + const key = JSON.stringify([coordinatorPath, workerPath, podRawInspectionLocation.pythonPath, podRawInspectionLocation.modulesPath]) + if (buildProbes.__waveformSubmissionBuild?.key === key) return buildProbes.__waveformSubmissionBuild.promise + const entry = { key, promise: waveformEvaluatorBuildIdentity(coordinatorPath, workerPath, podRawInspectionLocation) } + buildProbes.__waveformSubmissionBuild = entry + // Coalesce concurrent submissions, but never cache an identity after its probe. + entry.promise = entry.promise.finally(() => { + if (buildProbes.__waveformSubmissionBuild === entry) delete buildProbes.__waveformSubmissionBuild + }) + return entry.promise +} + +/** Resolve immutable build pins only after checking for an existing submission. */ +export async function resolveSubmissionBuild(bioDb: BetterSQLite3Database, raw: z.input) { + const input = submitBacktestSchema.parse(raw) + const existing = retryBacktestSubmission(bioDb, input) + if (existing) return { existing } + const experiment = new BacktestJobStore(bioDb).getExperiment(input.experimentId) + return resolveInstalledReplayBuild(experiment.protocol.mode) +} + +/** Read installed artifacts; never accept evaluator identity from an RPC caller. */ +export async function resolveInstalledReplayBuild(mode: z.infer['mode']) { + try { + const workerPath = packagedWorkerPath() + if (mode === 'signal_replay') { + const build = await sharedWaveformBuild(workerPath) + return { evaluatorBuildHash: build.hash, pythonBuildHash: build.python.hash } + } + return { evaluatorBuildHash: await workerBuildHash(workerPath) } + } + catch { throw new ReferenceInputError('WORKER_UNAVAILABLE', 'Packaged replay build identity cannot be read') } +} + +/** Public callers select revisions, never supply trusted baseline snapshots. */ +export function submitBacktest(configDb: BetterSQLite3Database, bioDb: BetterSQLite3Database, evaluatorBuildHash: string, + raw: z.input, pythonBuildHash?: string) { + const input = submitBacktestSchema.parse(raw) + const submissionHash = configHash(input) + const existing = retryBacktestSubmission(bioDb, input) + if (existing) return existing + const experiment = bioDb.select().from(bioSchema.biometricExperiments).where(eq(bioSchema.biometricExperiments.id, input.experimentId)).get() + if (!experiment) throw new ReferenceInputError('NOT_FOUND', 'Experiment not found') + const signalReplay = experimentProtocolSchema.parse(experiment.protocol).mode === 'signal_replay' + if (signalReplay && !pythonBuildHash) throw new ReferenceInputError('WORKER_UNAVAILABLE', 'Waveform submission requires the Python build identity') + const snapshots = configDb.transaction(() => bioDb.transaction(() => { + const profiles = new ProfileStore(configDb) + const binding = profiles.binding(experiment.side) + if (binding.generation !== input.expectedBindingGeneration) throw new ReferenceInputError('REVISION_CONFLICT', 'Profile binding changed; inspect the current side before evaluating') + const candidate = profiles.validateRevision(input.candidateRevisionId, experiment.side) + // This is the actual retrospective view used by existing stage endpoints. + // A desired-but-unacknowledged vitals profile is not a current baseline. + const currentBaseline = resolveReplayBaseline(configDb, bioDb, experiment.side) + const otherBaseline = signalReplay ? resolveReplayBaseline(configDb, bioDb, experiment.side === 'left' ? 'right' : 'left') : null + const current = currentBaseline.resolved + return { waveform: otherBaseline && pythonBuildHash + ? { pythonBuildHash, + otherSide: { manifestHash, resolvedHash: otherBaseline.hash, resolved: otherBaseline.resolved }, + baselineProvenance: { current: currentBaseline.provenance, otherSide: otherBaseline.provenance } } + : undefined, configs: [ + { label: 'default' as const, revisionId: null, manifestHash, resolvedHash: configHash(defaultConfig), resolved: defaultConfig }, + { label: 'current' as const, revisionId: currentBaseline.revisionId, manifestHash, resolvedHash: configHash(current), resolved: current }, + { label: 'candidate' as const, revisionId: candidate.revision.id, manifestHash, resolvedHash: candidate.hash, resolved: candidate.resolved }, + ] } + })) + return new BacktestJobStore(bioDb).enqueue({ experimentId: input.experimentId, idempotencyKey: input.idempotencyKey, + evaluatorBuildHash, submissionHash, split: input.split, configs: snapshots.configs, ...(snapshots.waveform ? { waveform: snapshots.waveform } : {}) }) +} diff --git a/src/biometrics/backtest/tests/admission-build.test.ts b/src/biometrics/backtest/tests/admission-build.test.ts new file mode 100644 index 000000000..0331ecfa1 --- /dev/null +++ b/src/biometrics/backtest/tests/admission-build.test.ts @@ -0,0 +1,32 @@ +// @vitest-environment node +import { build } from 'esbuild' +import { createRequire } from 'node:module' +import { runInNewContext } from 'node:vm' +import { expect, it } from 'vitest' + +it('retains Linux health collection when a bundler substitutes a macOS build platform', async () => { + const compiled = await build({ entryPoints: ['src/biometrics/backtest/admission.ts'], bundle: true, + platform: 'node', format: 'cjs', write: false, minify: true, + define: { 'process.platform': JSON.stringify('darwin') } }) + const nativeRequire = createRequire(import.meta.url) + const exported: { exports: { collectAdmissionSample?: (path: string, piezo: () => number) => Promise } } = { exports: {} } + const accesses: string[] = [] + const at = Date.now() + runInNewContext(compiled.outputFiles[0].text, { module: exported, exports: exported.exports, + process, setTimeout, clearTimeout, Buffer, + require: (name: string) => { + if (name === 'node:os') return { platform: () => 'darwin', availableParallelism: () => 4, loadavg: () => [1, 1, 1] } + if (name === 'node:fs/promises') return { + readFile: async (path: string) => { + accesses.push(path) + return path === '/proc/meminfo' ? 'MemAvailable: 1024 kB\n' : '35000\n' + }, + statfs: async () => ({ bavail: 100, bsize: 4096 }), + } + return nativeRequire(name) + } }) + expect(await exported.exports.collectAdmissionSample?.('/fixture', () => at)).toMatchObject({ + availableMemoryBytes: 1048576, freeDiskBytes: 409600, temperatureC: 35, loadPerCpu: 0.25, piezoReceivedAtMs: at, + }) + expect(accesses).toEqual(expect.arrayContaining(['/proc/meminfo', '/sys/class/thermal/thermal_zone0/temp'])) +}) diff --git a/src/biometrics/backtest/tests/admission-collector.test.ts b/src/biometrics/backtest/tests/admission-collector.test.ts new file mode 100644 index 000000000..ba8c99082 --- /dev/null +++ b/src/biometrics/backtest/tests/admission-collector.test.ts @@ -0,0 +1,39 @@ +// @vitest-environment node +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +const mocks = vi.hoisted(() => ({ readFile: vi.fn(), statfs: vi.fn(), loadavg: vi.fn(), availableParallelism: vi.fn() })) +vi.mock('node:fs/promises', () => ({ readFile: mocks.readFile, statfs: mocks.statfs })) +vi.mock('node:os', () => ({ loadavg: mocks.loadavg, availableParallelism: mocks.availableParallelism })) +import { collectAdmissionSample } from '../admission' +beforeEach(() => { + mocks.readFile.mockImplementation(async (path: string) => path === '/proc/meminfo' ? 'MemAvailable: 1024 kB\n' : '35000\n') + mocks.statfs.mockResolvedValue({ bavail: 100, bsize: 4096 }) + mocks.loadavg.mockReturnValue([2, 1, 1]) + mocks.availableParallelism.mockReturnValue(4) +}) +afterEach(() => { + vi.resetAllMocks() +}) +it('collects available disk/memory and a current scheduling probe with a piezo timestamp', async () => { + const before = Date.now() + const sample = await collectAdmissionSample('/fixture/data', () => before - 100) + expect(sample).toMatchObject({ availableMemoryBytes: 1048576, freeDiskBytes: 409600, loadPerCpu: 0.5, + temperatureC: 35, piezoReceivedAtMs: before - 100 }) + expect(sample?.measuredAtMs).toBeGreaterThanOrEqual(before) + expect(sample?.measuredAtMs).toBeLessThanOrEqual(Date.now()) + expect(sample?.eventLoopDelayMs).toBeGreaterThanOrEqual(0) + expect(mocks.statfs).toHaveBeenCalledWith('/fixture/data') +}) +it('returns unavailable when piezo has never been observed', async () => { + expect(await collectAdmissionSample('/fixture/data', () => undefined)).toBeNull() +}) +it('does not convert empty thermal files into a zero-degree reading', async () => { + mocks.readFile.mockImplementation(async (path: string) => path === '/proc/meminfo' ? 'MemAvailable: 1024 kB\n' : '\n') + expect(await collectAdmissionSample('/fixture/data', Date.now)).toBeNull() +}) +it('returns unavailable on collector failures and missing Linux health files', async () => { + mocks.statfs.mockRejectedValue(new Error('I/O error')) + expect(await collectAdmissionSample('/fixture/data', Date.now)).toBeNull() + mocks.statfs.mockResolvedValue({ bavail: 100, bsize: 4096 }) + mocks.readFile.mockRejectedValue(Object.assign(new Error('Missing Linux health file'), { code: 'ENOENT' })) + expect(await collectAdmissionSample('/fixture/data', Date.now)).toBeNull() +}) diff --git a/src/biometrics/backtest/tests/admission-service.test.ts b/src/biometrics/backtest/tests/admission-service.test.ts new file mode 100644 index 000000000..3b95a5961 --- /dev/null +++ b/src/biometrics/backtest/tests/admission-service.test.ts @@ -0,0 +1,65 @@ +// @vitest-environment node +import { afterEach, expect, it, vi } from 'vitest' +import type * as Admission from '../admission' +import { collectAdmissionSample } from '../admission' +import { replayAdmission, replayAdmitter } from '../admission-service' + +const frames = vi.hoisted(() => ({ 'piezo-dual': 1000 })) +vi.mock('@/src/streaming/piezoStream', () => ({ getSensorFrameTimes: () => frames })) +vi.mock('../admission', async original => ({ ...await original(), collectAdmissionSample: vi.fn() })) +const db = { all: vi.fn(() => [{ name: 'main', file: '/private/biometrics.db' }]) } as unknown as Parameters[0] +const policy = { qualificationId: 'test-evidence', minimumAvailableMemoryBytes: 100, minimumFreeDiskBytes: 100, + maximumLoadPerCpu: 1, maximumEventLoopDelayMs: 100, maximumPiezoAgeMs: 10000, maximumTemperatureC: 65, maximumSampleAgeMs: 1000 } +afterEach(() => { + vi.unstubAllEnvs() + vi.clearAllMocks() +}) + +it.each([undefined, '', '{', '{}', JSON.stringify({ ...policy, maximumPiezoAgeMs: 0 })])('denies missing or invalid server policy without collecting health: %s', async (value) => { + vi.stubEnv('SLEEPYPOD_BIOMETRICS_ADMISSION_POLICY', value) + expect(await replayAdmission(db)).toEqual({ configured: false, qualificationId: null, admitted: false, reasons: ['unqualified'] }) + expect(collectAdmissionSample).not.toHaveBeenCalled() +}) + +it('uses fresh piezo receipt times and reevaluates health for every admission', async () => { + vi.stubEnv('SLEEPYPOD_BIOMETRICS_ADMISSION_POLICY', JSON.stringify(policy)) + vi.mocked(collectAdmissionSample).mockImplementation(async (directory, piezo) => { + expect(directory).toBe('/private') + expect(piezo()).toBe(frames['piezo-dual']) + return { measuredAtMs: Date.now(), piezoReceivedAtMs: Date.now(), availableMemoryBytes: 200, freeDiskBytes: 200, + loadPerCpu: 0.2, eventLoopDelayMs: 2, temperatureC: 30 } + }) + expect(await replayAdmission(db)).toEqual({ configured: true, qualificationId: 'test-evidence', admitted: true, reasons: [] }) + vi.mocked(collectAdmissionSample).mockResolvedValueOnce(null) + expect(await replayAdmitter(db)()).toBe(false) + expect(collectAdmissionSample).toHaveBeenCalledTimes(2) +}) + +it('fails closed when collecting health throws', async () => { + vi.stubEnv('SLEEPYPOD_BIOMETRICS_ADMISSION_POLICY', JSON.stringify(policy)) + vi.mocked(collectAdmissionSample).mockRejectedValueOnce(new Error('unavailable')) + expect(await replayAdmission(db)).toEqual({ configured: true, qualificationId: 'test-evidence', admitted: false, reasons: ['health_unavailable'] }) +}) + +it('fails closed without a persistent database path or a configured policy', async () => { + const { replayHealthAdmission } = await import('../admission-service') + vi.stubEnv('SLEEPYPOD_BIOMETRICS_ADMISSION_POLICY', JSON.stringify(policy)) + const memory = { all: vi.fn(() => [{ name: 'main', file: '' }]) } as unknown as Parameters[0] + expect(await replayAdmission(memory)).toEqual({ configured: true, qualificationId: 'test-evidence', admitted: false, reasons: ['health_unavailable'] }) + expect(collectAdmissionSample).not.toHaveBeenCalled() + vi.stubEnv('SLEEPYPOD_BIOMETRICS_ADMISSION_POLICY', '') + expect(await replayHealthAdmission(db)()).toEqual({ allowed: false, sample: null }) +}) + +it('returns the admitted sample only through the internal health callback', async () => { + const { replayHealthAdmission } = await import('../admission-service') + vi.stubEnv('SLEEPYPOD_BIOMETRICS_ADMISSION_POLICY', JSON.stringify(policy)) + vi.mocked(collectAdmissionSample).mockImplementation(async () => ({ measuredAtMs: Date.now(), piezoReceivedAtMs: Date.now() - 250, + availableMemoryBytes: 200, freeDiskBytes: 200, loadPerCpu: 0.2, eventLoopDelayMs: 2, temperatureC: 30 })) + const measured = await replayHealthAdmission(db)() + expect(measured).toMatchObject({ allowed: true, sample: { qualificationId: 'test-evidence', ingestionAgeMs: expect.any(Number) } }) + expect(measured.sample?.ingestionAgeMs).toBeGreaterThanOrEqual(250) + expect(await replayAdmission(db)).not.toHaveProperty('receipt') + vi.mocked(collectAdmissionSample).mockResolvedValueOnce(null) + expect(await replayHealthAdmission(db)()).toEqual({ allowed: false, sample: null }) +}) diff --git a/src/biometrics/backtest/tests/admission.test.ts b/src/biometrics/backtest/tests/admission.test.ts new file mode 100644 index 000000000..a15e14650 --- /dev/null +++ b/src/biometrics/backtest/tests/admission.test.ts @@ -0,0 +1,39 @@ +// @vitest-environment node +import { describe, expect, it } from 'vitest' +import { evaluateAdmission, parseAvailableMemory, type AdmissionPolicy, type AdmissionSample } from '../admission' +// Test values deliberately are not deployment defaults. +const policy: AdmissionPolicy = { qualificationId: 'fixture', minimumAvailableMemoryBytes: 1000, minimumFreeDiskBytes: 2000, + maximumLoadPerCpu: 0.8, maximumEventLoopDelayMs: 50, maximumPiezoAgeMs: 1000, maximumTemperatureC: 60, maximumSampleAgeMs: 500 } +const sample: AdmissionSample = { measuredAtMs: 10000, availableMemoryBytes: 1000, freeDiskBytes: 2000, + loadPerCpu: 0.8, eventLoopDelayMs: 50, piezoReceivedAtMs: 9500, temperatureC: 60 } +describe('replay resource admission', () => { + it('admits fresh evidence at exact qualified boundaries', () => { + expect(evaluateAdmission(sample, policy, 10500)).toEqual({ admitted: true, reasons: [] }) + }) + it.each([ + ['availableMemoryBytes', 999, 'memory_pressure'], ['freeDiskBytes', 1999, 'disk_pressure'], + ['loadPerCpu', 0.81, 'cpu_pressure'], ['eventLoopDelayMs', 51, 'event_loop_delay'], + ['piezoReceivedAtMs', 8999, 'piezo_stale'], ['temperatureC', 61, 'temperature'], + ['measuredAtMs', 9499, 'stale_sample'], ['piezoReceivedAtMs', 10001, 'clock_skew'], + ['measuredAtMs', 10001, 'clock_skew'], + ])('defers for %s outside the qualified range', (key, value, reason) => { + expect(evaluateAdmission({ ...sample, [key]: value }, policy, 10000)).toEqual({ admitted: false, reasons: [reason] }) + }) + it('does not turn absent, malformed, or nonfinite health evidence into zeros', () => { + for (const invalid of [null, {}, { ...sample, availableMemoryBytes: undefined }, { ...sample, temperatureC: NaN }, { ...sample, freeDiskBytes: Infinity }]) { + expect(evaluateAdmission(invalid, policy, 10000)).toEqual({ admitted: false, reasons: ['health_unavailable'] }) + } + expect(evaluateAdmission(sample, null, 10000)).toEqual({ admitted: false, reasons: ['unqualified'] }) + expect(evaluateAdmission(sample, { ...policy, maximumLoadPerCpu: 0 }, 10000).admitted).toBe(false) + }) + it('reports simultaneous pressure without hiding subsequent reasons', () => { + expect(evaluateAdmission({ ...sample, availableMemoryBytes: 0, freeDiskBytes: 0, temperatureC: 70 }, policy, 10000).reasons) + .toEqual(['memory_pressure', 'disk_pressure', 'temperature']) + }) + it('reads reclaimable available Linux memory, not only unused pages', () => { + expect(parseAvailableMemory('MemTotal: 2000 kB\nMemFree: 100 kB\nMemAvailable: 1200 kB\n')).toBe(1228800) + for (const invalid of ['MemFree: 100 kB', 'MemAvailable: -1 kB', 'MemAvailable: 2 MB', 'MemAvailable: 99999999999999999 kB']) { + expect(() => parseAvailableMemory(invalid)).toThrow() + } + }) +}) diff --git a/src/biometrics/backtest/tests/artifacts.test.ts b/src/biometrics/backtest/tests/artifacts.test.ts new file mode 100644 index 000000000..83e66bb27 --- /dev/null +++ b/src/biometrics/backtest/tests/artifacts.test.ts @@ -0,0 +1,156 @@ +// @vitest-environment node +import { mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, utimesSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { spawn } from 'node:child_process' +import { once } from 'node:events' +import { createRequire } from 'node:module' +import { join } from 'node:path' +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { afterEach, beforeEach, expect, it } from 'vitest' +import * as schema from '@/src/db/biometrics-schema' +import { ReportArtifactStore, REPORT_BUDGET_BYTES } from '../artifacts' +import { BacktestJobStore } from '../jobs' +import { DatasetStore } from '../datasets' +import { backtestFixture } from './fixtures/backtest' +let directory: string +beforeEach(() => { + directory = mkdtempSync(join(tmpdir(), 'biometric-artifacts-')) +}) +afterEach(() => rmSync(directory, { recursive: true, force: true })) +const id = '11111111-1111-4111-8111-111111111111' +it('atomically retains complete data, verifies bytes and rejects traversal and corrupt files', () => { + const store = new ReportArtifactStore(directory) + const report = { traces: Array.from({ length: 1100 }, () => 'x'.repeat(2048)) } + const descriptor = store.write(id, 1, JSON.stringify(report), 0) + expect(store.read(descriptor)).toEqual(report) + expect(readdirSync(directory).filter(name => !name.startsWith('.publication.sqlite'))).toEqual([descriptor.name]) + expect(() => store.read({ ...descriptor, name: '../private.json' })).toThrow('descriptor') + writeFileSync(join(directory, descriptor.name), 'corrupt') + expect(() => store.read(descriptor)).toThrow('corrupt') +}) +it('counts staged storage against quota and only collects old unreferenced owned files', () => { + const store = new ReportArtifactStore(directory) + expect(() => store.write(id, 1, '{}', REPORT_BUDGET_BYTES)).toThrow('budget') + const committed = store.write(id, 1, '{"first":1}', 0) + const orphan = store.write(id, 2, '{"second":2}', 0) + const fresh = store.write(id, 3, '{"third":3}', 0) + const old = new Date(Date.now() - 7200000) + for (const name of [committed.name, orphan.name]) utimesSync(join(directory, name), old, old) + writeFileSync(join(directory, 'unrelated.txt'), 'keep') + expect(store.collect(new Set([committed.name]))).toBe(1) + expect(readdirSync(directory).filter(name => !name.startsWith('.publication.sqlite')).sort()).toEqual([committed.name, fresh.name, 'unrelated.txt'].sort()) +}) +it('publishes a descriptor under the lease, resolves it after reopening, and retains an orphan after failed DB commit', () => { + const databasePath = join(directory, 'biometrics.db') + const raw = new Database(databasePath) + try { + const db = drizzle(raw, { schema }) + migrate(db, { migrationsFolder: 'src/db/biometrics-migrations' }) + const now = Date.now() + db.insert(schema.sleepRecords).values({ side: 'left', enteredBedAt: new Date(now - 3600000), leftBedAt: new Date(now), sleepDurationSeconds: 3600, timesExitedBed: 0 }).run() + const dataset = new DatasetStore(db).create({ side: 'left', subjectId: 'fixture', sessionIds: [1], referenceImportIds: [], timezone: 'UTC', idempotencyKey: 'dataset' }) + const store = new BacktestJobStore(db) + const fixture = backtestFixture() + const experiment = store.createExperiment(dataset.id, fixture.job.protocol) + const enqueue = (key: string) => store.enqueue({ experimentId: experiment.id, configs: fixture.job.configs, evaluatorBuildHash: 'f'.repeat(64), idempotencyKey: key, split: 'tuning' }) + const job = enqueue('first') + const claimed = store.claim('fixture') + if (!claimed) throw new Error('Missing lease') + const report = { version: 'fixture', traces: Array.from({ length: 1100 }, () => 'x'.repeat(2048)) } + store.complete({ id: job.id, owner: 'fixture', token: claimed.fencingToken }, report) + const persisted = db.select().from(schema.biometricBacktestResults).get() + expect(persisted?.report).toMatchObject({ kind: 'biometric-report-artifact-v1' }) + expect(Buffer.byteLength(JSON.stringify(persisted?.report))).toBeLessThan(512) + const reopened = new Database(databasePath) + try { + expect(new BacktestJobStore(drizzle(reopened, { schema })).result(job.id)?.report).toEqual(report) + } + finally { reopened.close() } + const second = enqueue('second') + const secondLease = store.claim('fixture') + if (!secondLease) throw new Error('Missing second lease') + raw.exec('CREATE TRIGGER reject_result BEFORE INSERT ON biometric_backtest_results BEGIN SELECT RAISE(ABORT, \'full\'); END;') + expect(() => store.complete({ id: second.id, owner: 'fixture', token: secondLease.fencingToken }, report)).toThrow('full') + expect(store.get(second.id).state).toBe('running') + expect(store.result(second.id)).toBeNull() + const root = join(directory, 'biometric-report-artifacts') + expect(readdirSync(root).filter(name => !name.startsWith('.publication.sqlite'))).toHaveLength(2) + for (const name of readdirSync(root).filter(name => !name.startsWith('.publication.sqlite'))) { + expect(JSON.parse(readFileSync(join(root, name), 'utf8'))).toEqual(report) + utimesSync(join(root, name), new Date(0), new Date(0)) + } + expect(store.collectOrphanedArtifacts()).toBe(1) + expect(store.result(job.id)?.report).toEqual(report) + const hash = persisted?.contentHash + if (!hash) throw new Error('Missing report hash') + raw.exec('CREATE TRIGGER reject_release BEFORE UPDATE ON biometric_backtest_results BEGIN SELECT RAISE(ABORT, \'full\'); END;') + expect(() => store.releaseResult(job.id, hash)).toThrow('full') + expect(store.result(job.id)?.report).toEqual(report) + raw.exec('DROP TRIGGER reject_release') + const released = store.releaseResult(job.id, hash) + expect(released).toMatchObject({ available: false, report: null, contentHash: hash }) + expect(readdirSync(root).filter(name => !name.startsWith('.publication.sqlite'))).toEqual([]) + expect(store.releaseResult(job.id, hash)).toEqual(released) + expect(store.result(job.id)).toEqual(released) + } + finally { raw.close() } +}) + +it('reuses an identical staged artifact near quota and serializes other publishers', () => { + const store = new ReportArtifactStore(directory) + const text = JSON.stringify({ value: 'data' }) + const descriptor = store.write(id, 1, text, 0) + expect(store.write(id, 1, text, REPORT_BUDGET_BYTES - descriptor.byteLength)).toEqual(descriptor) + const contender = new ReportArtifactStore(directory) + store.withPublication(() => { + expect(() => contender.write(id, 2, '{}', 0)).toThrow(/locked/) + }) + expect(contender.write(id, 2, '{}', 0).byteLength).toBe(2) +}) + +it('releases the publication lock after a process dies without committing', async () => { + const sqliteModule = createRequire(import.meta.url).resolve('better-sqlite3') + const child = spawn(process.execPath, ['-e', ` + const Database = require(process.argv[1]); + const db = new Database(process.argv[2]); + db.exec('BEGIN IMMEDIATE'); + process.send('locked'); + setInterval(() => {}, 1000); + `, sqliteModule, join(directory, '.publication.sqlite')], { stdio: ['ignore', 'ignore', 'pipe', 'ipc'] }) + const exited = once(child, 'exit') + try { + await Promise.race([ + once(child, 'message'), + exited.then(() => { throw new Error('Lock holder exited before readiness') }), + ]) + const store = new ReportArtifactStore(directory) + expect(() => store.write(id, 1, '{}', 0)).toThrow(/locked/) + child.kill('SIGKILL') + await exited + const descriptor = store.write(id, 1, '{}', 0) + expect(store.read(descriptor)).toEqual({}) + } + finally { + child.kill('SIGKILL') + await exited + } +}) + +it('reuses an identical staged artifact, rejects empty or tampered files and only unlinks under the lock', () => { + const store = new ReportArtifactStore(directory) + expect(() => store.write(id, 1, '', 0)).toThrow('16 MiB') + const first = store.write(id, 1, '{"same":true}', 0) + expect(store.write(id, 1, '{"same":true}', 0)).toEqual(first) + expect(() => store.write(id, 1, '{"same":true}', REPORT_BUDGET_BYTES)).toThrow('budget') + writeFileSync(join(directory, first.name), '{"same":tru!}') + expect(() => store.read(first)).toThrow('corrupt') + expect(() => store.removeReleased(first)).toThrow('publication lock') + expect(store.withPublication(() => { + store.removeReleased(first) + return readdirSync(directory).includes(first.name) + })).toBe(false) + mkdirSync(join(directory, `${id}.2.${'a'.repeat(64)}.json`)) + expect(() => store.usage()).toThrow('Unexpected report artifact type') +}) diff --git a/src/biometrics/backtest/tests/child.test.ts b/src/biometrics/backtest/tests/child.test.ts new file mode 100644 index 000000000..b7ed5cc96 --- /dev/null +++ b/src/biometrics/backtest/tests/child.test.ts @@ -0,0 +1,84 @@ +// @vitest-environment node +import { execFile } from 'node:child_process' +import { mkdtemp, rm, writeFile, readFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { promisify } from 'node:util' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { dataHash } from '../../algorithms/canonical' +import { runBacktestChild } from '../child' +import { backtestFixture } from './fixtures/backtest' +let directory: string +let worker: string +beforeAll(async () => { + directory = await mkdtemp(join(tmpdir(), 'biometrics-worker-test-')) + worker = join(directory, 'worker.cjs') + await promisify(execFile)(process.execPath, ['scripts/build/biometrics-worker.mjs', directory]) +}) +afterAll(async () => rm(directory, { recursive: true, force: true })) + +describe('shipped worker process', () => { + it('executes a full frozen replay in a child without live dependencies', async () => { + const progress: number[] = [] + const report = await runBacktestChild(worker, backtestFixture(), { onProgress: completed => progress.push(completed) }) + expect(progress).toEqual([1]) + expect(report).toMatchObject({ version: 'processed-backtest-report-v1', status: 'insufficient_evidence', nights: [{ uniqueObservationCount: 10 }] }) + const dependencies = await readFile(join(directory, 'dependencies.json'), 'utf8') + expect(dependencies).not.toMatch(/src\/db\/|src\/hardware\/|drizzle-orm|better-sqlite3/) + }) + it('processes a retained twelve-hour window beyond the configuration hash size limit', async () => { + const input = backtestFixture() + const start = input.dataset.snapshot.sessions[0].startMs + for (const session of input.dataset.snapshot.sessions) { + session.endMs = start + 43200000 + session.vitals = Array.from({ length: 720 }, (_, id) => ({ id, timestampMs: start + id * 60000, heartRate: 60.1, hrv: 30, breathingRate: 14, processingRunId: null })) + } + input.dataset.manifest.splitProtocol.assignments[0].endMs = start + 43200000 + input.job.datasetHash = dataHash(input.dataset) + expect(await runBacktestChild(worker, input)).toMatchObject({ nights: [{ uniqueObservationCount: 720 }] }) + }) + + it('terminates stalled work on time budget and cancellation, including ignored SIGTERM', async () => { + const stalled = join(directory, 'stalled.cjs') + await writeFile(stalled, 'process.on(\'SIGTERM\', () => {}); process.stdin.resume(); setInterval(() => {}, 1000)') + await expect(runBacktestChild(stalled, {}, { timeoutMs: 100 })).rejects.toThrow('time budget') + const controller = new AbortController() + const running = runBacktestChild(stalled, {}, { signal: controller.signal }) + setTimeout(() => controller.abort(), 100) + await expect(running).rejects.toThrow('cancelled') + }) + it('rejects invalid inputs, malformed output and nonzero worker exits', async () => { + await expect(runBacktestChild(worker, {})).rejects.toThrow('failed') + const malformed = join(directory, 'malformed.cjs') + await writeFile(malformed, 'process.stdin.resume(); process.stdout.write(\'not JSON\\n\')') + await expect(runBacktestChild(malformed, {})).rejects.toThrow('invalid event') + await expect(runBacktestChild(worker, {}, { timeoutMs: 0 })).rejects.toThrow('time budget') + }) + it('rejects a complete report followed by dangling partial UTF-8 output', async () => { + const trailing = join(directory, 'trailing.cjs') + await writeFile(trailing, `const { spawn } = require('node:child_process') +const child = spawn(process.execPath, [${JSON.stringify(worker)}], { stdio: ['pipe', 'pipe', 'inherit'] }) +process.stdin.pipe(child.stdin) +child.stdout.on('data', chunk => process.stdout.write(chunk)) +child.on('close', code => { process.stdout.write(Buffer.from([0xc3])); process.exitCode = code })`) + await expect(runBacktestChild(trailing, backtestFixture())).rejects.toThrow('incomplete report') + }) + it('rejects oversized or already-cancelled replays before spawning', async () => { + await expect(runBacktestChild(worker, { padding: Array.from({ length: 5200 }, () => 'x'.repeat(2048)) })).rejects.toThrow('exceeds 10 MiB') + const controller = new AbortController() + controller.abort() + await expect(runBacktestChild(worker, {}, { signal: controller.signal })).rejects.toThrow('cancelled') + await expect(runBacktestChild(worker, {}, { timeoutMs: 1800001 })).rejects.toThrow('time budget') + }) + it('stops workers that flood diagnostics or keep talking after their result', async () => { + const noisy = join(directory, 'noisy.cjs') + await writeFile(noisy, 'process.stdin.resume(); process.stderr.write(\'x\'.repeat(20000)); process.stdin.on(\'end\', () => process.exit(0))') + await expect(runBacktestChild(noisy, {})).rejects.toThrow('diagnostic output exceeded') + const chatty = join(directory, 'chatty.cjs') + await writeFile(chatty, `process.stdin.resume(); process.stdin.on('end', () => { + process.stdout.write(JSON.stringify({ type: 'result', report: { version: 'processed-backtest-report-v1' } }) + '\\n') + process.stdout.write(JSON.stringify({ type: 'progress', completedNights: 1, totalNights: 1 }) + '\\n') + })`) + await expect(runBacktestChild(chatty, {})).rejects.toThrow('invalid event') + }) +}) diff --git a/src/biometrics/backtest/tests/common-support.test.ts b/src/biometrics/backtest/tests/common-support.test.ts new file mode 100644 index 000000000..88a34b91b --- /dev/null +++ b/src/biometrics/backtest/tests/common-support.test.ts @@ -0,0 +1,38 @@ +// @vitest-environment node +import { expect, it } from 'vitest' +import { intersectOutputSupport, restrictToSupport } from '../common-support' +import { compareStageOutputs } from '../stage-comparison' +import { previewReferenceImport } from '../../references/normalize' +it('intersects all three outputs and clips boundaries without filling gaps', () => { + const outputs = [[{ start: 0, end: 100 }], [{ start: 10, end: 40 }, { start: 50, end: 90 }], [{ start: 20, end: 60 }]] + const before = JSON.stringify(outputs) + expect(intersectOutputSupport(outputs, 25, 80)).toEqual([{ start: 25, end: 40 }, { start: 50, end: 60 }]) + expect(JSON.stringify(outputs)).toBe(before) + expect(intersectOutputSupport([outputs[0], [], outputs[2]], 0, 100)).toEqual([]) +}) +it('preserves stage labels while rejecting ambiguous support', () => { + expect(restrictToSupport([{ start: 0, end: 10, stage: 'deep' }], [{ start: 2, end: 4 }, { start: 6, end: 8 }])) + .toEqual([{ start: 2, end: 4, stage: 'deep' }, { start: 6, end: 8, stage: 'deep' }]) + expect(() => intersectOutputSupport([[{ start: 0, end: 10 }, { start: 5, end: 15 }]], 0, 20)).toThrow('nonoverlapping') + expect(() => intersectOutputSupport([], 0, 20)).toThrow('bounded') +}) +it('scores every pair on the same three-way denominator while keeping own coverage', () => { + const start = 1791673200000, end = start + 20000 + const full = [{ start, end, stage: 'light' as const }] + const current = [{ start, end, stage: 'deep' as const }] + const candidate = [{ start, end: start + 10000, stage: 'light' as const }] + const refs = previewReferenceImport({ side: 'left', subjectId: 'fixture', source: 'test', rows: [ + { recordKey: 'stage', metric: 'sleep_stages', value: 'light', unit: 'stage', startMs: start, endMs: end }, + ] }).measurements + const support = intersectOutputSupport([full, current, candidate], start, end) + for (const [baseline, evaluated] of [[full, current], [full, candidate], [current, candidate]]) { + const comparison = compareStageOutputs(baseline, evaluated, refs, start, end, support) + expect(comparison.commonSupportScope).toBe('all_configurations') + expect(comparison.commonSupport.baseline.pairedDurationMs).toBe(10000) + expect(comparison.commonSupport.candidate.pairedDurationMs).toBe(10000) + } + const comparison = compareStageOutputs(full, current, refs, start, end, support) + expect(comparison.ownSupport.baseline.pairedDurationMs).toBe(20000) + expect(comparison.ownSupport.candidate.pairedDurationMs).toBe(20000) + expect(compareStageOutputs(full, current, refs, start, end, []).differences.commonAgreement).toBeNull() +}) diff --git a/src/biometrics/backtest/tests/coordinator-entry.test.ts b/src/biometrics/backtest/tests/coordinator-entry.test.ts new file mode 100644 index 000000000..3c01bc07b --- /dev/null +++ b/src/biometrics/backtest/tests/coordinator-entry.test.ts @@ -0,0 +1,139 @@ +// @vitest-environment node +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import type { performCoordinatorTask } from '../coordinator-tasks' + +const mocks = vi.hoisted(() => ({ task: vi.fn(), open: vi.fn(), close: vi.fn(), pragma: vi.fn(), db: {} })) +vi.mock('../coordinator-tasks', () => ({ performCoordinatorTask: mocks.task })) +vi.mock('better-sqlite3', () => ({ default: class { + constructor(...args: unknown[]) { mocks.open(...args) } + close = mocks.close + pragma = mocks.pragma +} })) +vi.mock('drizzle-orm/better-sqlite3', () => ({ drizzle: () => mocks.db })) +type Options = Parameters[2] +let listeners: Record unknown> +let sent: Array> +let disconnect: ReturnType +let kill: ReturnType +let exit: ReturnType +async function start(budget = 'null') { + const original = process + disconnect = vi.fn(() => listeners.disconnect?.()) + kill = vi.fn() + exit = vi.fn() + vi.stubGlobal('process', Object.assign(Object.create(original), { + argv: ['node', 'coordinator', '/owned/biometrics.db', '/owned/worker.cjs', 'null', budget], + on: (event: string, listener: (...args: unknown[]) => unknown) => { listeners[event] = listener }, + send: (message: Record, callback?: () => void) => { + sent.push(message) + callback?.() + }, + disconnect, kill, exit, + })) + await import('../coordinator-entry') +} +const message = (value: unknown) => listeners.message(value) +beforeEach(() => { + vi.resetModules() + vi.clearAllMocks() + mocks.task.mockReset().mockResolvedValue({ ok: true }) + mocks.open.mockReset() + listeners = {} + sent = [] +}) +afterEach(() => { + vi.unstubAllGlobals() + vi.useRealTimers() +}) + +it('runs once, closes its database, and disconnects without killing completed work', async () => { + await start() + await message(null) + await message({ type: 'unexpected' }) + await message({ type: 'run', task: { type: 'collectArtifacts' } }) + await message({ type: 'run', task: { type: 'execute' } }) + expect(mocks.open).toHaveBeenCalledExactlyOnceWith('/owned/biometrics.db', { fileMustExist: true, timeout: 1000 }) + expect(mocks.pragma).toHaveBeenCalledWith('foreign_keys = ON') + expect(mocks.task).toHaveBeenCalledOnce() + expect(mocks.task.mock.calls[0][2]).toMatchObject({ workerPath: '/owned/worker.cjs', rawLocation: undefined, replayBudget: undefined }) + expect(sent).toEqual([{ type: 'result', result: { ok: true } }]) + expect(mocks.close).toHaveBeenCalledOnce() + expect(disconnect).toHaveBeenCalledOnce() + expect(kill).not.toHaveBeenCalled() + expect(exit).not.toHaveBeenCalled() +}) + +it.each([['valid', true], ['malformed', false], ['timeout', false]] as const)('correlates admission and fails closed for %s receipts', async (kind, allowed) => { + vi.useFakeTimers() + let options!: Options + mocks.task.mockImplementation(async (_db, _task, received: Options) => { + options = received + expect(options.admissionReceipt?.()).toBeNull() + return { admitted: await options.admit() } + }) + await start() + const running = message({ type: 'run', task: {} }) + expect(sent).toEqual([{ type: 'admission', sequence: 1 }]) + await message({ type: 'admission', sequence: 99, allowed: true }) + expect(options.admissionReceipt?.()).toBeNull() + if (kind === 'timeout') await vi.advanceTimersByTimeAsync(2000) + else await message({ type: 'admission', sequence: 1, allowed: kind === 'valid' ? true : 'yes', sample: null }) + await running + expect(sent.at(-1)).toEqual({ type: 'result', result: { admitted: allowed } }) + expect(options.admissionReceipt?.()).toEqual({ allowed, sample: null }) + expect(vi.getTimerCount()).toBe(0) +}) + +it('propagates cancellation and kills its owned group on premature supervisor disconnect', async () => { + await start() + await message({ type: 'cancel' }) + mocks.task.mockImplementation(async (_db, _task, options: Options) => { + expect(options.signal.aborted).toBe(true) + listeners.disconnect() + return null + }) + await message({ type: 'run', task: {} }) + if (process.platform !== 'win32') expect(kill).toHaveBeenCalledWith(-process.pid, 'SIGKILL') + expect(exit).toHaveBeenCalledExactlyOnceWith(1) +}) + +it.each([ + ['known', 'INVALID_INPUT', 'x'.repeat(1000)], + ['unexpected', 'COORDINATOR_FAILED', 'Biometrics coordinator failed'], +])('bounds known error text and suppresses unexpected exports', async (error, code, text) => { + await start() + const { ReferenceInputError } = await import('../../references/time') + mocks.task.mockRejectedValue(error === 'known' ? new ReferenceInputError('INVALID_INPUT', 'x'.repeat(1200)) : new Error('private source export')) + await message({ type: 'run', task: {} }) + expect(sent).toEqual([{ type: 'error', code, message: text }]) + expect(mocks.close).toHaveBeenCalledOnce() + expect(disconnect).toHaveBeenCalledOnce() + expect(exit).not.toHaveBeenCalled() +}) + +it('rejects oversized results without sending their contents', async () => { + mocks.task.mockResolvedValue({ private: 'x'.repeat(17 * 1024 * 1024) }) + await start() + await message({ type: 'run', task: {} }) + expect(sent).toEqual([{ type: 'error', code: 'COORDINATOR_FAILED', message: 'Biometrics coordinator failed' }]) + expect(mocks.close).toHaveBeenCalledOnce() +}) + +it('validates execution limits before opening storage', async () => { + await start('{"jobWallMs":-1}') + await message({ type: 'run', task: {} }) + expect(mocks.open).not.toHaveBeenCalled() + expect(mocks.task).not.toHaveBeenCalled() + expect(sent[0]).toMatchObject({ type: 'error', code: 'COORDINATOR_FAILED' }) +}) + +it('reports database-open failure without attempting work or leaking details', async () => { + mocks.open.mockImplementation(() => { + throw new Error('/private/path') + }) + await start() + await message({ type: 'run', task: {} }) + expect(mocks.task).not.toHaveBeenCalled() + expect(mocks.close).not.toHaveBeenCalled() + expect(sent).toEqual([{ type: 'error', code: 'COORDINATOR_FAILED', message: 'Biometrics coordinator failed' }]) +}) diff --git a/src/biometrics/backtest/tests/coordinator-queue.test.ts b/src/biometrics/backtest/tests/coordinator-queue.test.ts new file mode 100644 index 000000000..ca0fe3f0c --- /dev/null +++ b/src/biometrics/backtest/tests/coordinator-queue.test.ts @@ -0,0 +1,111 @@ +// @vitest-environment node +import { afterEach, expect, it, vi } from 'vitest' +import { CoordinatorQueue } from '../coordinator-queue' +const options = { databasePath: '/unused.db', coordinatorPath: '/unused.cjs', workerPath: '/unused-worker.cjs' } +afterEach(() => vi.useRealTimers()) +const report = { type: 'readSummary' as const, input: { id: '00000000-0000-4000-8000-000000000001' } } + +it('serializes report readers and frees the read slot after a failure without stopping replay', async () => { + const children: { task: string, resolve: (value: null) => void, reject: (error: Error) => void }[] = [] + const execute = vi.fn().mockImplementation((_options, task) => new Promise((resolve, reject) => { + children.push({ task: task.type, resolve, reject }) + })) + const queue = new CoordinatorQueue(options, execute) + const replay = queue.request({ type: 'execute', owner: 'test' }) + const failedRead = expect(queue.request(report)).rejects.toThrow('Unreadable report') + const nextRead = queue.request(report) + expect(children.map(child => child.task)).toEqual(['execute', 'readSummary']) + children[1].reject(new Error('Unreadable report')) + await failedRead + expect(children.map(child => child.task)).toEqual(['execute', 'readSummary', 'readSummary']) + children[2].resolve(null) + await nextRead + children[0].resolve(null) + await replay + await queue.stop() +}) + +it('serves sequential report reads while replay occupies the work slot', async () => { + let finishReplay: (() => void) | undefined + const execute = vi.fn().mockImplementation((_options, task) => task.type === 'execute' + ? new Promise((resolve) => { finishReplay = () => resolve(null) }) + : Promise.resolve(null)) + const queue = new CoordinatorQueue(options, execute) + const replay = queue.request({ type: 'execute', owner: 'test' }) + const scan = queue.request({ type: 'inspectRawSource', input: { afterName: null } }) + await queue.request(report) + await queue.request(report) + expect(execute.mock.calls.map(call => call[1].type)).toEqual(['execute', 'readSummary', 'readSummary']) + finishReplay?.() + await Promise.all([replay, scan]) + expect(execute.mock.calls.map(call => call[1].type)).toEqual(['execute', 'readSummary', 'readSummary', 'inspectRawSource']) + await queue.stop() +}) + +it('bounds both lanes and cancels both active children on shutdown without starting pending work', async () => { + const execute = vi.fn().mockImplementation((_options, _task, control) => new Promise((resolve) => { + control.signal.addEventListener('abort', () => resolve(null)) + })) + const queue = new CoordinatorQueue(options, execute) + const reads = Array.from({ length: 7 }, () => queue.request(report).catch(error => error)) + await expect(queue.request(report)).rejects.toMatchObject({ code: 'RESOURCE_LIMIT' }) + const replay = queue.request({ type: 'execute', owner: 'test' }) + await expect(queue.request({ type: 'collectArtifacts' })).rejects.toMatchObject({ code: 'RESOURCE_LIMIT' }) + expect(execute).toHaveBeenCalledTimes(2) + await queue.stop() + const outcomes = await Promise.all(reads) + await replay + expect(outcomes.slice(1).every(error => error instanceof Error)).toBe(true) + expect(execute.mock.calls.every(call => call[2].signal.aborted)).toBe(true) + expect(execute).toHaveBeenCalledTimes(2) +}) + +it('reserves read capacity while limiting and serializing active plus queued work', async () => { + let release: (() => void) | undefined + const execute = vi.fn().mockImplementationOnce(() => new Promise((resolve) => { + release = () => resolve(null) + })).mockResolvedValue(null) + const queue = new CoordinatorQueue(options, execute) + const requests = Array.from({ length: 7 }, () => queue.request({ type: 'collectArtifacts' })) + await expect(queue.request({ type: 'collectArtifacts' })).rejects.toMatchObject({ code: 'RESOURCE_LIMIT' }) + expect(execute).toHaveBeenCalledTimes(1) + release?.() + await Promise.all(requests) + expect(execute).toHaveBeenCalledTimes(7) + await queue.stop() +}) +it('expires queued requests without executing them and propagates cancellation to active work', async () => { + vi.useFakeTimers() + let release: (() => void) | undefined + const execute = vi.fn().mockImplementation((_options, _task, control) => new Promise((resolve) => { + release = () => resolve(null) + control.signal.addEventListener('abort', () => resolve(null)) + })) + const queue = new CoordinatorQueue(options, execute) + const running = queue.request({ type: 'collectArtifacts' }) + const pending = expect(queue.request({ type: 'collectArtifacts' })).rejects.toThrow('queue wait exceeded') + await vi.advanceTimersByTimeAsync(30000) + await pending + expect(execute).toHaveBeenCalledTimes(1) + await queue.stop() + await running + expect(execute.mock.calls[0][2].signal.aborted).toBe(true) + release?.() + await expect(queue.request({ type: 'collectArtifacts' })).rejects.toThrow('stopped') +}) +it('removes a cancelled queued request without waiting for the active operation', async () => { + let release: (() => void) | undefined + const execute = vi.fn().mockImplementation(() => new Promise((resolve) => { + release = () => resolve(null) + })) + const queue = new CoordinatorQueue(options, execute) + const first = queue.request({ type: 'collectArtifacts' }) + const controller = new AbortController() + const cancelled = queue.request({ type: 'collectArtifacts' }, { signal: controller.signal }) + controller.abort() + await expect(cancelled).rejects.toThrow('cancelled') + release?.() + await first + await queue.stop() + expect(execute).toHaveBeenCalledTimes(1) +}) diff --git a/src/biometrics/backtest/tests/coordinator-tasks.test.ts b/src/biometrics/backtest/tests/coordinator-tasks.test.ts new file mode 100644 index 000000000..c77d90eef --- /dev/null +++ b/src/biometrics/backtest/tests/coordinator-tasks.test.ts @@ -0,0 +1,241 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { build } from 'esbuild' +import { BacktestJobStore } from '../jobs' +import { DatasetStore } from '../datasets' +import { workerBuildHash } from '../child' +import { backtestFixture } from './fixtures/backtest' +import { datasetManifestSchema } from '../contracts' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { mkdtempSync, realpathSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { randomUUID } from 'node:crypto' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import * as schema from '@/src/db/biometrics-schema' +import { performCoordinatorTask, type CoordinatorTask } from '../coordinator-tasks' +import { podRawInspectionLocation, runRawInspectionChild } from '../raw-inspection-child' +import type * as RawInspection from '../raw-inspection-child' +import { runRawCollectionChild } from '../raw-collection-child' + +vi.mock('../raw-inspection-child', async original => ({ ...await original(), runRawInspectionChild: vi.fn() })) +vi.mock('../raw-collection-child', () => ({ runRawCollectionChild: vi.fn() })) +let directory: string, sqlite: Database.Database, db: ReturnType> +let controller: AbortController +let admit: ReturnType Promise>> +const location = { pythonPath: '/trusted/python', modulesPath: '/trusted/modules', archiveDirectory: '/trusted/archive' } +const execute = (task: CoordinatorTask) => performCoordinatorTask(db, task, { workerPath: '/trusted/worker', rawLocation: location, signal: controller.signal, admit }) +const scan = () => execute({ type: 'inspectRawSource', input: { afterName: 'previous.RAW.gz' } }) +const deferred = { status: 'deferred', nextCursor: 'previous.RAW.gz' } +function inspection() { + const hash = 'a'.repeat(64) + return { source_name: 'capture.RAW.gz', retained: false as const, artifact: { name: `${hash}.RAW.gz`, sha256: hash, compressed_bytes: 100, expanded_bytes: 200 }, + inventory: { version: 'retained-raw-inventory-v1' as const, artifact_sha256: hash, + decoded_records: 2, piezo_records: 1, pump_records: 1, usable_dual_frames: 1, empty_channel_frames: 0, + oversize_channel_frames: 0, invalid_timestamps: 0, timestamp_reversals: 0, duplicate_timestamps: 0, + first_event_ms: 1000, last_event_ms: 2000, + channels: { left1: { samples: 10, lost_markers: 0, truncated_tail_bytes: 0 }, right1: { samples: 10, lost_markers: 0, truncated_tail_bytes: 0 } }, + ordered_waveform_available: true, limitations: [] } } +} +beforeEach(() => { + vi.clearAllMocks() + directory = realpathSync(mkdtempSync(join(tmpdir(), 'coordinator-tasks-'))) + sqlite = new Database(join(directory, 'biometrics.db')) + sqlite.pragma('foreign_keys = ON') + db = drizzle(sqlite, { schema }) + migrate(db, { migrationsFolder: 'src/db/biometrics-migrations' }) + controller = new AbortController() + admit = vi.fn().mockResolvedValue(true) + vi.mocked(runRawInspectionChild).mockReset().mockResolvedValue({ status: 'complete', nextCursor: null }) + vi.mocked(runRawCollectionChild).mockReset().mockResolvedValue({ version: 'raw-collection-result-v1', released: 0, orphans: 0 }) +}) +afterEach(() => { + vi.useRealTimers() + sqlite.close() + rmSync(directory, { recursive: true, force: true }) +}) + +it('retains an immutable dataset, releases it, and leaves canonical observations intact', async () => { + const start = 1791673200000 + db.insert(schema.sleepRecords).values({ id: 1, side: 'left', enteredBedAt: new Date(start), leftBedAt: new Date(start + 3600000), sleepDurationSeconds: 3600, timesExitedBed: 0 }).run() + db.insert(schema.vitals).values({ side: 'left', timestamp: new Date(start), heartRate: 65 }).run() + const input = { side: 'left' as const, subjectId: 'fixture', sessionIds: [1], referenceImportIds: [], timezone: 'UTC', idempotencyKey: 'one' } + const dataset = await execute({ type: 'createDataset', input }) as { id: string, contentHash: string } + expect(await execute({ type: 'createDataset', input })).toEqual(dataset) + expect(await execute({ type: 'datasetDescriptor', input: { id: dataset.id } })).toMatchObject({ contentHash: dataset.contentHash, available: true }) + await execute({ type: 'releaseDataset', input: { id: dataset.id } }) + expect(await execute({ type: 'datasetDescriptor', input: { id: dataset.id } })).toMatchObject({ available: false }) + expect(sqlite.prepare('SELECT heart_rate FROM vitals').all()).toEqual([{ heart_rate: 65 }]) +}) + +it.each(['readResult', 'readSummary', 'releaseResult'] as const)('returns no %s for an unknown experiment without creating rows', async (type) => { + const id = randomUUID() + await expect(execute(type === 'releaseResult' ? { type, input: { id, expectedContentHash: 'a'.repeat(64) } } : { type, input: { id } })).rejects.toMatchObject({ code: 'NOT_FOUND' }) + expect(sqlite.prepare('SELECT count(*) AS n FROM biometric_experiments').get()).toEqual({ n: 0 }) +}) + +it('refuses caller-supplied execution paths before inspecting or collecting', async () => { + await expect(performCoordinatorTask(db, { type: 'inspectRawSource', input: { afterName: null }, workerPath: '/untrusted' }, { workerPath: '/trusted', signal: controller.signal, admit })).rejects.toThrow() + expect(runRawInspectionChild).not.toHaveBeenCalled() + expect(admit).not.toHaveBeenCalled() +}) + +it.each(['cancelled', 'denied'] as const)('does not launch a %s inspection or collection', async (reason) => { + if (reason === 'cancelled') controller.abort() + else admit.mockResolvedValue(false) + expect(await scan()).toEqual(deferred) + expect(await execute({ type: 'collectArtifacts' })).toBeNull() + expect(runRawInspectionChild).not.toHaveBeenCalled() + expect(runRawCollectionChild).not.toHaveBeenCalled() +}) + +it('records inspected evidence only after rechecking admission and pages the retained identity', async () => { + vi.mocked(runRawInspectionChild).mockResolvedValue({ status: 'inspected', nextCursor: 'capture.RAW.gz', inspection: inspection() }) + const result = await scan() + expect(result).toMatchObject({ status: 'inspected', source: { inspection: inspection(), availability: 'requires_reverification' } }) + expect(admit).toHaveBeenCalledTimes(2) + expect(await execute({ type: 'pageRawSources', input: { startMs: 1000, endMs: 3000 } })).toMatchObject({ items: [expect.objectContaining({ inspection: inspection() })], nextCursor: null }) +}) + +it('discards inspected evidence when final admission is denied', async () => { + vi.mocked(runRawInspectionChild).mockResolvedValue({ status: 'inspected', nextCursor: 'capture.RAW.gz', inspection: inspection() }) + admit.mockResolvedValueOnce(true).mockResolvedValueOnce(false) + expect(await scan()).toEqual(deferred) + expect(await execute({ type: 'pageRawSources', input: { startMs: 1000, endMs: 3000 } })).toMatchObject({ items: [] }) +}) + +it('passes through archive unavailability and propagates actual inspection failures', async () => { + vi.mocked(runRawInspectionChild).mockResolvedValueOnce({ status: 'archive_unavailable', nextCursor: null }).mockRejectedValueOnce(new Error('decoder failed')) + expect(await scan()).toEqual({ status: 'archive_unavailable', nextCursor: null }) + await expect(scan()).rejects.toThrow('decoder failed') +}) + +it.each(['denied', 'admission_error'] as const)('cancels an in-flight inspection on %s without persisting evidence', async (reason) => { + vi.useFakeTimers() + vi.mocked(runRawInspectionChild).mockImplementation(async (_location, _input, options) => new Promise((_resolve, reject) => { + options?.signal?.addEventListener('abort', () => reject(new Error('cancelled')), { once: true }) + })) + if (reason === 'denied') admit.mockResolvedValueOnce(true).mockResolvedValue(false) + else admit.mockResolvedValueOnce(true).mockRejectedValue(new Error('health unavailable')) + const running = scan() + await vi.advanceTimersByTimeAsync(5000) + expect(await running).toEqual(deferred) + expect(vi.getTimerCount()).toBe(0) +}) + +it('forwards caller cancellation to inspection and removes its health timer', async () => { + vi.useFakeTimers() + vi.mocked(runRawInspectionChild).mockImplementation(async (_location, _input, options) => { + controller.abort() + expect(options?.signal?.aborted).toBe(true) + return { status: 'complete', nextCursor: null } + }) + expect(await scan()).toEqual(deferred) + expect(vi.getTimerCount()).toBe(0) +}) + +it('collects only beside the actual database with fixed deployment paths', async () => { + expect(await execute({ type: 'collectArtifacts' })).toBeNull() + expect(runRawCollectionChild).toHaveBeenCalledExactlyOnceWith({ ...location, databasePath: join(directory, 'biometrics.db'), retainedDirectory: join(directory, 'biometric-raw') }, { signal: expect.any(AbortSignal) }) +}) + +it('cancels collection after a failed health recheck and clears supervision', async () => { + vi.useFakeTimers() + let aborted = false + vi.mocked(runRawCollectionChild).mockImplementation(async (_location, options) => new Promise((resolve) => { + options?.signal?.addEventListener('abort', () => { + aborted = true + resolve({ version: 'raw-collection-result-v1', released: 0, orphans: 0 }) + }, { once: true }) + })) + admit.mockResolvedValueOnce(true).mockRejectedValue(new Error('health unavailable')) + const running = execute({ type: 'collectArtifacts' }) + await vi.advanceTimersByTimeAsync(5000) + expect(await running).toBeNull() + expect(aborted).toBe(true) + expect(vi.getTimerCount()).toBe(0) +}) + +it('dispatches a retained evaluation through publication, paged traces and hash-fenced release', async () => { + const start = 1791673200000 + db.insert(schema.sleepRecords).values({ id: 1, side: 'left', enteredBedAt: new Date(start), leftBedAt: new Date(start + 600000), sleepDurationSeconds: 600, timesExitedBed: 0 }).run() + db.insert(schema.vitals).values({ side: 'left', timestamp: new Date(start), heartRate: 65 }).run() + const dataset = new DatasetStore(db).create({ side: 'left', subjectId: 'fixture', sessionIds: [1], referenceImportIds: [], timezone: 'UTC', idempotencyKey: 'evaluation' }) + const fixture = backtestFixture().job + const experiment = await execute({ type: 'createExperiment', input: { datasetId: dataset.id, protocol: fixture.protocol, idempotencyKey: 'experiment' } }) as { id: string } + const workerPath = join(directory, 'worker.cjs') + await build({ entryPoints: ['src/biometrics/backtest/worker-entry.ts'], outfile: workerPath, bundle: true, platform: 'node', target: 'node22', format: 'cjs' }) + const jobs = new BacktestJobStore(db) + const job = jobs.enqueue({ experimentId: experiment.id, evaluatorBuildHash: await workerBuildHash(workerPath), configs: fixture.configs, split: 'tuning', idempotencyKey: 'job' }) + const before = sqlite.prepare('SELECT * FROM vitals').all() + expect(await performCoordinatorTask(db, { type: 'execute', owner: 'test' }, { workerPath, signal: controller.signal, admit })).toEqual({ state: 'succeeded', jobId: job.id }) + const result = await execute({ type: 'readResult', input: { id: job.id } }) as NonNullable> + expect(result.available).toBe(true) + expect(await execute({ type: 'readSummary', input: { id: job.id } })).toMatchObject({ contentHash: result.contentHash, contentHashScope: 'complete_retained_report' }) + const trace = { id: job.id, configuration: 'default' as const, expectedContentHash: result.contentHash } + expect(await execute({ type: 'readTrace', input: trace })).toMatchObject({ provenance: 'versioned_stage_replay', downsampled: false }) + await expect(execute({ type: 'readNumericTrace', input: { ...trace, expectedContentHash: 'f'.repeat(64) } })).rejects.toMatchObject({ code: 'REVISION_CONFLICT' }) + const cohortId = datasetManifestSchema.parse(dataset.manifest).splitProtocol.assignments[0].cohortId + expect(await execute({ type: 'readReferenceTrace', input: { id: job.id, expectedContentHash: result.contentHash, cohortId, metric: 'heart_rate' } })).toMatchObject({ totalItems: 0, items: [], provenance: 'frozen_dataset_reference' }) + await expect(execute({ type: 'releaseResult', input: { id: job.id, expectedContentHash: 'f'.repeat(64) } })).rejects.toMatchObject({ code: 'REVISION_CONFLICT' }) + expect(await execute({ type: 'releaseResult', input: { id: job.id, expectedContentHash: result.contentHash } })).toMatchObject({ available: false, report: null }) + await expect(execute({ type: 'readTrace', input: trace })).rejects.toMatchObject({ code: 'RESULT_RELEASED' }) + expect(sqlite.prepare('SELECT * FROM vitals').all()).toEqual(before) +}) + +it('does not overlap slow health rechecks and waits for a pending denial before committing inspection', async () => { + vi.useFakeTimers() + let resolveHealth!: (allowed: boolean) => void + admit.mockResolvedValueOnce(true).mockImplementation(() => new Promise((resolve) => { + resolveHealth = resolve + })) + let resolveScan!: (result: Awaited>) => void + vi.mocked(runRawInspectionChild).mockImplementation(() => new Promise((resolve) => { + resolveScan = resolve + })) + const running = scan() + await vi.advanceTimersByTimeAsync(15000) + expect(admit).toHaveBeenCalledTimes(2) + resolveScan({ status: 'complete', nextCursor: null }) + resolveHealth(false) + expect(await running).toEqual(deferred) + expect(vi.getTimerCount()).toBe(0) +}) + +it('uses the pod deployment paths when no location override is supplied', async () => { + const defaults = { workerPath: '/trusted/worker', signal: controller.signal, admit } + expect(await performCoordinatorTask(db, { type: 'inspectRawSource', input: { afterName: null } }, defaults)).toEqual({ status: 'complete', nextCursor: null }) + expect(runRawInspectionChild).toHaveBeenCalledWith(podRawInspectionLocation, { afterName: null }, { signal: expect.any(AbortSignal) }) + expect(await performCoordinatorTask(db, { type: 'collectArtifacts' }, defaults)).toBeNull() + expect(runRawCollectionChild).toHaveBeenCalledWith({ ...podRawInspectionLocation, databasePath: join(directory, 'biometrics.db'), retainedDirectory: join(directory, 'biometric-raw') }, { signal: expect.any(AbortSignal) }) +}) + +it('does not overlap slow health rechecks while collecting', async () => { + vi.useFakeTimers() + let resolveHealth!: (allowed: boolean) => void + admit.mockResolvedValueOnce(true).mockImplementation(() => new Promise((resolve) => { + resolveHealth = resolve + })) + let finishCollection!: () => void + vi.mocked(runRawCollectionChild).mockImplementation(() => new Promise((resolve) => { + finishCollection = () => resolve({ version: 'raw-collection-result-v1', released: 0, orphans: 0 }) + })) + const running = execute({ type: 'collectArtifacts' }) + await vi.advanceTimersByTimeAsync(15000) + expect(admit).toHaveBeenCalledTimes(2) + resolveHealth(true) + await vi.advanceTimersByTimeAsync(5000) + expect(admit).toHaveBeenCalledTimes(3) + finishCollection() + resolveHealth(true) + expect(await running).toBeNull() + expect(vi.getTimerCount()).toBe(0) +}) + +it('dispatches waveform publication and shadow reports through the same task surface', async () => { + admit.mockResolvedValue(false) + const selection = { parentDatasetId: randomUUID(), expectedContentHash: 'a'.repeat(64), cohorts: [{ cohortId: 'night', sourceIds: ['b'.repeat(64)] }], idempotencyKey: 'waveform' } + expect(await execute({ type: 'createWaveformDataset', input: selection })).toEqual({ status: 'deferred' }) + await expect(execute({ type: 'readShadowReport', input: { id: randomUUID() } })).rejects.toMatchObject({ code: 'NOT_FOUND' }) +}) diff --git a/src/biometrics/backtest/tests/coordinator.test.ts b/src/biometrics/backtest/tests/coordinator.test.ts new file mode 100644 index 000000000..58c2c5c96 --- /dev/null +++ b/src/biometrics/backtest/tests/coordinator.test.ts @@ -0,0 +1,288 @@ +import { withBiometricBackgroundSlot } from '../../background-admission' +import * as cleanup from '../../shadow/process-cleanup' +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { execFile } from 'node:child_process' +import { mkdtemp, mkdir, rm, writeFile, readFile, utimes } from 'node:fs/promises' +import { join, resolve } from 'node:path' +import { tmpdir } from 'node:os' +import { promisify } from 'node:util' +import { afterAll, beforeAll, expect, it, vi } from 'vitest' +import { runCoordinatorTask } from '../coordinator' +import { BacktestJobStore } from '../jobs' +import { ShadowStore } from '../../shadow/store' +import { dataHash } from '../../algorithms/canonical' +import { DatasetStore } from '../datasets' +import { workerBuildHash } from '../child' +import { backtestFixture } from './fixtures/backtest' +import * as schema from '@/src/db/biometrics-schema' +let directory: string +let databasePath: string +let coordinatorPath: string +let workerPath: string + +it('computes shadow health and reference agreement in the shipped coordinator without canonical writes', async () => { + const sqlite = new Database(databasePath) + const at = 1700000000000, hash = 'a'.repeat(64) + const store = new ShadowStore(sqlite, { maximumRuns: 10, maximumPoints: 10, maximumBytes: 100000 }, () => at) + const run = store.create({ schema: 'biometric-shadow-plan-v1', side: 'left', subjectId: 'shadow-fixture', startMs: at, endMs: at + 1000, + candidateConfigHash: hash, activeConfigHash: hash, engineBuildHash: hash, qualificationHash: hash, + maximumPoints: 10, maximumBytes: 10000, metrics: ['heart_rate'], + monitoring: { expectedIntervalMs: 1000, maximumGapMs: 2000, heartbeatStaleMs: 2000, minimumCandidateCoverage: 0.8, maximumCoverageLoss: 0.1, minimumQuality: 0.8, + alignment: { version: 'reference-alignment-v1', pointToleranceMs: 0, clockOffsetMs: 0, offsetSource: 'fixed' } } }, 'shadow-report') + const lease = store.claim(run.id, 10000, () => {}) + store.append(lease, { sequence: 0, side: 'left', timestampMs: at, inputHash: null, candidateConfigHash: hash, activeConfigHash: hash, + readings: [{ metric: 'heart_rate', candidate: { value: 62, quality: 1 }, active: { value: 60, quality: 1 } }] }) + store.finish(lease, 'completed', 'complete') + const before = sqlite.prepare('SELECT * FROM vitals').all() + try { + const reference = { side: 'left' as const, subjectId: 'shadow-fixture', source: 'test', rows: [{ recordKey: 'hr', metric: 'heart_rate', value: 63, unit: 'bpm', timestampMs: at }] } + const report = await runCoordinatorTask(options(), { type: 'readShadowReport', input: { id: run.id, reference } }) + expect(report.metrics[0].comparison.differences.commonMae).toBe(-2) + expect(report.metrics[0].comparison.commonSupport.candidate.pairedCount).toBe(1) + expect(report.interpretation).toBe('prospective_observation_and_reference_agreement_not_activation_authorization') + const { reportHash, ...body } = report + expect(reportHash).toBe(dataHash(body)) + expect(sqlite.prepare('SELECT * FROM vitals').all()).toEqual(before) + await expect(runCoordinatorTask(options(), { type: 'readShadowReport', input: { id: run.id, reference: { ...reference, side: 'right' } } })).rejects.toMatchObject({ code: 'SUBJECT_MISMATCH' }) + } + finally { sqlite.close() } +}) +beforeAll(async () => { + directory = await mkdtemp(join(tmpdir(), 'biometric-coordinator-test-')) + await promisify(execFile)(process.execPath, ['scripts/build/biometrics-worker.mjs', directory]) + databasePath = join(directory, 'synthetic.db') + coordinatorPath = join(directory, 'coordinator.cjs') + workerPath = join(directory, 'worker.cjs') + const sqlite = new Database(databasePath) + sqlite.pragma('journal_mode = WAL') + const db = drizzle(sqlite, { schema }) + migrate(db, { migrationsFolder: 'src/db/biometrics-migrations' }) + const start = Date.UTC(2026, 0, 1) + db.insert(schema.sleepRecords).values({ id: 1, side: 'left', enteredBedAt: new Date(start), leftBedAt: new Date(start + 600000), sleepDurationSeconds: 600, timesExitedBed: 0 }).run() + db.insert(schema.vitals).values({ side: 'left', timestamp: new Date(start), heartRate: 60 }).run() + sqlite.close() +}) +afterAll(async () => rm(directory, { recursive: true, force: true })) +const options = () => ({ databasePath, coordinatorPath, workerPath, modulePath: resolve('node_modules') }) +const id = '11111111-1111-4111-8111-111111111111' +it('creates retained inputs off-process and preserves idempotent retries and source observations', async () => { + const input = { side: 'left' as const, subjectId: 'fixture', sessionIds: [1], referenceImportIds: [], timezone: 'UTC', idempotencyKey: 'coordinator-test' } + const first = await runCoordinatorTask(options(), { type: 'createDataset', input }) + const retry = await runCoordinatorTask(options(), { type: 'createDataset', input }) + expect(retry).toEqual(first) + expect(first.available).toBe(true) + const db = new Database(databasePath, { readonly: true }) + expect(db.prepare('SELECT count(*) n FROM vitals').get()).toEqual({ n: 1 }) + expect(db.prepare('SELECT count(*) n FROM biometric_datasets').get()).toEqual({ n: 1 }) + db.close() +}) +it('preserves known error codes without leaking database diagnostics', async () => { + await expect(runCoordinatorTask(options(), { type: 'readTrace', input: { id, configuration: 'default', expectedContentHash: 'a'.repeat(64) } })).rejects.toMatchObject({ code: 'NOT_FOUND' }) + await expect(runCoordinatorTask({ ...options(), databasePath: join(directory, 'missing.db') }, { type: 'readResult', input: { id } })).rejects.toMatchObject({ code: 'COORDINATOR_FAILED', message: 'Biometrics coordinator failed' }) +}) +it('requires fresh parent admission and supports a successful idle execution', async () => { + let checks = 0 + expect(await runCoordinatorTask(options(), { type: 'execute', owner: 'test' })).toEqual({ state: 'deferred' }) + expect(await runCoordinatorTask(options(), { type: 'execute', owner: 'test' }, { admit: async () => { + checks++ + return true + } })).toEqual({ state: 'idle' }) + expect(checks).toBe(1) +}) +it('keeps the calling event loop responsive during synchronous coordinator work', async () => { + const busy = join(directory, 'busy.cjs') + await writeFile(busy, 'process.on(\'message\',()=>{const end=Date.now()+500;while(Date.now()process.disconnect())})') + let ticks = 0 + const timer = setInterval(() => ticks++, 25) + try { + expect(await runCoordinatorTask({ ...options(), coordinatorPath: busy }, { type: 'collectArtifacts' })).toBeNull() + expect(ticks).toBeGreaterThan(4) + } + finally { clearInterval(timer) } +}) +it('bounds a stalled coordinator and permits subsequent work', async () => { + const stalled = join(directory, 'stalled.cjs') + await writeFile(stalled, 'process.on("message",()=>{});setInterval(()=>{},1000)') + await expect(runCoordinatorTask({ ...options(), coordinatorPath: stalled, timeoutMs: 100 }, { type: 'collectArtifacts' })).rejects.toThrow('time budget') + expect(await runCoordinatorTask(options(), { type: 'collectArtifacts' })).toBeNull() +}) +it('allows an explicit longer execution budget without extending read deadlines', async () => { + const fixture = join(directory, 'budget.cjs') + const replayBudget = { jobWallMs: 300000, waveformWallMs: 200000, waveformCpuSeconds: 180 } + await writeFile(fixture, `process.on('message',()=>{process.send({type:'result',result:JSON.parse(process.argv[5])},()=>process.disconnect())})`) + const configured = { ...options(), coordinatorPath: fixture, replayBudget } + expect(await runCoordinatorTask({ ...configured, timeoutMs: 200000 }, { type: 'execute', owner: 'test' })).toEqual(replayBudget) + expect(await runCoordinatorTask(configured, { type: 'collectArtifacts' })).toBeNull() + expect(() => runCoordinatorTask({ ...configured, timeoutMs: 200000 }, { type: 'collectArtifacts' })).toThrow('Invalid coordinator timeout') + expect(() => runCoordinatorTask({ ...configured, timeoutMs: 300001 }, { type: 'execute', owner: 'test' })).toThrow('Invalid coordinator timeout') + for (const changed of [{ jobWallMs: 500 }, { waveformWallMs: 400000 }, { waveformCpuSeconds: 201 }, { jobWallMs: 172800001 }]) { + expect(() => runCoordinatorTask({ ...configured, replayBudget: { ...replayBudget, ...changed } }, { type: 'execute', owner: 'test' })).toThrow() + } +}) +it('enforces the configured job deadline when no test timeout overrides it', async () => { + const stalled = join(directory, 'budget-stalled.cjs') + await writeFile(stalled, 'process.on("message",()=>{});setInterval(()=>{},1000)') + const began = performance.now() + await expect(runCoordinatorTask({ ...options(), coordinatorPath: stalled, + replayBudget: { jobWallMs: 1000, waveformWallMs: 1000, waveformCpuSeconds: 1 } }, + { type: 'execute', owner: 'test' })).rejects.toThrow('time budget') + expect(performance.now() - began).toBeLessThan(3000) +}) +it.skipIf(process.platform === 'win32')('kills evaluator descendants when a coordinator crashes', async () => { + const script = join(directory, 'crash.cjs'), pidPath = join(directory, 'descendant.pid') + await writeFile(script, `const {spawn}=require('node:child_process');const fs=require('node:fs');process.on('message',()=>{const child=spawn(process.execPath,['-e','setInterval(()=>{},1000)'],{stdio:'ignore'});fs.writeFileSync(${JSON.stringify(pidPath)},String(child.pid));process.exit(1)})`) + await expect(runCoordinatorTask({ ...options(), coordinatorPath: script }, { type: 'collectArtifacts' })).rejects.toThrow('complete result') + const pid = Number(await readFile(pidPath, 'utf8')) + let alive = true + for (let attempt = 0; attempt < 20 && alive; attempt++) { + try { + process.kill(pid, 0) + if (process.platform === 'linux' && (await readFile(`/proc/${pid}/stat`, 'utf8')).split(' ')[2] === 'Z') alive = false + } + catch { alive = false } + if (alive) await new Promise(resolve => setTimeout(resolve, 25)) + } + expect(alive).toBe(false) +}) + +it('runs a leased evaluation and preserves report identity across the process boundary', async () => { + const sqlite = new Database(databasePath) + const db = drizzle(sqlite, { schema }) + const dataset = new DatasetStore(db).create({ side: 'left', subjectId: 'fixture', sessionIds: [1], referenceImportIds: [], timezone: 'UTC', idempotencyKey: 'execute-dataset' }) + const jobs = new BacktestJobStore(db) + const fixture = backtestFixture().job + const experiment = jobs.createExperiment(dataset.id, fixture.protocol) + const job = jobs.enqueue({ experimentId: experiment.id, evaluatorBuildHash: await workerBuildHash(workerPath), configs: fixture.configs, split: 'tuning', idempotencyKey: 'execute-job' }) + try { + expect(await runCoordinatorTask(options(), { type: 'execute', owner: 'integration' }, { healthAdmission: async () => ({ allowed: true, sample: { qualificationId: 'integration-policy', checkedAtMs: Date.now(), ingestionAgeMs: 123 } }) })).toEqual({ state: 'succeeded', jobId: job.id }) + const result = await runCoordinatorTask(options(), { type: 'readResult', input: { id: job.id } }) + expect(result?.contentHash).toBe(jobs.result(job.id)?.contentHash) + expect(result?.available).toBe(true) + expect(result?.report).toMatchObject({ jobOperations: { memory: { scope: 'sum_rss_of_replay_process_group', attempts: expect.any(Number), exactKernelPeak: false } } }) + expect(result?.report).toMatchObject({ jobOperations: { ingestion: { allChecksMeasured: true, samples: 2, maximumObservedAgeMs: 123, qualificationId: 'integration-policy' } } }) + const summary = await runCoordinatorTask(options(), { type: 'readSummary', input: { id: job.id } }) + expect(summary).toMatchObject({ contentHash: result?.contentHash, contentHashScope: 'complete_retained_report' }) + expect(JSON.stringify(summary?.report)).not.toContain('"blocks"') + expect(JSON.stringify(result?.report)).toContain('"blocks"') + } + finally { sqlite.close() } +}) + +it('cooperatively interrupts a running lease before shutdown completes', async () => { + const slow = join(directory, 'slow-evaluator.cjs') + await writeFile(slow, 'process.stdin.resume();setInterval(()=>{},1000)') + const sqlite = new Database(databasePath) + const db = drizzle(sqlite, { schema }) + const dataset = new DatasetStore(db).create({ side: 'left', subjectId: 'fixture', sessionIds: [1], referenceImportIds: [], timezone: 'UTC', idempotencyKey: 'cancel-dataset' }) + const jobs = new BacktestJobStore(db), fixture = backtestFixture().job + const experiment = jobs.createExperiment(dataset.id, fixture.protocol) + const job = jobs.enqueue({ experimentId: experiment.id, evaluatorBuildHash: await workerBuildHash(slow), configs: fixture.configs, split: 'tuning', idempotencyKey: 'cancel-job' }) + const controller = new AbortController() + const running = runCoordinatorTask({ ...options(), workerPath: slow }, { type: 'execute', owner: 'cancel-test' }, { signal: controller.signal, admit: async () => true }) + const rejected = expect(running).rejects.toThrow('cancelled') + try { + for (let i = 0; i < 100 && jobs.get(job.id).state !== 'running'; i++) await new Promise(resolve => setTimeout(resolve, 10)) + expect(jobs.get(job.id).state).toBe('running') + controller.abort() + await rejected + expect(jobs.get(job.id)).toMatchObject({ state: 'interrupted', error: 'Worker host stopped' }) + expect(jobs.result(job.id)).toBeNull() + } + finally { + controller.abort() + await running.catch(() => {}) + sqlite.close() + } +}) + +it('inspects actual archives through the isolated coordinator and persists only admitted evidence', async () => { + const archiveDirectory = join(directory, 'archive') + await mkdir(archiveDirectory) + const pythonPath = resolve('modules/piezo-processor/.venv/bin/python') + const rawLocation = { pythonPath, modulesPath: resolve('modules'), archiveDirectory } + await promisify(execFile)(pythonPath, ['-c', 'import gzip,sys;from pathlib import Path;Path(sys.argv[1]).write_bytes(gzip.compress(bytes.fromhex(\'a26373657101646461746148a164747970656178\'),mtime=0))', join(archiveDirectory, 'a.RAW.gz')]) + const scan = () => ({ type: 'inspectRawSource' as const, input: { afterName: null } }) + expect(await runCoordinatorTask({ ...options(), rawLocation }, scan())).toEqual({ status: 'deferred', nextCursor: null }) + let checks = 0 + expect(await runCoordinatorTask({ ...options(), rawLocation }, scan(), { admit: async () => ++checks === 1 })).toEqual({ status: 'deferred', nextCursor: null }) + const sqlite = new Database(databasePath) + try { + expect(sqlite.prepare('SELECT count(*) n FROM biometric_raw_sources').get()).toEqual({ n: 0 }) + const result = await runCoordinatorTask({ ...options(), rawLocation }, scan(), { admit: async () => true }) + expect(result.status).toBe('inspected') + if (result.status !== 'inspected') throw new Error('Missing inspection') + expect(result.source).toMatchObject({ availability: 'requires_reverification', inspection: { source_name: 'a.RAW.gz', retained: false, inventory: { decoded_records: 1 } } }) + expect(sqlite.prepare('SELECT count(*) n FROM biometric_raw_sources').get()).toEqual({ n: 1 }) + expect(sqlite.prepare('SELECT count(*) n FROM biometric_raw_artifacts').get()).toEqual({ n: 0 }) + expect(await runCoordinatorTask(options(), { type: 'pageRawSources', input: { startMs: 0, endMs: 1 } })).toEqual({ items: [], nextCursor: null }) + await writeFile(join(archiveDirectory, 'a.RAW.gz'), 'corrupt') + expect(await runCoordinatorTask({ ...options(), rawLocation }, scan(), { admit: async () => true })).toEqual({ status: 'source_unavailable', nextCursor: 'a.RAW.gz' }) + expect(sqlite.prepare('SELECT unavailable_at_ms unavailable FROM biometric_raw_sources').get()).toEqual({ unavailable: null }) + expect(await runCoordinatorTask({ ...options(), rawLocation }, { type: 'inspectRawSource', input: { afterName: 'a.RAW.gz' } }, { admit: async () => true })).toEqual({ status: 'complete', nextCursor: null }) + } + finally { sqlite.close() } +}) + +it('cancels archive inspection when ongoing resource admission is withdrawn', async () => { + const pythonPath = join(directory, 'slow-inspection') + await writeFile(pythonPath, `#!${process.execPath}\nprocess.stdin.resume();setInterval(()=>{},1000);setTimeout(()=>process.exit(91),10000).unref();`, { mode: 0o700 }) + let checks = 0 + expect(await runCoordinatorTask({ ...options(), rawLocation: { pythonPath, modulesPath: directory, archiveDirectory: directory } }, + { type: 'inspectRawSource', input: { afterName: null } }, { admit: async () => ++checks === 1 })).toEqual({ status: 'deferred', nextCursor: null }) + expect(checks).toBe(2) +}, 10000) + +it('collects aged RAW orphans through the admitted coordinator without touching fresh files', async () => { + const retained = join(directory, 'biometric-raw') + await mkdir(retained, { recursive: true }) + const old = join(retained, `${'a'.repeat(64)}.RAW.gz`), fresh = join(retained, `${'b'.repeat(64)}.RAW.gz`) + await writeFile(old, 'old unpublished copy') + await writeFile(fresh, 'fresh unpublished copy') + await utimes(old, 0, 0) + const rawLocation = { pythonPath: resolve('modules/piezo-processor/.venv/bin/python'), modulesPath: resolve('modules'), archiveDirectory: directory } + await runCoordinatorTask({ ...options(), rawLocation }, { type: 'collectArtifacts' }, { admit: async () => false }) + expect(await readFile(old, 'utf8')).toBe('old unpublished copy') + await runCoordinatorTask({ ...options(), rawLocation }, { type: 'collectArtifacts' }, { admit: async () => true }) + await expect(readFile(old)).rejects.toMatchObject({ code: 'ENOENT' }) + expect(await readFile(fresh, 'utf8')).toBe('fresh unpublished copy') +}) + +it('does not fall back to permissive boolean admission when the health callback denies or fails', async () => { + for (const healthAdmission of [async () => ({ allowed: false, sample: null }), async () => { + throw new Error('missing health') + }]) { + expect(await runCoordinatorTask(options(), { type: 'execute', owner: 'deny-health' }, { admit: async () => true, healthAdmission })).toEqual({ state: 'deferred' }) + } +}) + +it('shares background capacity with shadow attempts while retaining report access', async () => { + let finish!: () => void + const busy = withBiometricBackgroundSlot('work', () => new Promise((resolve) => { + finish = resolve + })) + try { + for (const task of [{ type: 'execute', owner: 'test' }, { type: 'collectArtifacts' }] as const) { + await expect(runCoordinatorTask(options(), task)).rejects.toMatchObject({ code: 'RESOURCE_LIMIT' }) + } + await expect(runCoordinatorTask(options(), { type: 'readResult', input: { id: '00000000-0000-4000-8000-000000000001' } })).rejects.toMatchObject({ code: 'NOT_FOUND' }) + } + finally { + finish() + await busy + } + expect(await runCoordinatorTask(options(), { type: 'execute', owner: 'test' })).toEqual({ state: 'deferred' }) +}) +it('does not release background capacity when coordinator cleanup cannot be confirmed', async () => { + const observation = vi.spyOn(cleanup, 'confirmShadowProcessCleanup').mockResolvedValueOnce(false) + try { + await expect(runCoordinatorTask(options(), { type: 'execute', owner: 'test' })).rejects.toMatchObject({ code: 'BIOMETRIC_COORDINATOR_CLEANUP_UNCONFIRMED' }) + await expect(withBiometricBackgroundSlot('work', async () => null)).rejects.toThrow('cleanup is unconfirmed') + } + finally { + observation.mockRestore() + delete (globalThis as typeof globalThis & { __biometricBackgroundSlotsV1?: unknown }).__biometricBackgroundSlotsV1 + } +}) diff --git a/src/biometrics/backtest/tests/coverage.test.ts b/src/biometrics/backtest/tests/coverage.test.ts new file mode 100644 index 000000000..ebe6caed8 --- /dev/null +++ b/src/biometrics/backtest/tests/coverage.test.ts @@ -0,0 +1,19 @@ +// @vitest-environment node +import { expect, it } from 'vitest' +import { describeCoverage } from '../coverage' + +const T = 1791673200000 +it('reports actual timestamps, duplicates and cadence gaps without interpolating', () => { + expect(describeCoverage([T, T, T + 60000, T + 180000, T - 1, T + 600000], T, T + 600000, 60000)).toEqual({ + count: 3, duplicateTimestamps: 1, firstMs: T, lastMs: T + 180000, expectedCadenceMs: 60000, + gaps: [{ startMs: T + 120000, endMs: T + 180000 }, { startMs: T + 240000, endMs: T + 600000 }], gapDurationMs: 420000, + interpretation: 'cadence_availability_not_measured_support' }) + expect(describeCoverage([], T, T + 60000, 60000)).toMatchObject({ count: 0, firstMs: null, lastMs: null, gaps: [{ startMs: T, endMs: T + 60000 }], gapDurationMs: 60000 }) +}) +it('rejects unbounded or non-integer inputs', () => { + expect(() => describeCoverage([], T, T, 60000)).toThrow('Invalid bounded coverage input') + expect(() => describeCoverage([], T, T + 1, 0)).toThrow('Invalid bounded coverage input') + expect(() => describeCoverage([T + 0.5], T, T + 1, 1)).toThrow('Invalid bounded coverage input') + expect(() => describeCoverage([], T + 0.5, T + 1, 1)).toThrow('Invalid bounded coverage input') + expect(() => describeCoverage(Array.from({ length: 100001 }, () => T), T, T + 1, 1)).toThrow('Invalid bounded coverage input') +}) diff --git a/src/biometrics/backtest/tests/datasets.test.ts b/src/biometrics/backtest/tests/datasets.test.ts new file mode 100644 index 000000000..a545382dd --- /dev/null +++ b/src/biometrics/backtest/tests/datasets.test.ts @@ -0,0 +1,152 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import * as schema from '@/src/db/biometrics-schema' +import { previewReferenceImport } from '../../references/normalize' +import { ReferenceStore } from '../../references/store' +import { DatasetStore } from '../datasets' +let sqlite: Database.Database +let db: ReturnType> +let datasets: DatasetStore +let references: ReferenceStore +const startMs = 1791673200000 +const upload = { side: 'left' as const, subjectId: 'sleeper-A', source: 'fixture', rows: [ + { recordKey: 'hr', metric: 'heart_rate', value: 60, unit: 'bpm', timestampMs: startMs }, +] } +const request = { side: 'left' as const, subjectId: 'sleeper-A', sessionIds: [1], referenceImportIds: [] as string[], timezone: 'America/Los_Angeles', idempotencyKey: 'dataset-1' } +beforeEach(() => { + sqlite = new Database(':memory:') + db = drizzle(sqlite, { schema }) + migrate(db, { migrationsFolder: 'src/db/biometrics-migrations' }) + datasets = new DatasetStore(db) + references = new ReferenceStore(db) + db.insert(schema.sleepRecords).values({ id: 1, side: 'left', enteredBedAt: new Date(startMs), leftBedAt: new Date(startMs + 3600000), sleepDurationSeconds: 3600, timesExitedBed: 0 }).run() + db.insert(schema.vitals).values({ side: 'left', timestamp: new Date(startMs), heartRate: 65 }).run() +}) +afterEach(() => sqlite.close()) + +describe('retained processed datasets', () => { + it('keeps exact input bytes after reference correction/deletion and observation retention', () => { + const first = references.commit({ import: upload, previewHash: previewReferenceImport(upload).summary.contentHash }) + const dataset = datasets.create({ ...request, referenceImportIds: [first.id] }) + const before = datasets.read(dataset.id) + const correction = { ...upload, rows: [{ ...upload.rows[0], value: 90 }] } + references.commit({ import: correction, previewHash: previewReferenceImport(correction).summary.contentHash, supersedesImportId: first.id }) + references.delete(first.id) + db.delete(schema.vitals).run() + db.delete(schema.sleepRecords).run() + const after = datasets.read(dataset.id) + expect(after).toEqual(before) + expect(after.snapshot.sessions[0].vitals[0].heartRate).toBe(65) + expect(after.snapshot.references[0].measurements[0].value).toBe(60) + expect(after.manifest).toMatchObject({ capabilities: { signalReplay: false, stageReplay: true }, splitProtocol: { evidence: 'descriptive_only' } }) + expect(datasets.create({ ...request, referenceImportIds: [first.id] }).id).toBe(dataset.id) + }) + it('rejects cross-side sessions, sleeper mismatches, duplicate revisions and changed idempotency requests', () => { + expect(() => datasets.create({ ...request, side: 'right' })).toThrow('missing or belongs') + const first = references.commit({ import: upload, previewHash: previewReferenceImport(upload).summary.contentHash }) + expect(() => datasets.create({ ...request, subjectId: 'B', referenceImportIds: [first.id] })).toThrow('match') + const correction = { ...upload, rows: [{ ...upload.rows[0], value: 70 }] } + const second = references.commit({ import: correction, previewHash: previewReferenceImport(correction).summary.contentHash, supersedesImportId: first.id }) + expect(() => datasets.create({ ...request, referenceImportIds: [first.id, second.id] })).toThrow('one revision') + datasets.create(request) + expect(() => datasets.create({ ...request, timezone: 'UTC' })).toThrow('different selection') + }) + it('detects corrupted retained bytes and enforces aggregate storage quota atomically', () => { + const dataset = datasets.create(request) + sqlite.prepare('UPDATE biometric_datasets SET snapshot = json_set(snapshot, ?, ?) WHERE id = ?') + .run('$.sessions[0].vitals[0].heartRate', 99, dataset.id) + expect(() => datasets.read(dataset.id)).toThrow('integrity') + sqlite.prepare('UPDATE biometric_datasets SET byte_length = ?').run(64 * 1024 * 1024) + expect(() => datasets.create({ ...request, idempotencyKey: 'second' })).toThrow('storage budget') + expect(datasets.list('left')).toHaveLength(1) + expect(datasets.list('right')).toEqual([]) + }) + it('retains and verifies realistic multi-night rows beyond the configuration hash node budget', () => { + db.delete(schema.vitals).run() + db.delete(schema.sleepRecords).run() + const insert = sqlite.prepare('INSERT INTO vitals (side, timestamp, heart_rate) VALUES (?, ?, ?)') + sqlite.transaction(() => { + for (let night = 1; night <= 3; night++) { + const entered = startMs + night * 86400000 + db.insert(schema.sleepRecords).values({ id: night, side: 'left', enteredBedAt: new Date(entered), leftBedAt: new Date(entered + 43200000), sleepDurationSeconds: 43200, timesExitedBed: 0 }).run() + for (let minute = 0; minute < 720; minute++) insert.run('left', entered / 1000 + minute * 60, 60 + (minute % 10) / 10) + } + })() + const dataset = datasets.create({ ...request, sessionIds: [1, 2, 3] }) + expect(datasets.read(dataset.id).snapshot.sessions.reduce((total, session) => total + session.vitals.length, 0)).toBe(2160) + expect(datasets.descriptor(dataset.id).contentHash).toBe(dataset.contentHash) + }) + + it('records exact processing bundles when available and keeps missing provenance explicit', () => { + db.insert(schema.biometricProcessingRuns).values({ id: 'run-1', side: 'left', kind: 'live', bundle: { version: 'fixture' }, + configHash: 'fixture-hash', inputIdentity: { origin: 'fixture' }, startedAtMs: startMs }).run() + sqlite.prepare('UPDATE vitals SET processing_run_id = ?').run('run-1') + const frozen = datasets.read(datasets.create(request).id) + expect(frozen.snapshot.processingRuns[0]).toMatchObject({ id: 'run-1', calibrationHash: null, bundle: { version: 'fixture' } }) + expect(frozen.snapshot.sessions[0].vitals[0].processingRunId).toBe('run-1') + expect(frozen.manifest).toMatchObject({ coverage: [{ vitals: { count: 1, gapDurationMs: 3540000 } }] }) + }) + + it('rejects duplicate selections and processed snapshots beyond the retained row budget', () => { + expect(() => datasets.create({ ...request, sessionIds: [1, 1] })).toThrow('once') + expect(() => datasets.create({ ...request, referenceImportIds: ['11111111-1111-4111-8111-111111111111', '11111111-1111-4111-8111-111111111111'] })).toThrow('once') + sqlite.prepare('UPDATE sleep_records SET left_bed_at = ?').run(startMs / 1000 + 30000) + const insert = sqlite.prepare('INSERT INTO movement (side, timestamp, total_movement) VALUES (?, ?, ?)') + sqlite.transaction(() => { + for (let second = 0; second < 20001; second++) insert.run('left', startMs / 1000 + second, 5) + })() + expect(() => datasets.create(request)).toThrow('row budget') + db.delete(schema.movement).run() + db.insert(schema.movement).values({ side: 'left', timestamp: new Date(startMs), totalMovement: 3 }).run() + db.delete(schema.vitals).run() + const vitals = sqlite.prepare('INSERT INTO vitals (side, timestamp, heart_rate) VALUES (?, ?, ?)') + sqlite.transaction(() => { + for (let second = 0; second < 20001; second++) vitals.run('left', startMs / 1000 + second, 60) + })() + expect(() => datasets.create(request)).toThrow('row budget') + db.delete(schema.vitals).run() + expect(datasets.read(datasets.create(request).id).snapshot.sessions[0].movement).toMatchObject([{ timestampMs: startMs, totalMovement: 3 }]) + }) + + it('copies every reference page and sorts processing runs deterministically', () => { + const rows = Array.from({ length: 503 }, (_, i) => ({ recordKey: String(i), metric: 'heart_rate', unit: 'bpm', value: 60, timestampMs: startMs + i * 1000 })) + const paged = { ...upload, rows } + const imported = references.commit({ import: paged, previewHash: previewReferenceImport(paged).summary.contentHash }) + for (const id of ['run-b', 'run-a']) db.insert(schema.biometricProcessingRuns).values({ id, side: 'left', kind: 'live', bundle: {}, configHash: 'x', inputIdentity: {}, startedAtMs: startMs }).run() + db.insert(schema.vitals).values({ side: 'left', timestamp: new Date(startMs + 1000), heartRate: 66, processingRunId: 'run-b' }).run() + sqlite.prepare('UPDATE vitals SET processing_run_id = ? WHERE processing_run_id IS NULL').run('run-a') + const frozen = datasets.read(datasets.create({ ...request, referenceImportIds: [imported.id] }).id) + expect(frozen.snapshot.references[0].measurements).toHaveLength(503) + expect(frozen.snapshot.processingRuns.map(run => run.id)).toEqual(['run-a', 'run-b']) + }) + + it('reports missing, released and tampered datasets distinctly and pages per side', () => { + const missing = '00000000-0000-4000-8000-000000000000' + expect(() => datasets.read(missing)).toThrow('unavailable') + expect(() => datasets.descriptor(missing)).toThrow('not found') + expect(() => datasets.release(missing)).toThrow('not found') + const dataset = datasets.create(request) + sqlite.prepare('UPDATE biometric_datasets SET content_hash = ? WHERE id = ?').run('0'.repeat(64), dataset.id) + expect(() => datasets.descriptor(dataset.id)).toThrow('integrity') + sqlite.prepare('UPDATE biometric_datasets SET content_hash = ? WHERE id = ?').run(dataset.contentHash, dataset.id) + const released = datasets.release(dataset.id) + expect(released.available).toBe(false) + expect(datasets.descriptor(dataset.id)).toEqual(released) + expect(() => datasets.read(dataset.id)).toThrow('explicitly released') + let clock = Date.now() + 3600000 + datasets = new DatasetStore(db, () => clock++) + for (let index = 0; index < 51; index++) datasets.create({ ...request, idempotencyKey: `page-${index}` }) + const first = datasets.page('left') + expect(first.items).toHaveLength(50) + expect(first.items[0].available).toBe(true) + expect(first.nextCursor).toEqual({ createdAtMs: first.items[49].createdAtMs, id: first.items[49].id }) + const second = datasets.page('left', first.nextCursor ?? undefined) + expect(second.items).toHaveLength(2) + expect(second.items.at(-1)).toMatchObject({ id: dataset.id, available: false }) + expect(second.nextCursor).toBeNull() + expect(datasets.page('right').items).toEqual([]) + }) +}) diff --git a/src/biometrics/backtest/tests/discovery.test.ts b/src/biometrics/backtest/tests/discovery.test.ts new file mode 100644 index 000000000..2242bcfd0 --- /dev/null +++ b/src/biometrics/backtest/tests/discovery.test.ts @@ -0,0 +1,62 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { afterEach, beforeEach, expect, it } from 'vitest' +import * as schema from '@/src/db/biometrics-schema' +import { listReplaySessions, sessionCapabilities } from '../discovery' +let sqlite: Database.Database +let db: ReturnType> +const start = 1791673200000 +function session(id = 1, hours = 1, side: 'left' | 'right' = 'left') { + db.insert(schema.sleepRecords).values({ id, side, enteredBedAt: new Date(start), leftBedAt: new Date(start + hours * 3600000), sleepDurationSeconds: Math.max(0, hours * 3600), timesExitedBed: 0 }).run() +} +beforeEach(() => { + sqlite = new Database(':memory:') + db = drizzle(sqlite, { schema }) + migrate(db, { migrationsFolder: 'src/db/biometrics-migrations' }) +}) +afterEach(() => sqlite.close()) +it('pages equal start times without duplicates and keeps sides separate', () => { + for (let id = 1; id <= 22; id++) session(id) + session(23, 1, 'right') + const first = listReplaySessions(db, 'left') + expect(first.items.map(row => row.id)).toEqual(Array.from({ length: 20 }, (_, i) => 22 - i)) + expect(first.nextCursor).toEqual({ startMs: start, id: 3 }) + if (!first.nextCursor) throw new Error('Expected another page') + const next = listReplaySessions(db, 'left', first.nextCursor) + expect(next.items.map(row => row.id)).toEqual([2, 1]) + expect(next.nextCursor).toBeNull() + expect(listReplaySessions(db, 'right').items.map(row => row.id)).toEqual([23]) +}) +it('reports an empty session without implying retained waveforms or stage support', () => { + session() + expect(sessionCapabilities(db, 'left', 1)).toMatchObject({ comparison: false, stageReplay: false, signalReplay: false, + processed: { legacyUnknownProvenanceCount: 0, vitals: { count: 0 }, movement: { count: 0 } }, referenceMetrics: [], + limitations: expect.arrayContaining(['No stored vitals are available for comparison or stage replay.']) }) + expect(() => sessionCapabilities(db, 'right', 1)).toThrow('not found for this side') +}) +it.each([0, -1, 37])('rejects invalid or over-budget session duration %s hours', (hours) => { + session(1, hours) + expect(() => sessionCapabilities(db, 'left', 1)).toThrow('boundaries exceed') +}) +it.each([24, 25])('distinguishes comparison from the one-day stage replay limit at %s hours', (hours) => { + session(1, hours) + db.insert(schema.biometricProcessingRuns).values({ id: 'known', side: 'left', kind: 'live', bundle: {}, configHash: 'fixture', inputIdentity: {}, startedAtMs: start }).run() + db.insert(schema.vitals).values([ + { side: 'left', timestamp: new Date(start), heartRate: 60, processingRunId: 'known' }, + { side: 'left', timestamp: new Date(start + 60000), heartRate: 61 }, + { side: 'right', timestamp: new Date(start), heartRate: 70 }, + { side: 'left', timestamp: new Date(start + hours * 3600000), heartRate: 80 }, + ]).run() + db.insert(schema.movement).values([{ side: 'left', timestamp: new Date(start + 120000), totalMovement: 3 }, { side: 'right', timestamp: new Date(start + 120000), totalMovement: 9 }]).run() + const result = sessionCapabilities(db, 'left', 1) + expect(result).toMatchObject({ comparison: true, stageReplay: hours === 24, processed: { legacyUnknownProvenanceCount: 1, vitals: { count: 2 }, movement: { count: 1 } } }) + expect(result.limitations.includes('Stage replay currently requires a merged window of at most one day.')).toBe(hours > 24) +}) +it('bounds aggregate processed rows before returning an incomplete inventory', () => { + session(1, 6) + sqlite.exec(`WITH RECURSIVE rows(n) AS (SELECT 1 UNION ALL SELECT n+1 FROM rows WHERE n < 20001) + INSERT INTO vitals(side,timestamp,heart_rate) SELECT 'left', ${start / 1000} + n, 60 FROM rows`) + expect(() => sessionCapabilities(db, 'left', 1)).toThrow('processed-data row budget') +}) diff --git a/src/biometrics/backtest/tests/evaluator.test.ts b/src/biometrics/backtest/tests/evaluator.test.ts new file mode 100644 index 000000000..7196f435c --- /dev/null +++ b/src/biometrics/backtest/tests/evaluator.test.ts @@ -0,0 +1,150 @@ +// @vitest-environment node +import { describe, expect, it } from 'vitest' +import { configHash, dataHash } from '../../algorithms/canonical' +import { defaultConfig } from '../../algorithms/registry' +import { classifySleepStages } from '@/src/lib/sleep-stages' +import { evaluateBacktest } from '../evaluator' + +import { backtestFixture as fixture } from './fixtures/backtest' +import { stageObjectiveTestPolicy } from './fixtures/stage-objective' +import { pythonBuildHash, waveformBacktestFixture } from './fixtures/waveform' +const start = 1791673200000 + +describe('retained-data evaluator', () => { + it('deduplicates overlapping sessions, preserves shared classifier behavior and does not mutate input', () => { + const input = fixture(), before = JSON.stringify(input) + const report = evaluateBacktest(input) + expect(JSON.stringify(input)).toBe(before) + expect(report.nights[0].uniqueObservationCount).toBe(10) + const windows = report.nights[0].evaluations[0].windows + expect(windows).toEqual(report.nights[0].evaluations[1].windows) + const expected = classifySleepStages(input.dataset.snapshot.sessions[0].vitals.map(row => ({ ...row, timestamp: new Date(row.timestampMs) })), [], 0) + expect(windows[0].blocks).toEqual(expected.map(epoch => ({ start: epoch.start, end: Math.min(start + 600000, epoch.start + epoch.duration), stage: epoch.stage }))) + expect(windows[0].stage).toMatchObject({ pairedDurationMs: 600000, agreement: 1 }) + expect(windows[0].numeric[0]).toMatchObject({ metrics: { mae: 2 }, pairedCount: 1 }) + expect(report.status).toBe('insufficient_evidence') + expect(report.stageNightSummary).toMatchObject([{ totalNights: 1, scoredNights: 1, equalNightMeanAgreementDifference: 0 }]) + expect(report.nights[0].comparisons).toMatchObject([{ baseline: 'default', candidate: 'current', windows: [{ commonSupportScope: 'all_configurations', coverageReduced: false, differences: { commonAgreement: 0 } }] }]) + }) + it('rejects changed retained inputs and engine snapshots before evaluation', () => { + const input = fixture() + input.dataset.snapshot.sessions[0].vitals[0].heartRate = 100 + expect(() => evaluateBacktest(input)).toThrow('integrity mismatch') + const wrongEngine = fixture() + wrongEngine.job.configs[0].manifestHash = '0'.repeat(64) + expect(() => evaluateBacktest(wrongEngine)).toThrow('engine/configuration mismatch') + }) + it('honors fixed offsets for stage duration scoring without relabeling the original trace', () => { + const input = fixture() + input.job.protocol.clockOffsetMs = 60000 + input.job.protocolHash = configHash(input.job.protocol) + const report = evaluateBacktest(input) + expect(report.nights[0].evaluations[0].windows[0].stage?.pairedDurationMs).toBe(540000) + expect(report.nights[0].comparisons[0].windows[0].commonSupport.baseline.pairedDurationMs).toBe(540000) + expect(report.nights[0].comparisons[0].windows[0].commonSupport.candidate.pairedDurationMs).toBe(540000) + expect(report.nights[0].evaluations[0].windows[0].blocks[0].start).toBe(start) + }) + it('uses the frozen uncertainty policy while retaining insufficient-evidence status', () => { + const input = fixture() + const protocol = { ...input.job.protocol, nightUncertainty: { version: 'paired-night-percentile-v1' as const, + seed: 'retained-policy', resamples: 200, minimumNights: 3 } } + const report = evaluateBacktest({ ...input, job: { ...input.job, protocol, protocolHash: configHash(protocol) } }) + expect(report.stageNightSummary[0].uncertainty).toMatchObject({ policy: protocol.nightUncertainty, + status: 'insufficient_nights', scoredNights: 1, interval: null }) + expect(report.status).toBe('insufficient_evidence') + }) + it('applies the shared three-configuration support to every reported comparison', () => { + const input = fixture() + const candidate = { ...input.job.configs[0], label: 'candidate' as const } + const report = evaluateBacktest({ ...input, job: { ...input.job, configs: [...input.job.configs, candidate] } }) + expect(report.nights[0].comparisons).toHaveLength(3) + for (const comparison of report.nights[0].comparisons) { + expect(comparison.windows[0]).toMatchObject({ commonSupportScope: 'all_configurations', + commonSupport: { baseline: { pairedDurationMs: 600000 }, candidate: { pairedDurationMs: 600000 } } }) + } + }) +}) + +describe('retained waveform evaluator', () => { + const rehash = >(input: T) => ({ ...input, + job: { ...input.job, datasetHash: dataHash(input.dataset), protocolHash: configHash(input.job.protocol) }, waveform: { ...input.waveform, hash: dataHash(input.waveform.outputs) } }) + it('replays retained waveform outputs with numeric comparisons, subgroups and objective assessments', () => { + const input = waveformBacktestFixture({ candidate: true, stages: true, objective: true }) + const protocol = { ...input.job.protocol, stageObjectivePolicy: stageObjectiveTestPolicy, + nightUncertainty: { version: 'paired-night-percentile-v1' as const, seed: 'fixture', resamples: 200, minimumNights: 3 } } + const progress: number[] = [] + const report = evaluateBacktest(rehash({ ...input, job: { ...input.job, protocol } }), completed => progress.push(completed)) + expect(progress).toEqual([1]) + expect(report).toMatchObject({ version: 'waveform-backtest-report-v1', support: 'identical_retained_waveform_inputs', status: 'insufficient_evidence', + waveform: { pythonBuildHash, otherSideConfigHash: configHash(defaultConfig), resources: { expectedProcesses: 2, measuredProcesses: 0 } } }) + expect(report.limitations).toHaveLength(7) + const night = report.nights[0] + expect(night).toMatchObject({ uniqueObservationCount: 10, recordedObservationCount: 10, observationCountBasis: 'union_of_derived_timestamps' }) + expect(night.evaluations.map(row => row.windows[0].observationCount)).toEqual([10, 10, 10]) + expect(night.evaluations[2].windows[0].numeric[0]).toMatchObject({ metric: 'heart_rate', pairedCount: 1, metrics: { mae: 1 } }) + expect(night.evaluations[0].windows[0].stage).not.toBeNull() + expect(night.comparisons).toHaveLength(3) + expect(night.numericComparisons?.[0].metrics.map(row => [row.metric, row.baseline, row.candidate])).toEqual(expect.arrayContaining([['heart_rate', 'default', 'candidate'], ['breathing_rate', 'current', 'candidate']])) + expect(night.numericSubgroups?.map(group => group.name)).toEqual(['reference_points', 'reference_intervals', 'window_first_half', 'window_second_half']) + expect(night.numericSubgroups?.[0].numericComparisons[0]).toMatchObject({ excludedBoundaryReferenceCount: 0, excludedByMetric: { heart_rate: 0 } }) + expect(report.numericNightSummary?.length).toBeGreaterThan(0) + expect(report.numericSubgroupSummary?.[0]).toMatchObject({ name: 'reference_points', version: 'reference-defined-numeric-subgroups-v1', excludedBoundaryReferenceCount: 0 }) + expect(report.objectiveAssessments).toHaveLength(2) + expect(report.objectiveAssessments[0]).toMatchObject({ candidate: 'candidate', status: 'insufficient_evidence', evidence: { metricCompatible: true, nights: [{ cohortId: 'night', pairedReferences: 1, baselineObservationCount: 10 }] } }) + expect(report.stageObjectiveAssessments).toHaveLength(2) + expect(report.stageObjectiveAssessments[0]).toMatchObject({ candidate: 'candidate', status: 'insufficient_evidence', evidence: { nights: [{ cohortId: 'night', perStage: expect.arrayContaining([{ stage: 'light', referenceMs: 600000, f1Difference: 0 }]) }] } }) + }) + it('replays waveform numerics without stages when downstream stage replay is not requested', () => { + const input = waveformBacktestFixture() + const report = evaluateBacktest({ ...input, job: { ...input.job, waveform: { ...input.job.waveform, baselineProvenance: { + current: { source: 'acknowledged_runtime', generation: 1, effectiveSampleTimeMs: start, warning: null }, + otherSide: { source: 'unacknowledged_defaults', generation: null, effectiveSampleTimeMs: null, warning: 'defaults' } } } } }) + expect(report.waveform?.baselineProvenance?.otherSide.source).toBe('unacknowledged_defaults') + expect(report.nights[0].evaluations[0].windows[0]).toMatchObject({ stage: null, blocks: [] }) + expect(report.nights[0]).toMatchObject({ comparisons: [], numericSubgroups: [] }) + expect(report.nights[0].numericComparisons?.[0].metrics.map(row => [row.metric, row.baseline, row.candidate])).toEqual([['heart_rate', 'default', 'current'], ['breathing_rate', 'default', 'current'], ['window_ibi_variability_v1', 'default', 'current']]) + expect(report).toMatchObject({ stageNightSummary: [], objectiveAssessments: [], stageObjectiveAssessments: [], numericSubgroupSummary: [] }) + expect(report.nights[0].uniqueObservationCount).toBe(10) + }) + it('rejects waveform inputs that do not match the frozen job identity', () => { + const base = waveformBacktestFixture() + expect(() => evaluateBacktest({ ...base, waveform: undefined })).toThrow('Missing or corrupt waveform evaluation inputs') + expect(() => evaluateBacktest({ ...base, waveform: { ...base.waveform, hash: '0'.repeat(64) } })).toThrow('Missing or corrupt waveform evaluation inputs') + const otherSide = { ...base.job.waveform.otherSide, resolvedHash: '0'.repeat(64) } + expect(() => evaluateBacktest({ ...base, job: { ...base.job, waveform: { ...base.job.waveform, otherSide } } })).toThrow('Invalid other-side waveform configuration') + expect(() => evaluateBacktest(rehash({ ...base, waveform: { ...base.waveform, outputs: [] } }))).toThrow('Missing waveform cohort/configuration output') + const output = base.waveform.outputs[0] + for (const bad of [ + { ...output, cohortId: 'other' }, + { ...output, run: { ...output.run, build_hash: '0'.repeat(64) } }, + { ...output, run: { ...output.run, config_hashes: { ...output.run.config_hashes, right: '0'.repeat(64) } } }, + { ...output, run: { ...output.run, counts: { ...output.run.counts, scored_outputs: 1 } } }, + { ...output, run: { ...output.run, raw_records: 9 } }, + { ...output, run: { ...output.run, artifact_hashes: ['c'.repeat(64)] } }, + { ...output, readings: output.readings.map(row => ({ ...row, timestampMs: row.timestampMs + 600000 })) }, + { ...output, configHash: 'c'.repeat(64), readings: output.readings.map(row => ({ ...row, configHash: 'c'.repeat(64) })) }, + ]) { + expect(() => evaluateBacktest(rehash({ ...base, waveform: { ...base.waveform, outputs: [bad] } }))).toThrow('Waveform output identity mismatch') + } + expect(() => evaluateBacktest(rehash({ ...base, waveform: { ...base.waveform, outputs: [output, output] } }))).toThrow('Waveform output identity mismatch') + const processed = fixture() + expect(() => evaluateBacktest({ ...processed, waveform: base.waveform })).toThrow('Unexpected waveform inputs for processed replay') + }) + it('rejects empty splits, missing sessions and replay windows beyond one day', () => { + const empty = fixture() + expect(() => evaluateBacktest({ ...empty, job: { ...empty.job, split: 'holdout' } })).toThrow('No nights in selected split') + const missing = fixture() + missing.dataset.manifest.splitProtocol.assignments[0].sessionIds = [1, 3] + missing.job.datasetHash = dataHash(missing.dataset) + expect(() => evaluateBacktest(missing)).toThrow('Retained session missing') + const long = fixture() + long.dataset.snapshot.sessions[1].endMs = start + 86400001 + long.job.datasetHash = dataHash(long.dataset) + expect(() => evaluateBacktest(long)).toThrow('one-day evaluation budget') + const source = fixture() + const protocol = { ...source.job.protocol, mode: 'comparison' as const } + const night = evaluateBacktest({ ...source, job: { ...source.job, protocol, protocolHash: configHash(protocol) } }).nights[0] + expect(night.comparisons).toEqual([]) + expect(night.evaluations[0].windows[0]).toMatchObject({ stage: null, blocks: [] }) + }) +}) diff --git a/src/biometrics/backtest/tests/execution-budget.test.ts b/src/biometrics/backtest/tests/execution-budget.test.ts new file mode 100644 index 000000000..cd43e1283 --- /dev/null +++ b/src/biometrics/backtest/tests/execution-budget.test.ts @@ -0,0 +1,12 @@ +// @vitest-environment node +import { expect, it } from 'vitest' +import { replayBudgetSchema } from '../execution-budget' + +const budget = { jobWallMs: 3600000, waveformWallMs: 600000, waveformCpuSeconds: 300 } +it('accepts nested ceilings and rejects budgets that cannot fit inside the job', () => { + expect(replayBudgetSchema.parse(budget)).toEqual(budget) + expect(() => replayBudgetSchema.parse({ ...budget, waveformWallMs: budget.jobWallMs + 1000 })).toThrow('fit within the job budget') + expect(() => replayBudgetSchema.parse({ ...budget, waveformCpuSeconds: 601 })).toThrow('fit within the job budget') + expect(() => replayBudgetSchema.parse({ ...budget, jobWallMs: 999 })).toThrow() + expect(() => replayBudgetSchema.parse({ ...budget, extra: true })).toThrow() +}) diff --git a/src/biometrics/backtest/tests/fixtures/backtest.ts b/src/biometrics/backtest/tests/fixtures/backtest.ts new file mode 100644 index 000000000..6bf9ae9e0 --- /dev/null +++ b/src/biometrics/backtest/tests/fixtures/backtest.ts @@ -0,0 +1,21 @@ +import { configHash } from '../../../algorithms/canonical' +import { defaultConfig, manifestHash } from '../../../algorithms/registry' +import { previewReferenceImport } from '../../../references/normalize' +const start = 1791673200000 +const fixture = () => { + const session = { id: 1, startMs: start, endMs: start + 600000, sleepDurationSeconds: 600, timesExitedBed: 0, presentIntervals: null, notPresentIntervals: null, + vitals: Array.from({ length: 10 }, (_, id) => ({ id, timestampMs: start + id * 60000, heartRate: 60, hrv: 30, breathingRate: 14, processingRunId: null })), movement: [] } + const dataset = { manifest: { splitProtocol: { assignments: [{ cohortId: 'night', sessionIds: [1, 2], split: 'tuning', startMs: start, endMs: start + 600000 }] } }, + snapshot: { version: 'processed-dataset-v1' as const, side: 'left' as const, subjectId: 'A', sessions: [session, { ...session, id: 2 }], processingRuns: [], references: [{ + importId: 'ref', revision: 1, contentHash: 'fixture', source: 'fixture', timePolicy: {}, measurements: previewReferenceImport({ side: 'left', subjectId: 'A', source: 'fixture', rows: [ + { recordKey: 'stage', metric: 'sleep_stages', value: 'light', unit: 'stage', startMs: start, endMs: start + 600000 }, + { recordKey: 'hr', metric: 'heart_rate', value: 62, unit: 'bpm', timestampMs: start }, + ] }).measurements, + }] } } + const protocol = { version: 'descriptive-evaluation-v1' as const, mode: 'stage_replay' as const, assumedStageCalibrationQuality: 0, pointToleranceMs: 30000, clockOffsetMs: 0 } + return { dataset, job: { version: 'backtest-job-v1' as const, experimentId: '11111111-1111-4111-8111-111111111111', datasetId: '22222222-2222-4222-8222-222222222222', + idempotencyKey: 'fixture', evaluatorBuildHash: 'f'.repeat(64), split: 'tuning' as const, configs: ['default', 'current'].map(label => ({ label: label as 'default' | 'current', revisionId: null, resolved: defaultConfig, resolvedHash: configHash(defaultConfig), manifestHash })), + datasetHash: configHash(dataset), protocol, protocolHash: configHash(protocol), holdoutUse: 0, evidence: 'descriptive_only' as const } } +} +export { fixture as backtestFixture } + diff --git a/src/biometrics/backtest/tests/fixtures/stage-objective.ts b/src/biometrics/backtest/tests/fixtures/stage-objective.ts new file mode 100644 index 000000000..10b0df5e0 --- /dev/null +++ b/src/biometrics/backtest/tests/fixtures/stage-objective.ts @@ -0,0 +1,6 @@ +import type { StageObjectivePolicy } from '../../stage-objective' +export const stageObjectiveTestPolicy: StageObjectivePolicy = { version: 'stage-objective-v1', qualificationId: 'test-only', minimumNights: 3, + minimumPairedDurationMsPerNight: 800, minimumOutputCoverage: 0.9, maximumLostOutputFraction: 0.1, + minimumAgreementImprovement: 0.125, maximumNightAgreementRegression: 0.125, maximumRegressedNightFraction: 0.34, + maximumMacroF1Regression: 0.125, maximumStageF1Regression: 0.125, minimumStageReferenceMsPerNight: 200, + requiredStages: ['wake', 'light', 'deep', 'rem'], operational: { maximumWallMs: 1000, maximumPeakRssBytes: 1000, maximumIngestionAgeMs: 1000 } } diff --git a/src/biometrics/backtest/tests/fixtures/waveform.ts b/src/biometrics/backtest/tests/fixtures/waveform.ts new file mode 100644 index 000000000..4116b1124 --- /dev/null +++ b/src/biometrics/backtest/tests/fixtures/waveform.ts @@ -0,0 +1,69 @@ +import { eq } from 'drizzle-orm' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import * as schema from '@/src/db/biometrics-schema' +import { configHash, dataHash } from '../../../algorithms/canonical' +import { defaultConfig, manifestHash } from '../../../algorithms/registry' +import { datasetManifestSchema, datasetSnapshotSchema } from '../../contracts' +import type { ObjectivePolicy } from '../../objective-policy' +import { backtestFixture } from './backtest' + +export const artifactHash = 'a'.repeat(64) +export const pythonBuildHash = 'b'.repeat(64) + +/** Rewrite a frozen processed dataset row in place as a retained waveform dataset. */ +export function freezeWaveformDataset(db: BetterSQLite3Database, datasetId: string) { + const row = db.select().from(schema.biometricDatasets).where(eq(schema.biometricDatasets.id, datasetId)).get() + if (!row) throw new Error('Missing dataset fixture') + const processed = datasetSnapshotSchema.parse(row.snapshot) + const manifest = { ...datasetManifestSchema.parse(row.manifest), version: 'waveform-dataset-manifest-v1' as const } + const snapshot = { ...processed, version: 'waveform-dataset-v1' as const, waveformInputs: manifest.splitProtocol.assignments.map((assignment, index) => { + const sha256 = index.toString(16).padStart(64, '0') + return { cohortId: assignment.cohortId, input: { version: 'retained-waveform-input-v1' as const, retained_start_ms: assignment.startMs, score_start_ms: assignment.startMs, score_end_ms: assignment.endMs, + artifacts: [{ artifact: { name: `${sha256}.RAW.gz`, sha256, compressed_bytes: 1, expanded_bytes: 1 }, first_event_ms: assignment.startMs, last_event_ms: assignment.endMs }], contexts: [] } } + }) } + const contentHash = dataHash({ manifest, snapshot }) + db.update(schema.biometricDatasets).set({ manifest, snapshot, contentHash }).where(eq(schema.biometricDatasets.id, datasetId)).run() + return { manifest, snapshot, contentHash } +} +export const candidateConfig = () => { + const candidate = structuredClone(defaultConfig) + candidate.heart_rate.trackingToleranceBpm = 20 + return candidate +} +export const numericObjectiveTestPolicy: ObjectivePolicy = { version: 'numeric-objective-v1', qualificationId: 'test-only', metric: 'heart_rate', minimumNights: 3, + minimumPairedReferencesPerNight: 1, maximumLostObservationFraction: 0.1, minimumMaeImprovement: 1, maximumNightMaeRegression: 1, + maximumRegressedNightFraction: 0.34, maximumSubgroupMaeRegression: 1, requiredSubgroups: ['reference_points'], + operational: { maximumWallMs: 1000, maximumPeakRssBytes: 1000, maximumIngestionAgeMs: 1000 } } + +export function waveformReadings(configHashValue: string, startMs: number, count = 10, heartRate = 60) { + return Array.from({ length: count }, (_, index) => ({ side: 'left' as const, timestampMs: startMs + index * 60000, heartRate, hrvIndex: 30, + hrvMetric: 'window_ibi_variability_v1' as const, breathingRate: 14, qualityScore: 1, flags: null, hrRaw: heartRate, sourceSide: 'left' as const, + configHash: configHashValue, limitations: [] })) +} + +/** Signal-replay variant of the processed fixture: one retained night, waveform outputs for every configuration. */ +export function waveformBacktestFixture(options: { candidate?: boolean, stages?: boolean, objective?: boolean } = {}) { + const source = backtestFixture() + const start = source.dataset.snapshot.sessions[0].startMs + const end = source.dataset.manifest.splitProtocol.assignments[0].endMs + const otherSide = { manifestHash, resolvedHash: configHash(defaultConfig), resolved: defaultConfig } + const input = { version: 'retained-waveform-input-v1' as const, retained_start_ms: start, score_start_ms: start, score_end_ms: end, + artifacts: [{ artifact: { name: `${artifactHash}.RAW.gz`, sha256: artifactHash, compressed_bytes: 1, expanded_bytes: 1 }, first_event_ms: start, last_event_ms: end }], + contexts: [] } + const snapshot = { ...source.dataset.snapshot, version: 'waveform-dataset-v1' as const, waveformInputs: [{ cohortId: 'night', input }] } + const dataset = { ...source.dataset, snapshot } + const candidate = candidateConfig() + const configs = options.candidate + ? [...source.job.configs, { label: 'candidate' as const, revisionId: null, resolved: candidate, resolvedHash: configHash(candidate), manifestHash }] + : source.job.configs + const protocol = { ...source.job.protocol, mode: 'signal_replay' as const, ...(options.stages ? { waveformStages: true } : {}), + ...(options.objective ? { objectivePolicies: [numericObjectiveTestPolicy] } : {}) } + const outputs = [...new Set(configs.map(config => config.resolvedHash))].map((hash, index) => { + const readings = waveformReadings(hash, start, 10, 60 + index) + return { cohortId: 'night', configHash: hash, readings, run: { version: 'retained-waveform-run-v1' as const, build_hash: pythonBuildHash, + decoded_records: 10, raw_records: 10, counts: { records: 10, accepted_frames: 10, empty_channel_frames: 0, pump_gated_frames: 0, preroll_outputs: 0, scored_outputs: readings.length }, + limitations: [], config_hashes: { left: hash, right: otherSide.resolvedHash }, artifact_hashes: [artifactHash] } } + }) + const job = { ...source.job, configs, protocol, protocolHash: configHash(protocol), datasetHash: dataHash(dataset), waveform: { pythonBuildHash, otherSide } } + return { dataset, job, waveform: { hash: dataHash(outputs), outputs } } +} diff --git a/src/biometrics/backtest/tests/host-startup.test.ts b/src/biometrics/backtest/tests/host-startup.test.ts new file mode 100644 index 000000000..fc15ff775 --- /dev/null +++ b/src/biometrics/backtest/tests/host-startup.test.ts @@ -0,0 +1,25 @@ +// @vitest-environment node +import { afterEach, expect, it, vi } from 'vitest' +import { startConfiguredBacktestHost } from '../host-startup' +import { startBacktestHost } from '../host' +vi.mock('../host', () => ({ startBacktestHost: vi.fn(() => ({ started: true })) })) +const db = {} as Parameters[0] +const policy = { qualificationId: 'test-only', minimumAvailableMemoryBytes: 100, minimumFreeDiskBytes: 100, + maximumLoadPerCpu: 1, maximumEventLoopDelayMs: 100, maximumPiezoAgeMs: 10000, maximumTemperatureC: 65, maximumSampleAgeMs: 1000 } +afterEach(() => { + vi.unstubAllEnvs() + vi.clearAllMocks() +}) +it.each([undefined, '', '{}', '{', JSON.stringify({ ...policy, maximumLoadPerCpu: 0 })])('does not start background work without valid deployment policy: %s', (value) => { + vi.stubEnv('SLEEPYPOD_BIOMETRICS_ADMISSION_POLICY', value) + expect(startConfiguredBacktestHost(db)).toBeNull() + expect(startBacktestHost).not.toHaveBeenCalled() +}) +it('starts with a fresh admission closure and denies after policy removal', async () => { + vi.stubEnv('SLEEPYPOD_BIOMETRICS_ADMISSION_POLICY', JSON.stringify(policy)) + expect(startConfiguredBacktestHost(db)).toEqual({ started: true }) + expect(startBacktestHost).toHaveBeenCalledWith(db, expect.any(Function), expect.any(Function)) + const admit = vi.mocked(startBacktestHost).mock.calls[0][1] + vi.stubEnv('SLEEPYPOD_BIOMETRICS_ADMISSION_POLICY', undefined) + expect(await admit()).toBe(false) +}) diff --git a/src/biometrics/backtest/tests/host.test.ts b/src/biometrics/backtest/tests/host.test.ts new file mode 100644 index 000000000..a70409823 --- /dev/null +++ b/src/biometrics/backtest/tests/host.test.ts @@ -0,0 +1,122 @@ +// @vitest-environment node +import { afterEach, expect, it, vi } from 'vitest' +import { BacktestHost, startBacktestHost, stopBacktestHost } from '../host' +import { coordinatorRequest } from '../coordinator-service' +vi.mock('../coordinator-service', () => ({ coordinatorRequest: vi.fn() })) +afterEach(async () => { + await stopBacktestHost() + vi.restoreAllMocks() + vi.useRealTimers() +}) +it('never overlaps attempts and awaits aborted execution before shutdown completes', async () => { + vi.useFakeTimers() + let release: (() => void) | undefined + let observedSignal: AbortSignal | undefined + const execute = vi.fn((signal: AbortSignal) => { + observedSignal = signal + return new Promise((resolve) => { + release = resolve + }) + }) + const host = new BacktestHost({ execute, collect: vi.fn() }) + host.start() + await vi.advanceTimersByTimeAsync(0) + host.start() + await vi.advanceTimersByTimeAsync(60000) + expect(execute).toHaveBeenCalledTimes(1) + let finished = false + const stopping = host.stop().then(() => { + finished = true + }) + expect(observedSignal?.aborted).toBe(true) + await Promise.resolve() + expect(finished).toBe(false) + release?.() + await stopping + expect(finished).toBe(true) + await vi.advanceTimersByTimeAsync(60000) + expect(execute).toHaveBeenCalledTimes(1) + expect(vi.getTimerCount()).toBe(0) +}) +it('recovers from execution and cleanup errors without spinning or abandoning later work', async () => { + vi.useFakeTimers() + const execute = vi.fn().mockRejectedValueOnce(new Error('temporary')).mockResolvedValue({ state: 'idle' }) + const collect = vi.fn().mockRejectedValueOnce(new Error('temporary')).mockResolvedValue(undefined) + const onError = vi.fn() + const host = new BacktestHost({ execute, collect, onError }) + host.start() + await vi.advanceTimersByTimeAsync(0) + expect(onError.mock.calls).toEqual([['cleanup'], ['execution']]) + await vi.advanceTimersByTimeAsync(4999) + expect(execute).toHaveBeenCalledTimes(1) + await vi.advanceTimersByTimeAsync(1) + expect(execute).toHaveBeenCalledTimes(2) + expect(collect).toHaveBeenCalledTimes(1) + await vi.advanceTimersByTimeAsync(1800000) + expect(collect).toHaveBeenCalledTimes(2) + await host.stop() +}) +it('does not begin a job when shutdown arrives during cleanup', async () => { + vi.useFakeTimers() + let release: (() => void) | undefined + const execute = vi.fn() + const host = new BacktestHost({ execute, collect: () => new Promise((resolve) => { + release = resolve + }) }) + host.start() + const stopping = host.stop() + release?.() + await stopping + expect(execute).not.toHaveBeenCalled() + expect(vi.getTimerCount()).toBe(0) +}) +it('aborts the active cleanup on shutdown and waits for its acknowledgement', async () => { + vi.useFakeTimers() + const execute = vi.fn() + let aborted = false + const host = new BacktestHost({ execute, collect: signal => new Promise((resolve) => { + signal.addEventListener('abort', () => { + aborted = true + resolve() + }, { once: true }) + }) }) + host.start() + await host.stop() + expect(aborted).toBe(true) + expect(execute).not.toHaveBeenCalled() + expect(vi.getTimerCount()).toBe(0) +}) + +it('shares one background host and forwards health admission to cleanup and execution', async () => { + vi.useFakeTimers() + vi.mocked(coordinatorRequest).mockResolvedValue(null) + const db = {} as Parameters[0] + const admit = vi.fn().mockResolvedValue(false) + const healthAdmission = vi.fn().mockResolvedValue({ allowed: false, sample: null }) + const first = startBacktestHost(db, admit, healthAdmission) + expect(startBacktestHost(db, admit, healthAdmission)).toBe(first) + await vi.advanceTimersByTimeAsync(0) + expect(coordinatorRequest).toHaveBeenCalledTimes(2) + expect(coordinatorRequest).toHaveBeenNthCalledWith(1, db, { type: 'collectArtifacts' }, { signal: expect.any(AbortSignal), admit, healthAdmission }) + expect(coordinatorRequest).toHaveBeenNthCalledWith(2, db, { type: 'execute', owner: expect.stringMatching(/^core:/) }, { signal: expect.any(AbortSignal), admit, healthAdmission }) + const signal = vi.mocked(coordinatorRequest).mock.calls[0][2]?.signal + await stopBacktestHost() + expect(signal?.aborted).toBe(true) + expect(vi.getTimerCount()).toBe(0) + await stopBacktestHost() + expect(startBacktestHost(db, admit, healthAdmission)).not.toBe(first) +}) + +it('reports coordinator failures without exporting their private diagnostic content', async () => { + vi.useFakeTimers() + vi.mocked(coordinatorRequest).mockReset().mockRejectedValue(new Error('private-data')) + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + startBacktestHost({} as Parameters[0], async () => false) + await vi.advanceTimersByTimeAsync(0) + expect(warn.mock.calls).toEqual([ + ['[biometrics] Backtest cleanup failed; host will retry'], + ['[biometrics] Backtest execution failed; host will retry'], + ]) + await stopBacktestHost() + expect(vi.getTimerCount()).toBe(0) +}) diff --git a/src/biometrics/backtest/tests/job-operations.test.ts b/src/biometrics/backtest/tests/job-operations.test.ts new file mode 100644 index 000000000..851ca4d07 --- /dev/null +++ b/src/biometrics/backtest/tests/job-operations.test.ts @@ -0,0 +1,31 @@ +// @vitest-environment node +import { expect, it } from 'vitest' +import { createJobOperations } from '../job-operations' +const receipt = { allowed: true, sample: { qualificationId: 'qualified-build', checkedAtMs: 100000, ingestionAgeMs: 100 } } +it('retains bounded sampled ingestion evidence with monotonic duration and gaps', () => { + let time = 0 + const operations = createJobOperations(() => time) + operations.record(receipt) + time = 7000 + operations.record({ ...receipt, sample: { ...receipt.sample, ingestionAgeMs: 300 } }) + time = 7200 + expect(operations.finish()).toMatchObject({ wallMs: 7200, ingestion: { checks: 2, samples: 2, allChecksMeasured: true, + maximumObservedAgeMs: 300, maximumObservationGapMs: 7000, continuousMeasurement: false, qualificationId: 'qualified-build' } }) +}) +it('exposes missing, denied, and changing qualification evidence without inventing zero age', () => { + const empty = createJobOperations(() => 0) + empty.record(null) + expect(empty.finish().ingestion).toMatchObject({ allChecksMeasured: false, samples: 0, maximumObservedAgeMs: null, qualificationId: null }) + for (const bad of [null, { ...receipt, allowed: false }, { ...receipt, sample: { ...receipt.sample, ingestionAgeMs: -1 } }, + { ...receipt, sample: { ...receipt.sample, qualificationId: 'different' } }]) { + const operations = createJobOperations(() => 0) + operations.record(receipt) + operations.record(bad) + expect(operations.finish().ingestion).toMatchObject({ allChecksMeasured: false, qualificationId: null }) + } +}) +it('a single observation does not establish coverage over execution', () => { + const operations = createJobOperations(() => 0) + operations.record(receipt) + expect(operations.finish().ingestion.allChecksMeasured).toBe(false) +}) diff --git a/src/biometrics/backtest/tests/jobs.test.ts b/src/biometrics/backtest/tests/jobs.test.ts new file mode 100644 index 000000000..f6057b741 --- /dev/null +++ b/src/biometrics/backtest/tests/jobs.test.ts @@ -0,0 +1,276 @@ +// @vitest-environment node +import { mkdtempSync, readdirSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import * as schema from '@/src/db/biometrics-schema' +import { configHash } from '../../algorithms/canonical' +import { defaultConfig, manifestHash } from '../../algorithms/registry' +import { ReportArtifactStore } from '../artifacts' +import { DatasetStore } from '../datasets' +import { BacktestJobStore } from '../jobs' +import { freezeWaveformDataset } from './fixtures/waveform' +let sqlite: Database.Database +let db: ReturnType> +let store: BacktestJobStore +let datasets: DatasetStore +let now: number +let datasetId: string +let experimentId: string +const config = { label: 'default' as const, revisionId: null, resolved: defaultConfig, resolvedHash: configHash(defaultConfig), manifestHash } +const protocol = { version: 'descriptive-evaluation-v1' as const, mode: 'stage_replay' as const, assumedStageCalibrationQuality: 0, pointToleranceMs: 30000, clockOffsetMs: 0 } +const enqueue = (idempotencyKey = 'job-1', split: 'tuning' | 'validation' | 'holdout' = 'tuning') => store.enqueue({ experimentId, evaluatorBuildHash: 'f'.repeat(64), idempotencyKey, split, configs: [config] }) +const lease = (job: NonNullable>) => ({ id: job.id, owner: job.leaseOwner as string, token: job.fencingToken }) +beforeEach(() => { + now = 1791673200000 + sqlite = new Database(':memory:') + db = drizzle(sqlite, { schema }) + migrate(db, { migrationsFolder: 'src/db/biometrics-migrations' }) + for (let id = 1; id <= 3; id++) db.insert(schema.sleepRecords).values({ id, side: 'left', enteredBedAt: new Date(now + id * 86400000), + leftBedAt: new Date(now + id * 86400000 + 3600000), sleepDurationSeconds: 3600, timesExitedBed: 0 }).run() + datasets = new DatasetStore(db, () => now) + datasetId = datasets.create({ side: 'left', subjectId: 'A', sessionIds: [1, 2, 3], referenceImportIds: [], timezone: 'UTC', idempotencyKey: 'dataset' }).id + store = new BacktestJobStore(db, () => now) + experimentId = store.createExperiment(datasetId, protocol).id +}) +afterEach(() => sqlite.close()) + +describe('durable backtest leases and results', () => { + it('deduplicates requests, permits only one active worker, and atomically publishes the winning attempt', () => { + const job = enqueue() + expect(enqueue().id).toBe(job.id) + expect(() => enqueue('job-1', 'validation')).toThrow('changed') + const claimed = store.claim('worker-A') + expect(claimed).toMatchObject({ id: job.id, attempt: 1, fencingToken: 1 }) + expect(store.claim('worker-B')).toBeNull() + if (!claimed) throw new Error('Missing claim') + const token = lease(claimed) + store.heartbeat(token, { phase: 'evaluating', completedNights: 0, totalNights: 1 }) + store.complete(token, { status: 'insufficient_evidence' }) + expect(store.get(job.id).state).toBe('succeeded') + expect(sqlite.prepare('SELECT count(*) AS n FROM biometric_backtest_results').get()).toEqual({ n: 1 }) + expect(() => store.complete(token, {})).toThrow('no longer authoritative') + }) + it('releases inline report bytes idempotently without erasing holdout exposure or job identity', () => { + const job = enqueue('release-report', 'holdout') + expect(() => store.releaseResult(job.id, '0'.repeat(64))).toThrow('Only completed') + const claimed = store.claim('worker-A') + if (!claimed) throw new Error('Missing claim') + const published = store.complete(lease(claimed), { traces: ['retained until release'] }) + expect(() => store.releaseResult(job.id, '0'.repeat(64))).toThrow('hash changed') + expect(store.result(job.id)?.available).toBe(true) + const released = store.releaseResult(job.id, published.contentHash) + expect(released).toMatchObject({ available: false, report: null, releasedAtMs: now, + contentHash: published.contentHash, fencingToken: claimed.fencingToken, currentHoldoutUseCount: 1 }) + now += 1000 + expect(store.releaseResult(job.id, published.contentHash)).toEqual(released) + expect(store.result(job.id)).toEqual(released) + expect(enqueue('release-report', 'holdout').id).toBe(job.id) + expect(store.get(job.id).state).toBe('succeeded') + expect(sqlite.prepare('SELECT report FROM biometric_backtest_results').get()).toEqual({ report: '{}' }) + expect(sqlite.prepare('SELECT count(*) AS n FROM biometric_holdout_accesses').get()).toEqual({ n: 1 }) + }) + it('fences expired workers and retries crashes only within the attempt limit', () => { + const job = enqueue() + const first = store.claim('worker-A', 1000) + if (!first) throw new Error('Missing claim') + now += 1000 + expect(() => store.complete(lease(first), {})).toThrow('no longer authoritative') + const second = store.claim('worker-B', 1000) + expect(second).toMatchObject({ attempt: 2, fencingToken: 2 }) + expect(() => store.heartbeat(lease(first), { phase: 'loading', completedNights: 0, totalNights: 1 })).toThrow('no longer authoritative') + now += 1000 + expect(store.claim('worker-C', 1000)).toMatchObject({ attempt: 3 }) + now += 1000 + expect(store.claim('worker-D')).toBeNull() + expect(store.get(job.id)).toMatchObject({ state: 'failed', error: 'Worker lease expired' }) + }) + it('cancels queued jobs immediately and rejects publication after running cancellation', () => { + const queued = enqueue() + expect(store.cancel(queued.id).state).toBe('cancelled') + expect(store.claim('worker')).toBeNull() + const running = enqueue('second') + const claimed = store.claim('worker') + if (!claimed) throw new Error('Missing claim') + expect(store.cancel(running.id).state).toBe('cancel_requested') + expect(store.heartbeat(lease(claimed), { phase: 'loading', completedNights: 0, totalNights: 1 })).toEqual({ cancelRequested: true }) + expect(() => store.complete(lease(claimed), {})).toThrow('cannot publish') + store.acknowledgeCancellation(lease(claimed)) + expect(store.get(running.id).state).toBe('cancelled') + expect(sqlite.prepare('SELECT count(*) AS n FROM biometric_backtest_results').get()).toEqual({ n: 0 }) + }) + it('recovers abandoned cancellations without rerunning and rolls back failed result inserts', () => { + const job = enqueue() + const claimed = store.claim('worker', 1000) + if (!claimed) throw new Error('Missing claim') + sqlite.exec('CREATE TRIGGER reject_result BEFORE INSERT ON biometric_backtest_results BEGIN SELECT RAISE(ABORT, \'fixture failure\'); END;') + expect(() => store.complete(lease(claimed), {})).toThrow('fixture failure') + expect(store.get(job.id).state).toBe('running') + store.cancel(job.id) + now += 1000 + expect(store.claim('replacement')).toBeNull() + expect(store.get(job.id).state).toBe('cancelled') + }) + it('records holdout reuse across experiments and keeps duplicate enqueue from consuming it again', () => { + const first = enqueue('holdout', 'holdout') + expect(first.input).toMatchObject({ holdoutUse: 1, evidence: 'descriptive_only' }) + enqueue('holdout', 'holdout') + expect(store.holdoutUseCount(datasetId)).toBe(1) + experimentId = store.createExperiment(datasetId, protocol).id + expect(enqueue('holdout-again', 'holdout').input).toMatchObject({ holdoutUse: 2, evidence: 'reused_holdout_is_validation' }) + expect(store.holdoutUseCount(datasetId)).toBe(2) + const clone = datasets.create({ side: 'left', subjectId: 'different-asserted-name', sessionIds: [1, 2, 3], referenceImportIds: [], timezone: 'UTC', idempotencyKey: 'clone' }) + experimentId = store.createExperiment(clone.id, protocol).id + expect(enqueue('cloned-holdout', 'holdout').input).toMatchObject({ holdoutUse: 3, evidence: 'reused_holdout_is_validation' }) + expect(store.holdoutUseCount(datasetId)).toBe(3) + }) + it('rejects forged engine hashes and invalid progress without changing the lease', () => { + expect(() => store.enqueue({ experimentId, evaluatorBuildHash: 'f'.repeat(64), idempotencyKey: 'bad', split: 'tuning', configs: [{ ...config, resolvedHash: '0'.repeat(64) }] })).toThrow('does not match') + enqueue() + const claimed = store.claim('worker') + if (!claimed) throw new Error('Missing claim') + expect(() => store.heartbeat(lease(claimed), { phase: 'evaluating', completedNights: 2, totalNights: 1 })).toThrow('monotonic') + expect(store.get(claimed.id).progress).toMatchObject({ completedNights: 0 }) + store.fail(lease(claimed), 'bounded failure') + expect(store.get(claimed.id).state).toBe('failed') + }) +}) + +it('refuses signal experiments without immutable waveform inputs', () => { + expect(() => store.createExperiment(datasetId, { ...protocol, mode: 'signal_replay' })).toThrow('retained waveform inputs') +}) + +describe('experiment and job bookkeeping', () => { + it('deduplicates experiments by idempotency key and pages them newest first', () => { + const experiment = store.getExperiment(experimentId) + expect(experiment.protocol).toEqual(protocol) + expect(store.createExperiment(datasetId, protocol, experiment.idempotencyKey ?? undefined).id).toBe(experimentId) + expect(() => store.createExperiment(datasetId, { ...protocol, clockOffsetMs: 1 }, experiment.idempotencyKey ?? undefined)).toThrow('idempotency key changed') + expect(() => store.getExperiment('00000000-0000-4000-8000-000000000000')).toThrow('Experiment not found') + for (let index = 0; index < 51; index++) { + now += 1 + store.createExperiment(datasetId, protocol, `page-${index}`) + } + const first = store.pageExperiments('left') + expect(first.items).toHaveLength(50) + expect(first.nextCursor).toEqual({ createdAtMs: first.items[49].createdAtMs, id: first.items[49].id }) + const second = store.pageExperiments('left', first.nextCursor ?? undefined) + expect(second.items).toHaveLength(2) + expect(second.items.at(-1)?.id).toBe(experimentId) + expect(second.nextCursor).toBeNull() + expect(store.pageExperiments('right').items).toEqual([]) + }) + it('pages jobs with a cursor and reports missing jobs, datasets and results explicitly', () => { + for (let index = 0; index < 51; index++) { + now += 1 + enqueue(`job-${index}`) + } + const first = store.page('left') + expect(first.items).toHaveLength(50) + expect(first.nextCursor).not.toBeNull() + const second = store.page('left', first.nextCursor ?? undefined) + expect(second.items).toHaveLength(1) + expect(second.nextCursor).toBeNull() + expect(store.page('right').items).toEqual([]) + expect(() => store.get('00000000-0000-4000-8000-000000000000')).toThrow('Backtest job not found') + expect(() => store.holdoutUseCount('00000000-0000-4000-8000-000000000000')).toThrow('Dataset unavailable') + expect(store.result(first.items[0].id)).toBeNull() + expect(() => store.cancel('00000000-0000-4000-8000-000000000000')).toThrow('not found') + }) + it('rejects malformed enqueue requests before touching the queue', () => { + expect(() => store.enqueue({ experimentId, evaluatorBuildHash: 'f'.repeat(64), idempotencyKey: 'dup', split: 'tuning', configs: [config, config] })).toThrow('labels must be unique') + expect(() => store.enqueue({ experimentId: '00000000-0000-4000-8000-000000000000', evaluatorBuildHash: 'f'.repeat(64), idempotencyKey: 'missing', split: 'tuning', configs: [config] })).toThrow('Experiment not found') + const waveform = { pythonBuildHash: 'b'.repeat(64), otherSide: { manifestHash, resolvedHash: configHash(defaultConfig), resolved: defaultConfig } } + expect(() => store.enqueue({ experimentId, evaluatorBuildHash: 'f'.repeat(64), idempotencyKey: 'waveform', split: 'tuning', configs: [config], waveform })).toThrow('Waveform replay requires') + const single = datasets.create({ side: 'left', subjectId: 'A', sessionIds: [1], referenceImportIds: [], timezone: 'UTC', idempotencyKey: 'single' }) + experimentId = store.createExperiment(single.id, protocol).id + expect(() => enqueue('no-nights', 'validation')).toThrow('no nights assigned') + datasets.release(single.id) + expect(() => enqueue('released')).toThrow('explicitly released') + expect(sqlite.prepare('SELECT count(*) AS n FROM biometric_backtest_jobs').get()).toEqual({ n: 0 }) + }) + it('pins waveform replay jobs to the frozen other-side configuration', () => { + freezeWaveformDataset(db, datasetId) + const signal = { ...protocol, mode: 'signal_replay' as const, waveformStages: true } + experimentId = store.createExperiment(datasetId, signal).id + expect(store.getExperiment(experimentId).protocol).toEqual(signal) + expect(() => enqueue('no-waveform')).toThrow('Waveform replay requires') + const waveform = { pythonBuildHash: 'b'.repeat(64), otherSide: { manifestHash, resolvedHash: configHash(defaultConfig), resolved: defaultConfig } } + expect(() => store.enqueue({ experimentId, evaluatorBuildHash: 'f'.repeat(64), idempotencyKey: 'forged', split: 'tuning', configs: [config], + waveform: { ...waveform, otherSide: { ...waveform.otherSide, resolvedHash: '0'.repeat(64) } } })).toThrow('Other-side configuration identity mismatch') + const job = store.enqueue({ experimentId, evaluatorBuildHash: 'f'.repeat(64), idempotencyKey: 'waveform', split: 'tuning', configs: [config], waveform }) + expect(job.input).toMatchObject({ waveform, protocol: signal }) + }) + it('refuses stale leases for finished work and leaves finished jobs untouched by cancellation', () => { + const job = enqueue() + const claimed = store.claim('worker') + if (!claimed) throw new Error('Missing claim') + expect(() => store.acknowledgeCancellation(lease(claimed))).toThrow('has not been requested') + expect(store.interrupt(lease(claimed), 'Resource admission withdrawn')).toMatchObject({ state: 'interrupted', error: 'Resource admission withdrawn' }) + expect(() => store.heartbeat(lease(claimed), { phase: 'loading', completedNights: 0, totalNights: 1 })).toThrow('no longer authoritative') + const again = store.claim('worker') + if (!again) throw new Error('Missing claim') + store.complete(lease(again), { ok: true }) + expect(store.cancel(job.id).state).toBe('succeeded') + expect(() => store.interrupt(lease(again))).toThrow('no longer authoritative') + const third = enqueue('third') + const claimedThird = store.claim('worker') + if (!claimedThird) throw new Error('Missing claim') + store.cancel(third.id) + expect(store.interrupt(lease(claimedThird)).state).toBe('cancelled') + const fourth = enqueue('fourth') + const claimedFourth = store.claim('worker') + if (!claimedFourth) throw new Error('Missing claim') + store.cancel(fourth.id) + expect(store.fail(lease(claimedFourth), 'late failure').state).toBe('cancelled') + }) + it('detects corrupted results and refuses to release them', () => { + const job = enqueue() + const claimed = store.claim('worker') + if (!claimed) throw new Error('Missing claim') + const published = store.complete(lease(claimed), { report: 'bytes' }) + sqlite.prepare('UPDATE biometric_backtest_results SET report = ?').run(JSON.stringify({ report: 'tampered' })) + expect(() => store.result(job.id)).toThrow('integrity check failed') + sqlite.prepare('UPDATE biometric_backtest_results SET fencing_token = 99').run() + expect(() => store.result(job.id)).toThrow('integrity check failed') + expect(() => store.releaseResult(job.id, published.contentHash)).toThrow('integrity check failed') + expect(() => store.releaseResult(job.id, 'not-a-hash')).toThrow() + }) + it('refuses oversized reports and exhausted inline storage budgets', () => { + enqueue('first') + const first = store.claim('worker') + if (!first) throw new Error('Missing claim') + store.complete(lease(first), { ok: true }) + enqueue('second') + const claimed = store.claim('worker') + if (!claimed) throw new Error('Missing claim') + expect(() => store.complete(lease(claimed), { padding: Array.from({ length: 8200 }, () => 'x'.repeat(2048)) })).toThrow('exceeds 16 MiB') + expect(() => store.complete(lease(claimed), { padding: Array.from({ length: 1600 }, () => 'x'.repeat(2048)) })).toThrow('persistent artifact directory') + expect(() => store.collectOrphanedArtifacts()).toThrow('persistent artifact directory') + sqlite.prepare('UPDATE biometric_backtest_results SET report = ?').run(`"${'x'.repeat(63 * 1024 * 1024)}"`) + expect(() => store.complete(lease(claimed), { padding: Array.from({ length: 520 }, () => 'x'.repeat(2048)) })).toThrow('storage budget') + expect(store.get(claimed.id).state).toBe('running') + }) + it('stores large reports through an explicit artifact store and collects orphans', () => { + const directory = mkdtempSync(join(tmpdir(), 'biometric-jobs-artifacts-')) + try { + const artifacts = new ReportArtifactStore(directory) + const stored = new BacktestJobStore(db, () => now, artifacts) + const job = enqueue() + const claimed = stored.claim('worker') + if (!claimed) throw new Error('Missing claim') + const report = { padding: Array.from({ length: 1600 }, () => 'x'.repeat(2048)) } + const published = stored.complete(lease(claimed), report) + expect(stored.result(job.id)?.report).toEqual(report) + expect(stored.collectOrphanedArtifacts()).toBe(0) + const released = stored.releaseResult(job.id, published.contentHash) + expect(released).toMatchObject({ available: false, report: null }) + expect(readdirSync(directory).filter(name => name.endsWith('.json'))).toEqual([]) + expect(stored.releaseResult(job.id, published.contentHash)).toEqual(released) + } + finally { rmSync(directory, { recursive: true, force: true }) } + }) +}) diff --git a/src/biometrics/backtest/tests/location.test.ts b/src/biometrics/backtest/tests/location.test.ts new file mode 100644 index 000000000..c13d18ebf --- /dev/null +++ b/src/biometrics/backtest/tests/location.test.ts @@ -0,0 +1,25 @@ +// @vitest-environment node +import { resolve } from 'node:path' +import { beforeEach, expect, it, vi } from 'vitest' +import { existsSync } from 'node:fs' +import { packagedWorkerPath, packagedCoordinatorPath, packagedShadowHostPath } from '../location' + +vi.mock('node:fs', () => ({ existsSync: vi.fn() })) +beforeEach(() => vi.mocked(existsSync).mockReset()) + +it.each([ + ['worker', packagedWorkerPath], + ['coordinator', packagedCoordinatorPath], + ['shadow-host', packagedShadowHostPath], +] as const)('resolves only fixed %s packages, preferring the installed bundle', (name, locate) => { + const installed = resolve(`biometrics-worker/${name}.cjs`) + const development = resolve(`.next/standalone/biometrics-worker/${name}.cjs`) + vi.mocked(existsSync).mockReturnValue(true) + expect(locate()).toBe(installed) + expect(existsSync).toHaveBeenCalledExactlyOnceWith(installed) + vi.mocked(existsSync).mockReset().mockImplementation(path => path === development) + expect(locate()).toBe(development) + expect(vi.mocked(existsSync).mock.calls).toEqual([[installed], [development]]) + vi.mocked(existsSync).mockReset().mockReturnValue(false) + expect(locate).toThrow(expect.objectContaining({ code: 'WORKER_UNAVAILABLE' })) +}) diff --git a/src/biometrics/backtest/tests/night-summary-core.test.ts b/src/biometrics/backtest/tests/night-summary-core.test.ts new file mode 100644 index 000000000..f2a12cf10 --- /dev/null +++ b/src/biometrics/backtest/tests/night-summary-core.test.ts @@ -0,0 +1,69 @@ +// @vitest-environment node +import { createHash } from 'node:crypto' +import { describe, expect, it } from 'vitest' +import { previewReferenceImport } from '../../references/normalize' +import { createNightSummary } from '../night-summary-core' +import { compareStageOutputs } from '../stage-comparison' +import { createNightUncertainty } from '../uncertainty-core' + +const T = 1791673200000 +const digest = (text: string) => new Uint8Array(createHash('sha256').update(text).digest()) +const summarize = createNightSummary(createNightUncertainty(digest)) +const references = previewReferenceImport({ side: 'left', subjectId: 'fixture', source: 'test', rows: [ + { recordKey: 'first', metric: 'sleep_stages', value: 'light', unit: 'stage', startMs: T, endMs: T + 10000 }, + { recordKey: 'second', metric: 'sleep_stages', value: 'deep', unit: 'stage', startMs: T + 10000, endMs: T + 20000 }, +] }).measurements +const identity = { baseline: 'default', candidate: 'candidate', baselineHash: 'a'.repeat(64), candidateHash: 'b'.repeat(64) } +const block = (start: number, end: number, stage: 'light' | 'deep' | 'wake') => ({ start, end, stage }) +const window = (candidate: Array>, startMs = T, endMs = T + 20000, baseline = [block(T, T + 20000, 'light')]) => + ({ startMs, endMs, ...compareStageOutputs(baseline, candidate, references, startMs, endMs) }) +const improved = () => window([block(T, T + 10000, 'light'), block(T + 10000, T + 20000, 'deep')]) +const regressed = () => window([block(T, T + 20000, 'wake')]) +const unchanged = () => window([block(T, T + 20000, 'light')]) +const night = (cohortId: string, windows: ReturnType[]) => ({ cohortId, comparisons: [{ ...identity, windows }] }) + +describe('whole-night stage summary', () => { + it('aggregates disconnected windows per night before counting improved, regressed and unchanged nights', () => { + const [summary] = summarize([night('a', [improved()]), night('b', [regressed()]), night('c', [unchanged()])]) + expect(summary).toMatchObject({ ...identity, totalNights: 3, scoredNights: 3, unscoredNights: 0, improvedAgreementNights: 1, regressedAgreementNights: 1, + unchangedAgreementNights: 1, coverageReducedNights: 0, equalNightMeanAgreementDifference: 0, observedNightRange: { minimum: -0.5, maximum: 0.5 }, + uncertainty: { status: 'not_requested', interval: null }, interpretation: 'descriptive_night_aggregation_not_confidence_interval' }) + expect(summary.nights[0]).toMatchObject({ cohortId: 'a', evaluationDurationMs: 20000, commonOutputDurationMs: 20000, commonAgreementDifference: 0.5, coverageReduced: false, + commonSupport: { baseline: { agreement: 0.5, pairedDurationMs: 20000 }, candidate: { agreement: 1 } }, + ownSupport: { baseline: { outputMs: 20000, pairedMs: 20000, referenceMs: 20000, correctMs: 10000, agreement: 0.5, outputCoverage: 1, referenceCoverage: 1 } } }) + expect(summary.nights[0].commonSupport.candidate.perStage).toEqual(expect.arrayContaining([ + expect.objectContaining({ stage: 'light', precision: 1, recall: 1, f1: 1 }), expect.objectContaining({ stage: 'wake', precision: null, recall: null, f1: null })])) + expect(summary.pooledCommonSupport.candidate.pairedDurationMs).toBe(60000) + expect(summary.nights[1].commonMacroF1Difference).toBeLessThan(0) + }) + it('keeps unscored nights counted without inventing agreement', () => { + const empty = window([block(T + 10000, T + 20000, 'deep')], T, T + 20000, [block(T, T + 10000, 'light')]) + const [summary] = summarize([night('only', [empty])]) + expect(summary).toMatchObject({ totalNights: 1, scoredNights: 0, unscoredNights: 1, equalNightMeanAgreementDifference: null, observedNightRange: null, coverageReducedNights: 0 }) + expect(summary.nights[0]).toMatchObject({ commonAgreementDifference: null, commonMacroF1Difference: null, commonSupport: { baseline: { agreement: null, macroF1: null } } }) + expect(summarize([])).toEqual([]) + const [outside] = summarize([night('late', [window([], T + 30000, T + 40000, [])])]) + expect(outside.nights[0].ownSupport.candidate).toMatchObject({ agreement: null, referenceCoverage: null, outputCoverage: 0 }) + }) + it('groups each comparison identity across nights and reports a bootstrap interval when requested', () => { + const other = { ...identity, candidate: 'current', candidateHash: 'c'.repeat(64) } + const nights = ['a', 'b', 'c', 'd'].map((cohortId, index) => ({ cohortId, comparisons: [ + { ...identity, windows: [index % 2 ? improved() : unchanged()] }, { ...other, windows: [regressed()] }] })) + const summaries = summarize(nights, { version: 'paired-night-percentile-v1', seed: 'fixture', resamples: 200, minimumNights: 3 }) + expect(summaries.map(row => [row.candidate, row.totalNights])).toEqual([['candidate', 4], ['current', 4]]) + expect(summaries[0].uncertainty).toMatchObject({ status: 'estimated', scoredNights: 4 }) + expect(summaries[0].uncertainty.interval?.lower).toBeLessThanOrEqual(summaries[0].uncertainty.interval?.upper ?? -1) + expect(summaries[1].uncertainty).toMatchObject({ status: 'degenerate', interval: null }) + }) + it('rejects oversized, duplicated and malformed nightly inputs', () => { + expect(() => summarize(Array.from({ length: 61 }, (_, index) => night(String(index), [improved()])))).toThrow('Invalid whole-night summary input') + expect(() => summarize([night('a', [improved()]), night('a', [improved()])])).toThrow('Invalid whole-night summary input') + expect(() => summarize([{ cohortId: 'a', comparisons: Array.from({ length: 4 }, () => ({ ...identity, windows: [improved()] })) }])).toThrow('Too many comparisons') + expect(() => summarize([{ cohortId: 'a', comparisons: [{ ...identity, windows: [improved()] }, { ...identity, windows: [regressed()] }] }])).toThrow('Duplicate comparison') + expect(() => summarize([night('a', [{ ...improved(), endMs: T }])])).toThrow('bounded and nonoverlapping') + expect(() => summarize([night('a', [{ ...improved(), startMs: T + 0.5 }])])).toThrow('bounded and nonoverlapping') + expect(() => summarize([night('a', [{ ...improved(), endMs: T + 86400001 }])])).toThrow('bounded and nonoverlapping') + expect(() => summarize([night('a', [improved(), { ...improved(), startMs: T + 10000, endMs: T + 30000 }])])).toThrow('bounded and nonoverlapping') + expect(() => summarize([night('a', Array.from({ length: 61 }, (_, index) => ({ ...improved(), startMs: T + index * 20000, endMs: T + index * 20000 + 10000 })))])).toThrow('bounded and nonoverlapping') + }) +}) diff --git a/src/biometrics/backtest/tests/night-summary.test.ts b/src/biometrics/backtest/tests/night-summary.test.ts new file mode 100644 index 000000000..9de654059 --- /dev/null +++ b/src/biometrics/backtest/tests/night-summary.test.ts @@ -0,0 +1,56 @@ +// @vitest-environment node +import { expect, it } from 'vitest' +import { compareStageOutputs } from '../stage-comparison' +import { summarizeStageNights } from '../night-summary' +import { previewReferenceImport } from '../../references/normalize' +const startMs = 1791673200000 +function night(cohortId: string, duration: number, improved: boolean, missing = false) { + const endMs = startMs + duration + const references = missing + ? [] + : previewReferenceImport({ side: 'left', subjectId: 'fixture', source: 'test', rows: [ + { recordKey: 'stage', metric: 'sleep_stages', value: 'light', unit: 'stage', startMs, endMs }, + ] }).measurements + const score = compareStageOutputs([{ start: startMs, end: endMs, stage: improved ? 'deep' : 'light' }], + [{ start: startMs, end: endMs, stage: improved ? 'light' : 'deep' }], references, startMs, endMs) + return { cohortId, comparisons: [{ baseline: 'default', candidate: 'candidate', baselineHash: 'a', candidateHash: 'b', windows: [{ ...score, startMs, endMs }] }] } +} +it('keeps equal-night effects distinct from pooled duration effects and includes unscored nights', () => { + const result = summarizeStageNights([night('long', 900000, true), night('short', 100000, false), night('unknown', 500000, true, true)])[0] + expect(result).toMatchObject({ totalNights: 3, scoredNights: 2, unscoredNights: 1, improvedAgreementNights: 1, + regressedAgreementNights: 1, equalNightMeanAgreementDifference: 0, observedNightRange: { minimum: -1, maximum: 1 } }) + expect(result.pooledCommonSupport.candidate.agreement).toBe(0.9) + expect(result.pooledCommonSupport.baseline.agreement).toBe(0.1) + expect(result.nights[2].commonAgreementDifference).toBeNull() +}) +it('combines window confusion durations before computing nightly F1', () => { + const first = night('one', 900000, true), second = night('two', 100000, false) + second.comparisons[0].windows[0].startMs += 900000 + second.comparisons[0].windows[0].endMs += 900000 + first.comparisons[0].windows.push(second.comparisons[0].windows[0]) + const result = summarizeStageNights([first])[0] + expect(result.nights[0].commonSupport.candidate.agreement).toBe(0.9) + expect(result.nights[0].commonSupport.candidate.macroF1).toBeCloseTo(0.9 / 1.9) + expect(result.nights[0].commonSupport.candidate.pairedDurationMs).toBe(1000000) +}) +it('never merges different hashes or counts duplicated cohort/comparison evidence twice', () => { + const first = night('one', 100000, true), second = night('two', 100000, true) + second.comparisons[0].candidateHash = 'different' + expect(summarizeStageNights([first, second])).toHaveLength(2) + expect(() => summarizeStageNights([first, first])).toThrow('whole-night') + first.comparisons.push(first.comparisons[0]) + expect(() => summarizeStageNights([first])).toThrow('Duplicate comparison') +}) +it('keeps all-missing evidence unknown', () => { + const result = summarizeStageNights([night('missing', 100000, true, true)])[0] + expect(result.equalNightMeanAgreementDifference).toBeNull() + expect(result.observedNightRange).toBeNull() + expect(result.pooledCommonSupport.candidate.agreement).toBeNull() + expect(summarizeStageNights([])).toEqual([]) +}) + +it('rejects overlapping windows instead of double-counting their support', () => { + const input = night('one', 100000, true) + input.comparisons[0].windows.push(input.comparisons[0].windows[0]) + expect(() => summarizeStageNights([input])).toThrow('nonoverlapping') +}) diff --git a/src/biometrics/backtest/tests/numeric-comparison.test.ts b/src/biometrics/backtest/tests/numeric-comparison.test.ts new file mode 100644 index 000000000..87f759ef5 --- /dev/null +++ b/src/biometrics/backtest/tests/numeric-comparison.test.ts @@ -0,0 +1,59 @@ +// @vitest-environment node +import { expect, it } from 'vitest' +import { compareNumericConfigurations } from '../numeric-comparison' +import { DEFAULT_ALIGNMENT } from '../../references/alignment' +import { previewReferenceImport } from '../../references/normalize' + +const T = Date.UTC(2026, 9, 10, 23) +const refs = (rows: unknown[]) => previewReferenceImport({ side: 'left', source: 'test', subjectId: 'test', rows }).measurements +const point = (timestampMs: number, value = 60) => ({ recordKey: String(timestampMs), metric: 'heart_rate', timestampMs, value, unit: 'bpm' }) +const output = (label: 'default' | 'current' | 'candidate', rows: [number, number | null][]) => ({ label, configHash: label, + observations: rows.map(([offset, value]) => ({ timestampMs: T + offset, value })) }) + +it('exposes dropped difficult readings despite improved own-support error', () => { + const [result] = compareNumericConfigurations('heart_rate', null, + [output('default', [[0, 60], [60000, 100]]), output('candidate', [[0, 60]])], + refs([point(T), point(T + 60000)]), DEFAULT_ALIGNMENT) + expect(result).toMatchObject({ commonObservationCount: 1, lostObservationTimestamps: 1, addedObservationTimestamps: 0, + coverageReduced: true, differences: { ownMae: -20, commonMae: 0, referenceCoverage: -0.5 } }) +}) +it('recomputes interval aggregation on identical sample support', () => { + const [result] = compareNumericConfigurations('heart_rate', null, + [output('default', [[0, 60], [60000, 100]]), output('candidate', [[0, 60]])], + refs([{ recordKey: 'interval', metric: 'heart_rate', startMs: T, endMs: T + 120000, value: 60, unit: 'bpm', aggregation: 'mean' }]), DEFAULT_ALIGNMENT) + expect(result.differences).toMatchObject({ ownMae: -20, commonMae: 0 }) + expect(result.commonSupport.baseline.metrics?.mae).toBe(0) +}) +it('uses all three configurations even for the default-current comparison', () => { + const results = compareNumericConfigurations('heart_rate', null, + [output('default', [[0, 60], [60000, 90]]), output('current', [[0, 62], [60000, 60]]), output('candidate', [[0, 61]])], + refs([point(T), point(T + 60000)]), DEFAULT_ALIGNMENT) + expect(results).toHaveLength(3) + expect(results.every(result => result.commonObservationCount === 1)).toBe(true) + expect(results.find(result => result.candidate === 'current')?.differences.commonMae).toBe(2) +}) +it('does not treat nearby output timestamps as common support', () => { + const [result] = compareNumericConfigurations('heart_rate', null, + [output('default', [[0, 60]]), output('candidate', [[1, 60]])], refs([point(T)]), DEFAULT_ALIGNMENT) + expect(result).toMatchObject({ commonObservationCount: 0, differences: { commonMae: null, ownMae: 0 }, + lostObservationTimestamps: 1, addedObservationTimestamps: 1 }) +}) +it('excludes duplicate timestamps including null-value duplicates', () => { + const [result] = compareNumericConfigurations('heart_rate', null, + [output('default', [[0, 60], [0, null], [60000, 60]]), output('candidate', [[0, 60], [60000, 62]])], + refs([point(T), point(T + 60000)]), DEFAULT_ALIGNMENT) + expect(result).toMatchObject({ commonObservationCount: 1, ambiguousObservationTimestamps: { baseline: 1, candidate: 0 }, + differences: { commonMae: 2, commonSignedBias: 2 } }) +}) +it('preserves fixed clock offsets and null metrics without compatible reference evidence', () => { + const outputs = [output('default', [[0, 60]]), output('candidate', [[0, 62]])] + const [shifted] = compareNumericConfigurations('heart_rate', null, outputs, refs([point(T + 60000)]), + { ...DEFAULT_ALIGNMENT, pointToleranceMs: 0, clockOffsetMs: 60000 }) + expect(shifted.differences.commonMae).toBe(2) + const [missing] = compareNumericConfigurations('heart_rate', null, outputs, [], DEFAULT_ALIGNMENT) + expect(missing.differences.commonMae).toBeNull() + const [incompatible] = compareNumericConfigurations('window_ibi_variability_v1', 300000, outputs, + refs([{ recordKey: 'watch', metric: 'hrv_rmssd', timestampMs: T, value: 40, unit: 'ms', windowMs: 300000 }]), DEFAULT_ALIGNMENT) + expect(incompatible.commonSupport.baseline.status).toBe('incompatible') + expect(incompatible.differences.commonMae).toBeNull() +}) diff --git a/src/biometrics/backtest/tests/numeric-night-summary.test.ts b/src/biometrics/backtest/tests/numeric-night-summary.test.ts new file mode 100644 index 000000000..ba1dce8c6 --- /dev/null +++ b/src/biometrics/backtest/tests/numeric-night-summary.test.ts @@ -0,0 +1,53 @@ +// @vitest-environment node +import { createHash } from 'node:crypto' +import { expect, it } from 'vitest' +import { createNumericNightSummary } from '../numeric-night-summary' +import { createNightUncertainty } from '../uncertainty-core' +import { compareNumericConfigurations } from '../numeric-comparison' +import { DEFAULT_ALIGNMENT } from '../../references/alignment' +import { previewReferenceImport } from '../../references/normalize' +const summarize = createNumericNightSummary(createNightUncertainty(text => createHash('sha256').update(text).digest(), 'equal_night_mean_common_mae_difference')) +const T = Date.UTC(2026, 9, 10) +const policy = { version: 'paired-night-percentile-v1' as const, seed: 'numeric', minimumNights: 3, resamples: 200 } +function window(offset: number, count: number, error: number, missing = false) { + const observations = Array.from({ length: count }, (_, i) => ({ timestampMs: T + offset + i * 60000, value: 60 })) + const references = missing ? [] : previewReferenceImport({ side: 'left', source: 'fixture', subjectId: 'fixture', rows: observations.map((row, i) => ({ recordKey: String(i), metric: 'heart_rate', timestampMs: row.timestampMs, value: 60, unit: 'bpm' })) }).measurements + return { startMs: T + offset, endMs: T + offset + count * 60000, metrics: compareNumericConfigurations('heart_rate', null, [ + { label: 'default', configHash: 'baseline', observations }, + { label: 'candidate', configHash: 'candidate', observations: observations.map(row => ({ ...row, value: row.value + error })) }, + ], references, DEFAULT_ALIGNMENT) } +} +it('combines windows by paired reference count before giving nights equal weight', () => { + const [result] = summarize([ + { cohortId: 'a', numericComparisons: [window(0, 1, 10), window(120000, 3, 2)] }, + { cohortId: 'b', numericComparisons: [window(0, 1, 0)] }, + { cohortId: 'missing', numericComparisons: [window(0, 1, 0, true)] }, + ], policy) + expect(result).toMatchObject({ totalNights: 3, scoredNights: 2, unscoredNights: 1, equalNightMeanMaeDifference: 2, + pooledCommonSupport: { candidate: { mae: 3.2, pairedCount: 5 } }, + uncertainty: { status: 'insufficient_nights', estimate: 2, omittedNights: 1, statistic: 'equal_night_mean_common_mae_difference' } }) + expect(result.nights[0].commonMaeDifference).toBe(4) + expect(result.pooledCommonSupport.candidate).not.toHaveProperty('p95AbsoluteError') +}) +it('bootstraps numeric differences above one in metric units reproducibly', () => { + const nights = [2, 10, 30].map((error, i) => ({ cohortId: String(i), numericComparisons: [window(0, 1, error)] })) + const [result] = summarize(nights, policy) + expect(result.uncertainty.status).toBe('estimated') + expect(result.uncertainty).toEqual(summarize([...nights].reverse(), policy)[0].uncertainty) + expect(result.uncertainty.interval?.upper).toBeGreaterThan(10) + expect(result.uncertainty.estimate).toBe(14) +}) +it('keeps missing evidence unknown and rejects duplicate or overlapping evidence', () => { + const night = { cohortId: 'one', numericComparisons: [window(0, 1, 0, true)] } + expect(summarize([night])[0]).toMatchObject({ equalNightMeanMaeDifference: null, scoredNights: 0, uncertainty: { interval: null, estimate: null } }) + expect(() => summarize([night, night])).toThrow('cohorts') + expect(() => summarize([{ ...night, numericComparisons: [window(0, 2, 0), window(60000, 1, 0)] }])).toThrow('windows') + const duplicate = window(0, 1, 0) + duplicate.metrics.push(duplicate.metrics[0]) + expect(() => summarize([{ ...night, numericComparisons: [duplicate] }])).toThrow('Duplicate') +}) +it('does not combine different immutable configurations', () => { + const first = window(0, 1, 2), second = window(0, 1, 3) + second.metrics[0].candidateHash = 'other' + expect(summarize([{ cohortId: 'a', numericComparisons: [first] }, { cohortId: 'b', numericComparisons: [second] }])).toHaveLength(2) +}) diff --git a/src/biometrics/backtest/tests/numeric-subgroups.test.ts b/src/biometrics/backtest/tests/numeric-subgroups.test.ts new file mode 100644 index 000000000..e3476e23f --- /dev/null +++ b/src/biometrics/backtest/tests/numeric-subgroups.test.ts @@ -0,0 +1,28 @@ +// @vitest-environment node +import { expect, it } from 'vitest' +import { subgroupReferences, subgroupBoundaryExclusions, subgroupObjectiveEvidence } from '../numeric-subgroups' +import { previewReferenceImport } from '../../references/normalize' +const T = Date.UTC(2026, 9, 10) +const window = { startMs: T, endMs: T + 120000 } +const rows = previewReferenceImport({ side: 'left', source: 'test', subjectId: 'test', rows: [ + { recordKey: 'early', metric: 'heart_rate', timestampMs: T, value: 60, unit: 'bpm' }, + { recordKey: 'middle', metric: 'heart_rate', timestampMs: T + 60000, value: 60, unit: 'bpm' }, + { recordKey: 'first', metric: 'heart_rate', startMs: T, endMs: T + 60000, value: 60, unit: 'bpm', aggregation: 'mean' }, + { recordKey: 'spanning', metric: 'heart_rate', startMs: T + 30000, endMs: T + 90000, value: 60, unit: 'bpm', aggregation: 'mean' }, +] }).measurements +it('keeps point/interval semantics and midpoint boundaries without splitting reference records', () => { + expect(subgroupReferences('reference_points', rows, window).map(row => row.recordKey)).toEqual(['early', 'middle']) + expect(subgroupReferences('reference_intervals', rows, window).map(row => row.recordKey)).toEqual(['first', 'spanning']) + expect(subgroupReferences('window_first_half', rows, window).map(row => row.recordKey)).toEqual(['early', 'first']) + expect(subgroupReferences('window_second_half', rows, window).map(row => row.recordKey)).toEqual(['middle']) + expect(subgroupBoundaryExclusions('window_first_half', rows, window)).toBe(1) + expect(subgroupBoundaryExclusions('reference_intervals', rows, window)).toBe(0) +}) +it('weights supported nights equally and never hides unsupported nights in the subgroup mean', () => { + const identity = { baseline: 'default', candidate: 'candidate', metric: 'heart_rate', baselineHash: 'a', candidateHash: 'b' } + const nights = [{ commonMaeDifference: -4, commonSupport: { candidate: { pairedCount: 100 } } }, { commonMaeDifference: 6, commonSupport: { candidate: { pairedCount: 10 } } }] + const summary = { ...identity, name: 'window_second_half', nights } + expect(subgroupObjectiveEvidence([summary], identity, 10)).toEqual([{ name: 'window_second_half', pairedNights: 2, maeDifference: 1 }]) + expect(subgroupObjectiveEvidence([summary], identity, 11)).toEqual([{ name: 'window_second_half', pairedNights: 1, maeDifference: null }]) + expect(subgroupObjectiveEvidence([summary], { ...identity, candidateHash: 'different' }, 10)).toEqual([]) +}) diff --git a/src/biometrics/backtest/tests/numeric-traces.test.ts b/src/biometrics/backtest/tests/numeric-traces.test.ts new file mode 100644 index 000000000..423e0940d --- /dev/null +++ b/src/biometrics/backtest/tests/numeric-traces.test.ts @@ -0,0 +1,46 @@ +// @vitest-environment node +import { expect, it } from 'vitest' +import { pageNumericTraces } from '../numeric-trace-page' +const hash = 'b'.repeat(64) +const request = { id: '11111111-1111-4111-8111-111111111111', expectedContentHash: 'a'.repeat(64), configuration: 'candidate' as const, limit: 2 } +function fixture() { + const reading = { side: 'left', sourceSide: 'right', configHash: hash, timestampMs: 1000, heartRate: 60, breathingRate: null, hrvIndex: null, + hrvMetric: 'window_ibi_variability_v1', qualityScore: 0, flags: null, limitations: ['unknown_context'], hrRaw: 65 } + return { version: 'waveform-backtest-report-v1', side: 'left', protocol: { mode: 'signal_replay', clockOffsetMs: 60000 }, + nights: [{ cohortId: 'one', evaluations: ['default', 'candidate'].map(label => ({ label, configHash: hash, revisionId: null, windows: [{ startMs: 1000, endMs: 5000 }] })) }], + waveform: { outputs: [{ cohortId: 'one', configHash: hash, readings: [ + { ...reading, timestampMs: 999 }, reading, { ...reading, side: 'right', heartRate: 99 }, + { ...reading, timestampMs: 2000, heartRate: null }, { ...reading, timestampMs: 4000 }, { ...reading, timestampMs: 5000 }, + ] }] } } +} +it('pages exact dataset-side readings without offset, downsampling, opposite-side values or private extra fields', () => { + const report = fixture(), before = structuredClone(report) + const first = pageNumericTraces(report, request) + expect(first).toMatchObject({ totalItems: 3, side: 'left', downsampled: false, clockOffsetMs: 60000, + provenance: 'retained_waveform_recomputation', nextCursor: { offset: 2, contentHash: request.expectedContentHash } }) + expect(first.items.map(row => row.timestampMs)).toEqual([1000, 2000]) + expect(first.items[1].heartRate).toBeNull() + expect(first.items[0]).not.toHaveProperty('hrRaw') + expect(first.items[0].sourceSide).toBe('right') + const last = pageNumericTraces(report, { ...request, offset: first.nextCursor?.offset }) + expect(last.items.map(row => row.timestampMs)).toEqual([4000]) + expect(last.nextCursor).toBeNull() + expect(pageNumericTraces(report, { ...request, configuration: 'default' }).items).toEqual(first.items) + expect(report).toEqual(before) +}) +it('rejects missing provenance, ambiguous output identity and invalid selections', () => { + expect(() => pageNumericTraces({ ...fixture(), side: undefined }, request)).toThrow('provenance') + const duplicate = fixture() + duplicate.waveform.outputs.push(duplicate.waveform.outputs[0]) + expect(() => pageNumericTraces(duplicate, request)).toThrow('ambiguous') + const mismatch = fixture() + mismatch.waveform.outputs[0].readings[1].configHash = 'c'.repeat(64) + expect(() => pageNumericTraces(mismatch, request)).toThrow('configuration mismatch') + const repeated = fixture() + repeated.nights[0].evaluations.push(repeated.nights[0].evaluations[1]) + expect(() => pageNumericTraces(repeated, request)).toThrow('Duplicate numeric configuration') + expect(() => pageNumericTraces(fixture(), { ...request, cohortId: 'absent' })).toThrow('not evaluated') + expect(() => pageNumericTraces(fixture(), { ...request, configuration: 'current' })).toThrow('not evaluated') + expect(() => pageNumericTraces(fixture(), { ...request, offset: 4 })).toThrow('cursor') + expect(() => pageNumericTraces(fixture(), { ...request, limit: 501 })).toThrow() +}) diff --git a/src/biometrics/backtest/tests/objective-finalization.test.ts b/src/biometrics/backtest/tests/objective-finalization.test.ts new file mode 100644 index 000000000..2b4e8b7f1 --- /dev/null +++ b/src/biometrics/backtest/tests/objective-finalization.test.ts @@ -0,0 +1,118 @@ +// @vitest-environment node +import { expect, it } from 'vitest' +import { assessNumericObjective, type ObjectivePolicy, type ObjectiveEvidence } from '../objective-policy' +import { finalizeObjectiveReport, finalizeStageObjectiveReport, objectiveOperations } from '../objective-finalization' +import { assessStageObjective, type StageObjectiveEvidence } from '../stage-objective' +import { stageObjectiveTestPolicy } from './fixtures/stage-objective' +import type { createJobOperations } from '../job-operations' +const policy: ObjectivePolicy = { version: 'numeric-objective-v1', qualificationId: 'qualified-test', metric: 'heart_rate', minimumNights: 3, + minimumPairedReferencesPerNight: 10, maximumLostObservationFraction: 0.1, minimumMaeImprovement: 2, maximumNightMaeRegression: 1, + maximumRegressedNightFraction: 0.34, maximumSubgroupMaeRegression: 1, requiredSubgroups: [], + operational: { maximumWallMs: 1000, maximumPeakRssBytes: 1000, maximumIngestionAgeMs: 1000, + observations: { version: 'sampled-process-group-v1', maximumMemoryGapMs: 1000, maximumIngestionGapMs: 5000 } } } +const evidence: ObjectiveEvidence = { metricCompatible: true, split: 'holdout', holdoutUse: 1, intervalUpper: -2, subgroups: [], operational: null, + nights: ['a', 'b', 'c'].map(cohortId => ({ cohortId, pairedReferences: 10, maeDifference: -2, baselineObservationCount: 10, lostObservations: 1 })) } +const operations = (): ReturnType['finish']> => ({ version: 'backtest-job-operations-v1', wallMs: 1000, + scope: 'admission_through_prepublication', memory: { version: 'process-group-rss-observations-v1', scope: 'sum_rss_of_replay_process_group', + attempts: 3, samples: 3, allSamplesAvailable: true, peakObservedRssBytes: 1000, maximumObservationGapMs: 1000, intervalMs: 1000, exactKernelPeak: false }, + ingestion: { source: 'supervisor_live_piezo_receipt_age', checks: 2, samples: 2, allChecksMeasured: true, qualificationId: 'qualified-test', + maximumObservedAgeMs: 1000, maximumObservationGapMs: 5000, nominalIntervalMs: 5000, continuousMeasurement: false } }) +const report = () => ({ objectiveAssessments: ['default', 'current'].map(baseline => ({ baseline, candidate: 'candidate', baselineHash: baseline, + candidateHash: 'candidate', ...assessNumericObjective(policy, evidence, null) })) }) +it('finalizes measured operational evidence only under explicit sampled policy and trusted designation', () => { + const result = finalizeObjectiveReport(report(), policy, operations(), policy) + expect(result.objectiveAssessments).toEqual(expect.arrayContaining([expect.objectContaining({ status: 'improved', evidence: { ...evidence, operational: objectiveOperations(operations(), policy) } })])) + expect(finalizeObjectiveReport(report(), policy, operations(), null).objectiveAssessments).toEqual(expect.arrayContaining([expect.objectContaining({ status: 'insufficient_evidence', missingEvidence: ['policy_not_designated'] })])) +}) +it('does not repurpose sampled peaks for legacy policies or incomplete observations', () => { + const legacy = { ...policy, operational: { maximumWallMs: 1000, maximumPeakRssBytes: 1000, maximumIngestionAgeMs: 1000 } } + expect(objectiveOperations(operations(), legacy)).toBeNull() + for (const patch of [{ memory: null }, { memory: { ...operations().memory, samples: 1 } }, + { ingestion: { ...operations().ingestion, allChecksMeasured: false } }, { ingestion: { ...operations().ingestion, qualificationId: null } }]) { + expect(objectiveOperations({ ...operations(), ...patch }, policy)).toBeNull() + } +}) +it('treats sampling gaps and qualification mismatches as missing evidence at strict boundaries', () => { + for (const patch of [{ maximumIngestionGapMs: 5001 }, { maximumMemoryGapMs: 1001 }, { qualificationId: 'different' }]) { + const operational = objectiveOperations(operations(), policy) + if (!operational?.observations) throw new Error('Missing measured operations') + expect(assessNumericObjective(policy, { ...evidence, operational: { ...operational, observations: { ...operational.observations, ...patch } } }, policy).status).toBe('insufficient_evidence') + } +}) +it('replaces worker-supplied operational evidence and rejects mismatched policies or missing comparisons', () => { + const forged = report() + for (const row of forged.objectiveAssessments) row.evidence.operational = { wallMs: 0, peakRssBytes: 0, maximumIngestionAgeMs: 0 } + expect(finalizeObjectiveReport(forged, policy, null, policy).objectiveAssessments).toEqual(expect.arrayContaining([expect.objectContaining({ status: 'insufficient_evidence', missingEvidence: ['operational_evidence_unavailable'] })])) + expect(() => finalizeObjectiveReport(report(), { ...policy, minimumMaeImprovement: 3 }, operations(), policy)).toThrow('identity') + expect(() => finalizeObjectiveReport({ objectiveAssessments: report().objectiveAssessments.slice(0, 1) }, policy, operations(), policy)).toThrow() +}) + +const metrics = ['heart_rate', 'breathing_rate', 'window_ibi_variability_v1'] as const +const policies = metrics.map(metric => ({ ...policy, metric })) +const multiReport = () => ({ objectiveAssessments: policies.flatMap(item => ['default', 'current'].map(baseline => ({ + baseline, candidate: 'candidate', baselineHash: baseline, candidateHash: 'candidate', ...assessNumericObjective(item, evidence, null), +}))) }) + +it('reassesses both baselines independently for all three numeric metrics', () => { + const result = finalizeObjectiveReport(multiReport(), policies, operations(), policies) + expect(result.objectiveAssessments).toHaveLength(6) + for (const metric of metrics) { + expect(result.objectiveAssessments).toEqual(expect.arrayContaining(['default', 'current'].map(baseline => expect.objectContaining({ + baseline, policy: expect.objectContaining({ metric }), status: 'improved', designated: true, + })))) + } + const partial = finalizeObjectiveReport(multiReport(), policies, operations(), policies.slice(0, 2)) + expect(partial.objectiveAssessments).toEqual(expect.arrayContaining([expect.objectContaining({ + policy: expect.objectContaining({ metric: 'window_ibi_variability_v1' }), status: 'insufficient_evidence', missingEvidence: ['policy_not_designated'], + })])) +}) + +it('rejects missing, duplicated, substituted or policy-mismatched metric/baseline pairs', () => { + const source = multiReport() + const rows = source.objectiveAssessments + for (const objectiveAssessments of [rows.slice(1), [...rows, rows[0]], [rows[0], rows[0], ...rows.slice(2)], + rows.map((row, index) => index === 5 ? { ...row, policy: { ...row.policy, minimumMaeImprovement: 3 } } : row)]) { + expect(() => finalizeObjectiveReport({ objectiveAssessments }, policies, operations(), policies)).toThrow() + } + expect(() => finalizeObjectiveReport(source, [policies[0], policies[0], policies[2]], operations(), policies)).toThrow('Duplicate') + expect(() => finalizeObjectiveReport(source, policies, operations(), [policies[0], policies[0]])).toThrow('Duplicate') +}) + +it('uses each metric policy for operational bounds and replaces child telemetry for every metric', () => { + const strict = policies.map(item => item.metric === 'breathing_rate' + ? { ...item, operational: { ...item.operational, maximumWallMs: 999 } } + : item) + const raw = { objectiveAssessments: strict.flatMap(item => ['default', 'current'].map(baseline => ({ + baseline, candidate: 'candidate', baselineHash: baseline, candidateHash: 'candidate', + ...assessNumericObjective(item, { ...evidence, operational: { wallMs: 0, peakRssBytes: 0, maximumIngestionAgeMs: 0 } }, item), + }))) } + const result = finalizeObjectiveReport(raw, strict, operations(), strict) + for (const row of result.objectiveAssessments as Array<{ policy: ObjectivePolicy, status: string, evidence: ObjectiveEvidence }>) { + expect(row.evidence.operational?.wallMs).toBe(1000) + expect(row.status).toBe(row.policy.metric === 'breathing_rate' ? 'regressed' : 'improved') + } +}) +it('passes reports through without a numeric policy and rejects mismatched assessment identities', () => { + expect(finalizeObjectiveReport({ untouched: true }, undefined, operations(), null)).toEqual({ untouched: true }) + expect(finalizeObjectiveReport({ untouched: true }, [], operations(), null)).toEqual({ untouched: true }) + expect(finalizeObjectiveReport(report(), [policy], operations(), [policy]).objectiveAssessments).toHaveLength(2) + expect(() => finalizeObjectiveReport(report(), [policy, policy], operations(), null)).toThrow('Duplicate numeric objective metric') + expect(() => finalizeObjectiveReport(report(), policy, operations(), [policy, policy])).toThrow('Duplicate numeric policy designation') + const foreign = { ...policy, metric: 'breathing_rate' as const } + expect(() => finalizeObjectiveReport(report(), foreign, operations(), null)).toThrow('identity mismatch') + const duplicated = report() + duplicated.objectiveAssessments[1].baseline = 'default' + expect(() => finalizeObjectiveReport(duplicated, policy, operations(), null)).toThrow('identity mismatch') +}) +it('finalizes stage objective reports only against the frozen policy identity', () => { + expect(finalizeStageObjectiveReport({ untouched: true }, undefined, operations(), null)).toEqual({ untouched: true }) + const stagePolicy = { ...stageObjectiveTestPolicy, operational: { ...stageObjectiveTestPolicy.operational, observations: { version: 'sampled-process-group-v1' as const, maximumMemoryGapMs: 1000, maximumIngestionGapMs: 5000 } } } + const stageEvidence: StageObjectiveEvidence = { split: 'holdout', holdoutUse: 1, intervalLower: 0.125, operational: null, + nights: ['a', 'b', 'c'].map(cohortId => ({ cohortId, pairedDurationMs: 800, agreementDifference: 0.125, macroF1Difference: 0.125, candidateOutputCoverage: 0.9, + baselineOutputMs: 1000, commonOutputMs: 900, perStage: stagePolicy.requiredStages.map(stage => ({ stage, referenceMs: 200, f1Difference: 0.125 })) })) } + const stageReport = { stageObjectiveAssessments: ['default', 'current'].map(baseline => ({ baseline, candidate: 'candidate', baselineHash: baseline, candidateHash: 'candidate', ...assessStageObjective(stagePolicy, stageEvidence, null) })) } + const measured = { ...operations(), ingestion: { ...operations().ingestion, qualificationId: stagePolicy.qualificationId } } + expect(finalizeStageObjectiveReport(stageReport, stagePolicy, measured, stagePolicy).stageObjectiveAssessments).toEqual(expect.arrayContaining([expect.objectContaining({ status: 'improved' })])) + expect(() => finalizeStageObjectiveReport({ stageObjectiveAssessments: [stageReport.stageObjectiveAssessments[0], stageReport.stageObjectiveAssessments[0]] }, stagePolicy, operations(), null)).toThrow('identity mismatch') + expect(() => finalizeStageObjectiveReport(stageReport, { ...stagePolicy, minimumNights: 4 }, operations(), null)).toThrow('identity mismatch') +}) diff --git a/src/biometrics/backtest/tests/objective-policy.test.ts b/src/biometrics/backtest/tests/objective-policy.test.ts new file mode 100644 index 000000000..4f7e1e463 --- /dev/null +++ b/src/biometrics/backtest/tests/objective-policy.test.ts @@ -0,0 +1,73 @@ +// @vitest-environment node +import { expect, it } from 'vitest' +import { assessNumericObjective, type ObjectiveEvidence, type ObjectivePolicy } from '../objective-policy' +import { experimentProtocolSchema } from '../contracts' +import { configHash } from '../../algorithms/canonical' +const policy: ObjectivePolicy = { version: 'numeric-objective-v1', qualificationId: 'test-only', metric: 'heart_rate', minimumNights: 3, + minimumPairedReferencesPerNight: 10, maximumLostObservationFraction: 0.1, minimumMaeImprovement: 2, + maximumNightMaeRegression: 1, maximumRegressedNightFraction: 0.34, maximumSubgroupMaeRegression: 1, requiredSubgroups: ['motion'], + operational: { maximumWallMs: 1000, maximumPeakRssBytes: 1000, maximumIngestionAgeMs: 1000 } } +const evidence = (): ObjectiveEvidence => ({ metricCompatible: true, split: 'holdout', holdoutUse: 1, + nights: ['a', 'b', 'c'].map(cohortId => ({ cohortId, pairedReferences: 10, maeDifference: -2, baselineObservationCount: 10, lostObservations: 1 })), + intervalUpper: -2, subgroups: [{ name: 'motion', pairedNights: 3, maeDifference: 1 }], operational: { wallMs: 1000, peakRssBytes: 1000, maximumIngestionAgeMs: 1000 } }) +it('only a designated qualified policy can award improvement at exact thresholds', () => { + expect(assessNumericObjective(policy, evidence(), policy).status).toBe('improved') + expect(assessNumericObjective(policy, evidence(), null)).toMatchObject({ status: 'insufficient_evidence', missingEvidence: ['policy_not_designated'] }) + expect(assessNumericObjective(policy, evidence(), { ...policy, qualificationId: 'different' }).status).toBe('insufficient_evidence') +}) +it('requires independent support, uncertainty, subgroups and measured operations', () => { + for (const patch of [{ split: 'tuning' as const }, { holdoutUse: 2 }, { intervalUpper: null }, { subgroups: [] }, { operational: null }, + { nights: evidence().nights.slice(0, 2) }, { nights: evidence().nights.map(row => ({ ...row, pairedReferences: 9 })) }]) { + expect(assessNumericObjective(policy, { ...evidence(), ...patch }, policy).status).toBe('insufficient_evidence') + } + expect(assessNumericObjective(policy, { ...evidence(), metricCompatible: false }, policy).status).toBe('incompatible') +}) +it('checks strict operational and coverage boundaries', () => { + for (const key of ['wallMs', 'peakRssBytes', 'maximumIngestionAgeMs'] as const) { + expect(assessNumericObjective(policy, { ...evidence(), operational: { wallMs: 1000, peakRssBytes: 1000, maximumIngestionAgeMs: 1000, [key]: 1001 } }, policy).status).toBe('regressed') + } + const data = evidence() + data.nights[0].lostObservations = 2 + expect(assessNumericObjective(policy, data, policy).failedThresholds).toContain('coverage_loss') +}) +it('does not hide night or subgroup regressions behind a better mean', () => { + const data = evidence() + data.nights[0].maeDifference = 1.01 + expect(assessNumericObjective(policy, data, policy).failedThresholds).toContain('night_regression') + data.nights[0].maeDifference = 1 + data.nights[1].maeDifference = 1 + expect(assessNumericObjective(policy, data, policy).failedThresholds).toContain('regressed_night_fraction') + data.subgroups[0].maeDifference = 1.01 + expect(assessNumericObjective(policy, data, policy).failedThresholds).toContain('subgroup_regression:motion') + expect(assessNumericObjective(policy, { ...evidence(), intervalUpper: -1.99 }, policy).status).toBe('no_material_change') +}) +it('pins thresholds in protocol identity and rejects duplicate evidence', () => { + const base = { version: 'descriptive-evaluation-v1', mode: 'signal_replay', assumedStageCalibrationQuality: 0, pointToleranceMs: 30000, clockOffsetMs: 0, objectivePolicy: policy } + expect(experimentProtocolSchema.safeParse({ ...base, mode: 'comparison' }).success).toBe(false) + const first = experimentProtocolSchema.parse(base) + const second = experimentProtocolSchema.parse({ ...base, objectivePolicy: { ...policy, minimumMaeImprovement: 3 } }) + expect(configHash(first)).not.toBe(configHash(second)) + const data = evidence() + data.nights.push(data.nights[0]) + expect(() => assessNumericObjective(policy, data, policy)).toThrow('identity') +}) + +it('freezes one policy per numeric metric without rewriting legacy protocols', () => { + const base = { version: 'descriptive-evaluation-v1', mode: 'signal_replay', assumedStageCalibrationQuality: 0, pointToleranceMs: 30000, clockOffsetMs: 0 } + const policies = ['heart_rate', 'breathing_rate', 'window_ibi_variability_v1'].map(metric => ({ ...policy, metric })) + expect(experimentProtocolSchema.parse({ ...base, objectivePolicies: policies }).objectivePolicies).toEqual(policies) + expect(experimentProtocolSchema.parse({ ...base, objectivePolicy: policy })).not.toHaveProperty('objectivePolicies') + for (const patch of [{ objectivePolicies: [] }, { objectivePolicies: [policy, policy] }, { objectivePolicies: policies, objectivePolicy: policy }, + { objectivePolicies: policies, mode: 'comparison' }, { objectivePolicies: policies, mode: 'stage_replay' }]) { + expect(experimentProtocolSchema.safeParse({ ...base, ...patch }).success).toBe(false) + } +}) +it('treats absent baseline coverage as missing evidence and unrequested sampling as unauthorized', () => { + const uncovered = evidence() + uncovered.nights[0].baselineObservationCount = 0 + uncovered.nights[0].lostObservations = 0 + expect(assessNumericObjective(policy, uncovered, policy)).toMatchObject({ status: 'insufficient_evidence', missingEvidence: expect.arrayContaining(['coverage_unavailable']) }) + const sampled = { ...evidence(), operational: { wallMs: 1000, peakRssBytes: 1000, maximumIngestionAgeMs: 1000, + observations: { version: 'sampled-process-group-v1' as const, qualificationId: policy.qualificationId, maximumMemoryGapMs: 1000, maximumIngestionGapMs: 1000 } } } + expect(assessNumericObjective(policy, sampled, policy)).toMatchObject({ status: 'insufficient_evidence', missingEvidence: expect.arrayContaining(['sampled_memory_not_authorized_by_policy']) }) +}) diff --git a/src/biometrics/backtest/tests/process-group-memory.test.ts b/src/biometrics/backtest/tests/process-group-memory.test.ts new file mode 100644 index 000000000..06bffa927 --- /dev/null +++ b/src/biometrics/backtest/tests/process-group-memory.test.ts @@ -0,0 +1,107 @@ +// @vitest-environment node +import { mkdtemp, mkdir, writeFile, rm } from 'node:fs/promises' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { afterEach, expect, it, vi } from 'vitest' +import { monitorProcessGroupMemory, sampleProcessGroupRss } from '../process-group-memory' +let root: string | undefined +afterEach(async () => { + vi.useRealTimers() + if (root) await rm(root, { recursive: true, force: true }) + root = undefined +}) +async function processFixture(pid: number, group: number, rss: string | null, state = 'S') { + if (!root) root = await mkdtemp(join(tmpdir(), 'replay-proc-')) + const directory = join(root, String(pid)) + await mkdir(directory, { recursive: true }) + await writeFile(join(directory, 'stat'), `${pid} (name with ) parentheses) ${state} 1 ${group} 0 0\n`) + if (rss !== null) await writeFile(join(directory, 'status'), rss) + return directory +} +it('sums only the owned process group and treats disappearing members as an observation race', async () => { + await processFixture(10, 10, 'VmRSS:\t100 kB\n') + await processFixture(11, 10, 'VmRSS:\t200 kB\n') + await processFixture(12, 12, 'VmRSS:\t999 kB\n') + await processFixture(13, 10, null) + await processFixture(14, 10, 'Name: zombie\n', 'Z') + expect(await sampleProcessGroupRss(10, root)).toBe(300 * 1024) + expect(await sampleProcessGroupRss(99, root)).toBeNull() +}) +it('reports unavailable memory for malformed owned RSS and excessive group size', async () => { + const directory = await processFixture(10, 10, 'VmRSS: unknown\n') + expect(await sampleProcessGroupRss(10, root)).toBeNull() + await writeFile(join(directory, 'status'), 'VmRSS: 99999999999999999999999 kB\n') + expect(await sampleProcessGroupRss(10, root)).toBeNull() + for (let pid = 10; pid < 75; pid++) await processFixture(pid, 10, 'VmRSS: 1 kB\n') + expect(await sampleProcessGroupRss(10, root)).toBeNull() +}) +it('reports unavailable memory for invalid groups, unreadable tables and malformed procfs records', async () => { + expect(await sampleProcessGroupRss(0, tmpdir())).toBeNull() + expect(await sampleProcessGroupRss(1.5, tmpdir())).toBeNull() + expect(await sampleProcessGroupRss(10, join(tmpdir(), 'replay-proc-missing'))).toBeNull() + const directory = await processFixture(10, 10, 'VmRSS:\t100 kB\n') + for (const stat of ['x'.repeat(16385), '10 (name S 1 10 0 0\n', '10 (name) S 1\n']) { + await writeFile(join(directory, 'stat'), stat) + expect(await sampleProcessGroupRss(10, root)).toBeNull() + } + await writeFile(join(directory, 'stat'), '10 (name) S 1 10 0 0\n') + await writeFile(join(directory, 'status'), `VmRSS:\t100 kB\n${'x'.repeat(16385)}`) + expect(await sampleProcessGroupRss(10, root)).toBeNull() + await writeFile(join(directory, 'status'), 'VmRSS:\t100 kB\n') + expect(await sampleProcessGroupRss(10, root)).toBe(100 * 1024) + if (!root) throw new Error('Missing fixture root') + await Promise.all(Array.from({ length: 4100 }, (_, index) => mkdir(join(root as string, String(index + 100)), { recursive: true }))) + expect(await sampleProcessGroupRss(10, root)).toBeNull() +}) +it('falls back to the platform sampler, records failed reads and samples on its interval', async () => { + vi.useFakeTimers() + const platform = monitorProcessGroupMemory(1) + await platform.sample() + expect(platform.snapshot()).toMatchObject({ attempts: 1, intervalMs: 1000, exactKernelPeak: false }) + await platform.stop() + const read = vi.fn().mockRejectedValueOnce(new Error('procfs unavailable')).mockResolvedValue(700) + const monitor = monitorProcessGroupMemory(1, { sample: read }) + await monitor.sample() + expect(monitor.snapshot()).toMatchObject({ attempts: 1, samples: 0, allSamplesAvailable: false, peakObservedRssBytes: null }) + await vi.advanceTimersByTimeAsync(2000) + await monitor.stop() + expect(read).toHaveBeenCalledTimes(3) + expect(monitor.snapshot()).toMatchObject({ attempts: 3, samples: 2, allSamplesAvailable: false, peakObservedRssBytes: 700 }) +}) +it('retains sampled peaks, missing samples and gaps and stops its periodic work', async () => { + vi.useFakeTimers() + let time = 0 + const read = vi.fn().mockResolvedValueOnce(100).mockResolvedValueOnce(null).mockResolvedValue(300) + const monitor = monitorProcessGroupMemory(1, { sample: read, now: () => time }) + await monitor.sample() + time = 2000 + await monitor.sample() + time = 2200 + await monitor.sample() + expect(monitor.snapshot()).toMatchObject({ attempts: 3, samples: 2, allSamplesAvailable: false, + peakObservedRssBytes: 300, maximumObservationGapMs: 2000, exactKernelPeak: false }) + await monitor.stop() + await vi.advanceTimersByTimeAsync(5000) + await monitor.sample() + expect(read).toHaveBeenCalledTimes(3) +}) +it('serializes slow reads and waits for them when stopped', async () => { + let resolve: (value: number) => void = () => {} + const read = vi.fn(() => new Promise((done) => { + resolve = done + })) + const monitor = monitorProcessGroupMemory(1, { sample: read }) + const first = monitor.sample(), second = monitor.sample() + await Promise.resolve() + expect(read).toHaveBeenCalledTimes(1) + let stopped = false + const stopping = monitor.stop().then(() => { + stopped = true + }) + await Promise.resolve() + expect(stopped).toBe(false) + resolve(500) + await Promise.all([first, second, stopping]) + expect(stopped).toBe(true) + expect(monitor.snapshot().peakObservedRssBytes).toBe(500) +}) diff --git a/src/biometrics/backtest/tests/raw-artifact-references.test.ts b/src/biometrics/backtest/tests/raw-artifact-references.test.ts new file mode 100644 index 000000000..585f5bae4 --- /dev/null +++ b/src/biometrics/backtest/tests/raw-artifact-references.test.ts @@ -0,0 +1,95 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { afterEach, beforeEach, expect, it } from 'vitest' +import * as schema from '@/src/db/biometrics-schema' + +let sqlite: Database.Database +let db: ReturnType> +beforeEach(() => { + sqlite = new Database(':memory:') + sqlite.pragma('foreign_keys = ON') + db = drizzle(sqlite, { schema }) + migrate(db, { migrationsFolder: 'src/db/biometrics-migrations' }) + for (const id of ['dataset-a', 'dataset-b']) db.insert(schema.biometricDatasets).values({ id, side: 'left', subjectId: 'fixture', + idempotencyKey: id, requestHash: id, contentHash: id, manifest: {}, snapshot: {}, byteLength: 4, createdAtMs: 1000 }).run() + for (const contentHash of ['hash-a', 'hash-b']) db.insert(schema.biometricRawArtifacts).values({ contentHash, + name: `${contentHash}.RAW.gz`, compressedBytes: 100, expandedBytes: 200, createdAtMs: 1000 }).run() +}) +afterEach(() => sqlite.close()) + +it('shares retained bytes across datasets while preserving unique immutable positions', () => { + const link = { id: 'link-a', datasetId: 'dataset-a', artifactHash: 'hash-a', position: 0 } + db.insert(schema.biometricDatasetRawArtifacts).values(link).run() + db.insert(schema.biometricDatasetRawArtifacts).values({ ...link, id: 'link-b', datasetId: 'dataset-b' }).run() + expect(() => db.insert(schema.biometricDatasetRawArtifacts).values({ ...link, id: 'duplicate', position: 1 }).run()).toThrow('UNIQUE') + expect(() => db.insert(schema.biometricDatasetRawArtifacts).values({ ...link, id: 'reordered', artifactHash: 'hash-b' }).run()).toThrow('UNIQUE') + // Releasing one association leaves another live reference and retains the + // original position/identity for audit even after all references are released. + sqlite.prepare('UPDATE biometric_dataset_raw_artifacts SET released_at_ms = 2000 WHERE id = ?').run('link-a') + expect(sqlite.prepare('SELECT dataset_id FROM biometric_dataset_raw_artifacts WHERE artifact_hash = ? AND released_at_ms IS NULL').all('hash-a')) + .toEqual([{ dataset_id: 'dataset-b' }]) + expect(() => db.delete(schema.biometricRawArtifacts).run()).toThrow('FOREIGN KEY') + expect(() => db.delete(schema.biometricDatasets).run()).toThrow('FOREIGN KEY') +}) + +it('rejects dangling artifact references and duplicate storage names', () => { + expect(() => db.insert(schema.biometricDatasetRawArtifacts).values({ id: 'missing-dataset', datasetId: 'missing', artifactHash: 'hash-a', position: 0 }).run()).toThrow('FOREIGN KEY') + expect(() => db.insert(schema.biometricDatasetRawArtifacts).values({ id: 'missing-artifact', datasetId: 'dataset-a', artifactHash: 'missing', position: 0 }).run()).toThrow('FOREIGN KEY') + expect(() => db.insert(schema.biometricRawArtifacts).values({ contentHash: 'another-hash', name: 'hash-a.RAW.gz', + compressedBytes: 100, expandedBytes: 200, createdAtMs: 1000 }).run()).toThrow('UNIQUE') +}) + +it('releases only its own references and makes shared bytes collectible after the final release', async () => { + const { DatasetStore } = await import('../datasets') + let now = 2000 + const store = new DatasetStore(db, () => now) + for (const datasetId of ['dataset-a', 'dataset-b']) db.insert(schema.biometricDatasetRawArtifacts).values({ + id: datasetId, datasetId, artifactHash: 'hash-a', position: 0, + }).run() + expect(store.release('dataset-a').releasedAtMs).toBe(2000) + expect(sqlite.prepare('SELECT released_at_ms FROM biometric_raw_artifacts WHERE content_hash = ?').get('hash-a')).toEqual({ released_at_ms: null }) + expect(sqlite.prepare('SELECT released_at_ms FROM biometric_dataset_raw_artifacts WHERE dataset_id = ?').get('dataset-b')).toEqual({ released_at_ms: null }) + now = 3000 + store.release('dataset-b') + expect(sqlite.prepare('SELECT released_at_ms FROM biometric_raw_artifacts WHERE content_hash = ?').get('hash-a')).toEqual({ released_at_ms: 3000 }) + // Unrelated unreferenced artifacts are not touched by a dataset release. + expect(sqlite.prepare('SELECT released_at_ms FROM biometric_raw_artifacts WHERE content_hash = ?').get('hash-b')).toEqual({ released_at_ms: null }) + now = 4000 + expect(store.release('dataset-a').releasedAtMs).toBe(2000) + expect(sqlite.prepare('SELECT released_at_ms FROM biometric_dataset_raw_artifacts WHERE dataset_id = ?').get('dataset-a')).toEqual({ released_at_ms: 2000 }) + expect(sqlite.prepare('SELECT released_at_ms FROM biometric_raw_artifacts WHERE content_hash = ?').get('hash-a')).toEqual({ released_at_ms: 3000 }) +}) + +it.each(['queued', 'running', 'cancel_requested', 'interrupted'] as const)('preserves all retained inputs while a job is %s', async (state) => { + const { DatasetStore } = await import('../datasets') + db.insert(schema.biometricDatasetRawArtifacts).values({ id: 'link', datasetId: 'dataset-a', artifactHash: 'hash-a', position: 0 }).run() + db.insert(schema.biometricExperiments).values({ id: 'experiment', datasetId: 'dataset-a', side: 'left', protocol: {}, protocolHash: 'fixture', createdAtMs: 1000 }).run() + db.insert(schema.biometricBacktestJobs).values({ id: 'job', experimentId: 'experiment', idempotencyKey: 'job', requestHash: 'fixture', + input: {}, state, progress: {}, createdAtMs: 1000, updatedAtMs: 1000 }).run() + expect(() => new DatasetStore(db).release('dataset-a')).toThrow('Cancel or finish') + for (const table of ['biometric_datasets', 'biometric_raw_artifacts', 'biometric_dataset_raw_artifacts']) { + expect(sqlite.prepare(`SELECT count(*) AS n FROM ${table} WHERE released_at_ms IS NOT NULL`).get()).toEqual({ n: 0 }) + } +}) + +it('rolls back reference release if persisting the dataset tombstone fails', async () => { + const { DatasetStore } = await import('../datasets') + db.insert(schema.biometricDatasetRawArtifacts).values({ id: 'link', datasetId: 'dataset-a', artifactHash: 'hash-a', position: 0 }).run() + sqlite.exec('CREATE TRIGGER fail_release BEFORE UPDATE ON biometric_datasets BEGIN SELECT RAISE(ABORT, \'injected failure\'); END') + expect(() => new DatasetStore(db).release('dataset-a')).toThrow('injected failure') + expect(sqlite.prepare('SELECT released_at_ms FROM biometric_dataset_raw_artifacts').get()).toEqual({ released_at_ms: null }) + expect(sqlite.prepare('SELECT released_at_ms FROM biometric_raw_artifacts WHERE content_hash = ?').get('hash-a')).toEqual({ released_at_ms: null }) +}) + +it.each(['cancelled', 'succeeded', 'failed'] as const)('allows release after a job becomes %s', async (state) => { + const { DatasetStore } = await import('../datasets') + db.insert(schema.biometricDatasetRawArtifacts).values({ id: 'link', datasetId: 'dataset-a', artifactHash: 'hash-a', position: 0 }).run() + db.insert(schema.biometricExperiments).values({ id: 'experiment', datasetId: 'dataset-a', side: 'left', protocol: {}, protocolHash: 'fixture', createdAtMs: 1000 }).run() + db.insert(schema.biometricBacktestJobs).values({ id: 'job', experimentId: 'experiment', idempotencyKey: 'job', requestHash: 'fixture', + input: {}, state, progress: {}, createdAtMs: 1000, updatedAtMs: 1000 }).run() + expect(new DatasetStore(db, () => 2000).release('dataset-a').releasedAtMs).toBe(2000) + expect(sqlite.prepare('SELECT released_at_ms FROM biometric_dataset_raw_artifacts').get()).toEqual({ released_at_ms: 2000 }) + expect(sqlite.prepare('SELECT state FROM biometric_backtest_jobs').get()).toEqual({ state }) +}) diff --git a/src/biometrics/backtest/tests/raw-catalog.test.ts b/src/biometrics/backtest/tests/raw-catalog.test.ts new file mode 100644 index 000000000..d019aadb1 --- /dev/null +++ b/src/biometrics/backtest/tests/raw-catalog.test.ts @@ -0,0 +1,107 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import * as schema from '@/src/db/biometrics-schema' +import { RawSourceCatalog } from '../raw-catalog' + +let sqlite: Database.Database +let catalog: RawSourceCatalog +let now: number +const digest = 'a'.repeat(64) +function inspection(source = 'capture.RAW.gz', sha256 = digest) { + return { source_name: source, retained: false, artifact: { name: `${sha256}.RAW.gz`, sha256, compressed_bytes: 100, expanded_bytes: 200 }, + inventory: { version: 'retained-raw-inventory-v1', artifact_sha256: sha256, + decoded_records: 2, piezo_records: 1, pump_records: 1, usable_dual_frames: 1, empty_channel_frames: 0, + oversize_channel_frames: 0, invalid_timestamps: 0, timestamp_reversals: 0, duplicate_timestamps: 0, + first_event_ms: 1000, last_event_ms: 2000, + channels: { left1: { samples: 10, lost_markers: 1, truncated_tail_bytes: 0 }, right1: { samples: 10, lost_markers: 0, truncated_tail_bytes: 0 } }, + ordered_waveform_available: true, limitations: ['Fixture bounds do not establish continuous coverage.'] } } +} +beforeEach(() => { + now = 3000 + sqlite = new Database(':memory:') + sqlite.pragma('foreign_keys = ON') + const db = drizzle(sqlite, { schema }) + migrate(db, { migrationsFolder: 'src/db/biometrics-migrations' }) + catalog = new RawSourceCatalog(db, () => now) +}) +afterEach(() => sqlite.close()) + +describe('inspected RAW source catalog', () => { + it('persists bounded evidence without claiming retained or current availability', () => { + const input = inspection() + const id = catalog.record(input) + expect(catalog.read(id)).toEqual({ id, inspection: input, inspectedAtMs: 3000, unavailableAtMs: null, availability: 'requires_reverification' }) + now = 4000 + expect(catalog.record(input)).toBe(id) + const reopened = new RawSourceCatalog(drizzle(sqlite, { schema })) + expect(reopened.read(id).inspectedAtMs).toBe(4000) + expect(sqlite.prepare('SELECT * FROM biometric_raw_artifacts').all()).toEqual([]) + expect(sqlite.prepare('SELECT * FROM biometric_datasets').all()).toEqual([]) + }) + + it('preserves replaced identities and prevents a late failure from invalidating newer bytes', () => { + const oldId = catalog.record(inspection()) + now = 5000 + const newId = catalog.record(inspection('capture.RAW.gz', 'b'.repeat(64))) + expect(newId).not.toBe(oldId) + expect(catalog.read(oldId)).toMatchObject({ unavailableAtMs: 5000, availability: 'unavailable', inspection: inspection() }) + now = 6000 + catalog.markUnavailable(oldId) + expect(catalog.read(oldId).unavailableAtMs).toBe(5000) + expect(catalog.page({ startMs: 1000, endMs: 2001 }).items.map(row => row.id)).toEqual([newId]) + catalog.markUnavailable(newId) + expect(catalog.page({ startMs: 1000, endMs: 2001 }).items).toEqual([]) + now = 7000 + expect(catalog.record(inspection())).toBe(oldId) + expect(catalog.read(oldId)).toMatchObject({ unavailableAtMs: null, inspectedAtMs: 7000 }) + }) + + it('rejects conflicting evidence atomically and validates cross-field identities', () => { + const id = catalog.record(inspection()) + const conflict = inspection() + conflict.artifact.expanded_bytes++ + expect(() => catalog.record(conflict)).toThrow('different inspection evidence') + expect(catalog.read(id).inspection).toEqual(inspection()) + const changed = inspection() + changed.inventory.artifact_sha256 = 'b'.repeat(64) + expect(() => catalog.record(changed)).toThrow('inspected bytes') + expect(() => catalog.record({ ...inspection(), retained: true })).toThrow() + expect(() => catalog.record(inspection('../escape.RAW.gz'))).toThrow() + expect(() => catalog.record({ ...inspection(), extra: true })).toThrow() + expect(() => catalog.read('0'.repeat(64))).toThrow('not found') + }) + + it('rejects inconsistent counts, event bounds, descriptors and capabilities', () => { + const changes = [ + { first_event_ms: null }, { first_event_ms: 3000 }, { decoded_records: 1 }, + { empty_channel_frames: 1 }, { ordered_waveform_available: false }, { invalid_timestamps: 1 }, + { timestamp_reversals: 1 }, { oversize_channel_frames: 1 }, + { channels: { ...inspection().inventory.channels, left1: { samples: 1, lost_markers: 2, truncated_tail_bytes: 0 } } }, + ] + for (const change of changes) expect(() => catalog.record({ ...inspection(), inventory: { ...inspection().inventory, ...change } })).toThrow() + expect(() => catalog.record({ ...inspection(), artifact: { ...inspection().artifact, name: 'wrong.RAW.gz' } })).toThrow('content hash') + expect(() => catalog.record({ ...inspection(), artifact: { ...inspection().artifact, expanded_bytes: Number.MAX_SAFE_INTEGER + 1 } })).toThrow() + }) + + it('pages potential overlap with half-open request boundaries and no duplicate identities', () => { + const ids = Array.from({ length: 52 }, (_, index) => catalog.record(inspection(`capture-${index}.RAW.gz`))).sort() + const first = catalog.page({ startMs: 2000, endMs: 3000 }) + expect(first.items.map(row => row.id)).toEqual(ids.slice(0, 50)) + expect(first.nextCursor).toBe(ids[49]) + const second = catalog.page({ startMs: 2000, endMs: 3000, afterId: first.nextCursor ?? undefined }) + expect(second.items.map(row => row.id)).toEqual(ids.slice(50)) + expect(second.nextCursor).toBeNull() + expect(catalog.page({ startMs: 0, endMs: 1000 }).items).toEqual([]) + expect(catalog.page({ startMs: 2001, endMs: 3000 }).items).toEqual([]) + expect(() => catalog.page({ startMs: 1000, endMs: 1000 })).toThrow('nonempty') + }) + + it('detects corrupted indexed metadata before returning an inspected descriptor', () => { + const id = catalog.record(inspection()) + sqlite.prepare('UPDATE biometric_raw_sources SET first_event_ms = 999 WHERE id = ?').run(id) + expect(() => catalog.read(id)).toThrow('integrity check') + }) +}) diff --git a/src/biometrics/backtest/tests/raw-collection-child.test.ts b/src/biometrics/backtest/tests/raw-collection-child.test.ts new file mode 100644 index 000000000..b93a8eb7f --- /dev/null +++ b/src/biometrics/backtest/tests/raw-collection-child.test.ts @@ -0,0 +1,57 @@ +// @vitest-environment node +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, expect, it } from 'vitest' +import { runRawCollectionChild } from '../raw-collection-child' +let directory: string, executable: string +const location = () => ({ pythonPath: executable, modulesPath: directory, databasePath: join(directory, 'db'), retainedDirectory: join(directory, 'raw') }) +beforeEach(async () => { + directory = await mkdtemp(join(tmpdir(), 'collection-child-')) + executable = join(directory, 'worker') +}) +afterEach(() => rm(directory, { recursive: true, force: true })) +async function worker(body: string) { + await writeFile(executable, `#!${process.execPath}\nlet input='';process.stdin.on('data',c=>input+=c);process.stdin.on('end',()=>{${body}});`, { mode: 0o700 }) +} +const result = { version: 'raw-collection-result-v1', released: 2, orphans: 1 } +const emit = `process.stdout.write(${JSON.stringify(JSON.stringify(result))});` +it('uses fixed module, trusted paths and an input with no caller-controlled retention settings', async () => { + const args = ['-B', '-m', 'common.biometrics.collection_worker', '--database-path', location().databasePath, '--retained-directory', location().retainedDirectory] + await worker(`if(JSON.stringify(process.argv.slice(2))!==${JSON.stringify(JSON.stringify(args))} || input!==${JSON.stringify(JSON.stringify({ version: 'raw-collection-work-v1' }))})process.exit(1);${emit}`) + expect(await runRawCollectionChild(location())).toEqual(result) +}) +it('rejects malformed output and unsuccessful exit', async () => { + await worker('process.stdout.write("{}");') + await expect(runRawCollectionChild(location())).rejects.toThrow('invalid or incomplete') + await worker(`${emit}process.exitCode=1;`) + await expect(runRawCollectionChild(location())).rejects.toThrow('invalid or incomplete') +}) +it('bounds output, diagnostics and time, and cancels a running child', async () => { + await worker('process.stdout.write("x".repeat(1025));') + await expect(runRawCollectionChild(location())).rejects.toThrow('output exceeded') + await worker('process.stderr.write("x".repeat(16385));') + await expect(runRawCollectionChild(location())).rejects.toThrow('diagnostics exceeded') + await worker('setInterval(()=>{},1000);') + await expect(runRawCollectionChild(location(), { timeoutMs: 50 })).rejects.toThrow('time budget') + const controller = new AbortController() + const promise = runRawCollectionChild(location(), { signal: controller.signal }) + setTimeout(() => controller.abort(), 30) + await expect(promise).rejects.toThrow('cancelled') +}) +it('rejects invalid budgets, pre-cancelled work and unstartable workers, and escalates to SIGKILL', async () => { + expect(() => runRawCollectionChild(location(), { timeoutMs: 120001 })).toThrow('time budget') + expect(() => runRawCollectionChild(location(), { timeoutMs: 9 })).toThrow('time budget') + const controller = new AbortController() + controller.abort() + await expect(runRawCollectionChild(location(), { signal: controller.signal })).rejects.toThrow('cancelled') + await expect(runRawCollectionChild({ ...location(), pythonPath: join(directory, 'missing') })).rejects.toThrow('could not start') + await worker('process.on("SIGTERM",()=>{});setInterval(()=>{},1000);') + await expect(runRawCollectionChild(location(), { timeoutMs: 1500 })).rejects.toThrow('time budget') +}) +it('ignores output after a failure and tolerates bounded diagnostics on success', async () => { + await worker('process.stderr.write("x".repeat(16385));setTimeout(()=>{process.stderr.write("more");process.stdout.write("late")},150);process.on("SIGTERM",()=>setTimeout(()=>process.exit(0),200));') + await expect(runRawCollectionChild(location())).rejects.toThrow('diagnostics exceeded') + await worker(`process.stderr.write("note");${emit}`) + expect(await runRawCollectionChild(location())).toEqual(result) +}) diff --git a/src/biometrics/backtest/tests/raw-inspection-child.test.ts b/src/biometrics/backtest/tests/raw-inspection-child.test.ts new file mode 100644 index 000000000..89d9ca649 --- /dev/null +++ b/src/biometrics/backtest/tests/raw-inspection-child.test.ts @@ -0,0 +1,58 @@ +// @vitest-environment node +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, expect, it } from 'vitest' +import { runRawInspectionChild } from '../raw-inspection-child' +let directory: string, executable: string +const input = { afterName: null } +const location = () => ({ pythonPath: executable, modulesPath: directory, archiveDirectory: directory }) +beforeEach(async () => { + directory = await mkdtemp(join(tmpdir(), 'inspection-child-')) + executable = join(directory, 'worker') +}) +afterEach(() => rm(directory, { recursive: true, force: true })) +async function worker(body: string) { + await writeFile(executable, `#!${process.execPath}\nsetTimeout(() => process.exit(91), 3000).unref(); let input='';process.stdin.on('data',c=>input+=c);process.stdin.on('end',()=>{${body}});`, { mode: 0o700 }) +} +const emit = (value: unknown) => `process.stdout.write(${JSON.stringify(JSON.stringify(value))});` +it('uses the fixed module and clean environment and sends only a validated cursor', async () => { + await worker(`if(JSON.stringify(process.argv.slice(2))!==${JSON.stringify(JSON.stringify(['-B', '-m', 'common.biometrics.inspection_worker', '--archive-directory', directory]))} || process.env.RAW_TEST_SECRET || process.env.PYTHONPATH!==${JSON.stringify(directory)} || input!==${JSON.stringify(JSON.stringify({ version: 'raw-inspection-work-v1', after_name: null }))})process.exit(1);${emit({ status: 'complete', nextCursor: null })}`) + process.env.RAW_TEST_SECRET = 'private' + try { + expect(await runRawInspectionChild(location(), input)).toEqual({ status: 'complete', nextCursor: null }) + } + finally { delete process.env.RAW_TEST_SECRET } + expect(() => runRawInspectionChild(location(), { afterName: '../a.RAW.gz' })).toThrow() + expect(() => runRawInspectionChild(location(), input, { timeoutMs: 120001 })).toThrow('time budget') +}) +it.each([ + '{', '{}', '{"status":"complete","nextCursor":null}\n{}', + '{"status":"source_unavailable","nextCursor":"../a.RAW.gz"}', + '{"status":"complete","nextCursor":null,"extra":true}', +])('rejects malformed or additional output %s', async (output) => { + await worker(`process.stdout.write(${JSON.stringify(output)});`) + await expect(runRawInspectionChild(location(), input)).rejects.toThrow('invalid or incomplete') +}) +it('requires advancing cursors, valid UTF-8 and successful exit', async () => { + await worker(emit({ status: 'source_unavailable', nextCursor: 'a.RAW.gz' })) + await expect(runRawInspectionChild(location(), { afterName: 'a.RAW.gz' })).rejects.toThrow('invalid or incomplete') + await worker(`${emit({ status: 'complete', nextCursor: null })}process.stdout.write(Buffer.from([0xc3]));`) + await expect(runRawInspectionChild(location(), input)).rejects.toThrow('invalid or incomplete') + await worker(`${emit({ status: 'complete', nextCursor: null })}process.exitCode=1;`) + await expect(runRawInspectionChild(location(), input)).rejects.toThrow('invalid or incomplete') +}) +it('bounds output, diagnostics, elapsed time and cancellation', async () => { + await worker('process.stdout.write(\'x\'.repeat(65537));') + await expect(runRawInspectionChild(location(), input)).rejects.toThrow('output exceeded') + await worker('process.stderr.write(\'x\'.repeat(16385));') + await expect(runRawInspectionChild(location(), input)).rejects.toThrow('diagnostics exceeded') + await worker('setInterval(()=>{},1000);') + await expect(runRawInspectionChild(location(), input, { timeoutMs: 30 })).rejects.toThrow('time budget') + const controller = new AbortController() + const running = runRawInspectionChild(location(), input, { signal: controller.signal }) + controller.abort() + await expect(running).rejects.toThrow('cancelled') + await expect(runRawInspectionChild(location(), input, { signal: controller.signal })).rejects.toThrow('cancelled') + await expect(runRawInspectionChild({ ...location(), pythonPath: join(directory, 'missing') }, input)).rejects.toThrow('could not start') +}) diff --git a/src/biometrics/backtest/tests/raw-publication-child.test.ts b/src/biometrics/backtest/tests/raw-publication-child.test.ts new file mode 100644 index 000000000..63d0756fd --- /dev/null +++ b/src/biometrics/backtest/tests/raw-publication-child.test.ts @@ -0,0 +1,67 @@ +// @vitest-environment node +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, expect, it } from 'vitest' +import { runRawPublicationChild } from '../raw-publication-child' +let directory: string, executable: string +const hash = 'a'.repeat(64) +// This protocol fixture never runs Python or publishes rows. +const work = { prepared: { contentHash: hash }, selections: [] } as unknown as Parameters[1] +const result = { type: 'result', id: '11111111-1111-4111-8111-111111111111', contentHash: hash, available: true } +const location = () => ({ pythonPath: executable, modulesPath: directory, archiveDirectory: directory, retainedDirectory: join(directory, 'raw'), databasePath: join(directory, 'test.db') }) +beforeEach(async () => { + directory = await mkdtemp(join(tmpdir(), 'publication-child-')) + executable = join(directory, 'worker') +}) +afterEach(() => rm(directory, { recursive: true, force: true })) +async function worker(body: string) { + await writeFile(executable, `#!${process.execPath}\nsetTimeout(() => process.exit(91), 3000).unref(); let input='';process.stdin.on('data',c=>input+=c);process.stdin.on('end',()=>{${body}});`, { mode: 0o700 }) +} +const emit = (value: unknown) => `process.stdout.write(${JSON.stringify(JSON.stringify(value))});` +it('uses fixed publication arguments, a clean environment and the versioned envelope', async () => { + await worker(`if(JSON.stringify(process.argv.slice(2))!==${JSON.stringify(JSON.stringify(['-B', '-m', 'common.biometrics.publication_worker', '--database', location().databasePath, '--archive-directory', directory, '--retained-directory', location().retainedDirectory]))} || process.env.RAW_TEST_SECRET || process.env.PYTHONPATH!==${JSON.stringify(directory)} || input!==${JSON.stringify(JSON.stringify({ version: 'waveform-publication-work-v1', ...work }))})process.exit(1);${emit(result)}`) + process.env.RAW_TEST_SECRET = 'private' + try { + expect(await runRawPublicationChild(location(), work)).toEqual(result) + } + finally { delete process.env.RAW_TEST_SECRET } + expect(() => runRawPublicationChild(location(), work, { timeoutMs: 120001 })).toThrow('time budget') +}) +it.each([ + {}, { ...result, contentHash: 'b'.repeat(64) }, { ...result, id: 'bad' }, { ...result, extra: true }, +])('rejects invalid identities or unexpected fields %#', async (output) => { + await worker(emit(output)) + await expect(runRawPublicationChild(location(), work)).rejects.toThrow('invalid or incomplete') +}) +it('rejects extra output, invalid UTF-8 and unsuccessful exit after a response', async () => { + for (const suffix of ['process.stdout.write("{}");', 'process.stdout.write(Buffer.from([0xc3]));', 'process.exitCode=1;']) { + await worker(`${emit(result)}${suffix}`) + await expect(runRawPublicationChild(location(), work)).rejects.toThrow('invalid or incomplete') + } +}) +it('bounds input, output, diagnostics and execution, and cancels work', async () => { + await expect(runRawPublicationChild(location(), { ...work, padding: 'x'.repeat(10 * 1024 * 1024) } as typeof work)).rejects.toThrow('input exceeds') + await worker('process.stdout.write("x".repeat(65537));') + await expect(runRawPublicationChild(location(), work)).rejects.toThrow('output exceeded') + await worker('process.stderr.write("x".repeat(16385));') + await expect(runRawPublicationChild(location(), work)).rejects.toThrow('diagnostics exceeded') + await worker('setInterval(()=>{},1000);') + await expect(runRawPublicationChild(location(), work, { timeoutMs: 30 })).rejects.toThrow('time budget') + const controller = new AbortController() + const running = runRawPublicationChild(location(), work, { signal: controller.signal }) + controller.abort() + await expect(running).rejects.toThrow('cancelled') + await expect(runRawPublicationChild(location(), work, { signal: controller.signal })).rejects.toThrow('cancelled') + await expect(runRawPublicationChild({ ...location(), pythonPath: join(directory, 'missing') }, work)).rejects.toThrow('could not start') +}) +it('escalates to SIGKILL, ignores output after a failure and tolerates bounded diagnostics', async () => { + await worker('process.on("SIGTERM",()=>{});setInterval(()=>{},1000);') + await expect(runRawPublicationChild(location(), work, { timeoutMs: 1500 })).rejects.toThrow('time budget') + await worker('process.stderr.write("x".repeat(16385));setTimeout(()=>{process.stderr.write("more");process.stdout.write("late")},150);process.on("SIGTERM",()=>setTimeout(()=>process.exit(0),200));') + await expect(runRawPublicationChild(location(), work)).rejects.toThrow('diagnostics exceeded') + await writeFile(executable, `#!${process.execPath}\nprocess.exit(3)`, { mode: 0o700 }) + await expect(runRawPublicationChild(location(), { ...work, padding: 'x'.repeat(4 * 1024 * 1024) } as typeof work)).rejects.toThrow('input pipe closed') + await worker(`process.stderr.write("note");${emit(result)}`) + expect(await runRawPublicationChild(location(), work)).toEqual(result) +}) diff --git a/src/biometrics/backtest/tests/reference-traces.test.ts b/src/biometrics/backtest/tests/reference-traces.test.ts new file mode 100644 index 000000000..8b8d7ed08 --- /dev/null +++ b/src/biometrics/backtest/tests/reference-traces.test.ts @@ -0,0 +1,50 @@ +// @vitest-environment node +import { expect, it } from 'vitest' +import { pageReferenceTraces } from '../reference-trace-page' +import { evaluateBacktest } from '../evaluator' +import { backtestFixture } from './fixtures/backtest' +import { previewReferenceImport } from '../../references/normalize' +const request = { id: '11111111-1111-4111-8111-111111111111', expectedContentHash: 'a'.repeat(64), cohortId: 'night', metric: 'heart_rate' as const, limit: 1 } +function fixture() { + const work = backtestFixture(), report = evaluateBacktest(work) + return { report, dataset: { ...work.dataset, contentHash: report.datasetHash } } +} +it('pages frozen source revisions without annotations or submitted identity and preserves point times', () => { + const { report, dataset } = fixture() + const row = dataset.snapshot.references[0].measurements[1] + row.original.annotation = 'private note' + row.original.subjectId = 'private subject' + dataset.snapshot.references.push({ ...dataset.snapshot.references[0], importId: 'second', revision: 2, source: 'another source' }) + const first = pageReferenceTraces(report, dataset, request) + expect(first).toMatchObject({ totalItems: 2, side: 'left', provenance: 'frozen_dataset_reference', timestampBasis: 'reference_source_time', downsampled: false, + nextCursor: { offset: 1, contentHash: request.expectedContentHash }, items: [{ source: 'fixture', importRevision: 1, temporalKind: 'point', value: 62, startMs: row.startMs }] }) + expect(JSON.stringify(first)).not.toContain('private') + expect(first.items[0]).not.toHaveProperty('original') + const second = pageReferenceTraces(report, dataset, { ...request, offset: first.nextCursor?.offset }) + expect(second.items[0].importId).toBe('second') + expect(second.nextCursor).toBeNull() +}) +it('keeps interval semantics and incompatible HRV visible without claiming a paired score', () => { + const { report, dataset } = fixture() + const start = dataset.snapshot.sessions[0].startMs + dataset.snapshot.references[0].measurements.push(...previewReferenceImport({ side: 'left', source: 'watch', subjectId: 'A', rows: [ + { recordKey: 'hrv', metric: 'hrv_rmssd', value: 30, unit: 'ms', startMs: start, endMs: start + 300000, windowMs: 300000 }, + ] }).measurements) + const page = pageReferenceTraces(report, dataset, { ...request, metric: 'window_ibi_variability_v1' }) + expect(page.items[0]).toMatchObject({ temporalKind: 'interval', metric: 'hrv_rmssd', compatibility: 'incompatible_metric_definition', endMs: start + 300000 }) + expect(page.items[0]).not.toHaveProperty('paired') +}) +it('filters scoring windows and fails on stale identities or invalid pages', () => { + const { report, dataset } = fixture() + const row = dataset.snapshot.references[0].measurements[1] + dataset.snapshot.references[0].measurements.push({ ...row, startMs: dataset.snapshot.sessions[0].endMs, endMs: dataset.snapshot.sessions[0].endMs }) + expect(pageReferenceTraces(report, dataset, request).totalItems).toBe(1) + expect(() => pageReferenceTraces(report, { ...dataset, contentHash: 'other' }, request)).toThrow('hash changed') + expect(() => pageReferenceTraces(report, dataset, { ...request, cohortId: 'absent' })).toThrow('not evaluated') + expect(() => pageReferenceTraces(report, dataset, { ...request, offset: 2 })).toThrow('cursor') +}) +it('refuses reports without a readable identity or scoring windows', () => { + const { report, dataset } = fixture() + expect(() => pageReferenceTraces({ ...report, datasetId: 'not-a-uuid' }, dataset, request)).toThrow('identity is unavailable') + expect(() => pageReferenceTraces({ ...report, nights: [{ ...report.nights[0], evaluations: [] }] }, dataset, request)).toThrow('windows unavailable') +}) diff --git a/src/biometrics/backtest/tests/report-summary.test.ts b/src/biometrics/backtest/tests/report-summary.test.ts new file mode 100644 index 000000000..8b19c486f --- /dev/null +++ b/src/biometrics/backtest/tests/report-summary.test.ts @@ -0,0 +1,34 @@ +// @vitest-environment node +import { expect, it } from 'vitest' +import { evaluateBacktest } from '../evaluator' +import { dataHash } from '../../algorithms/canonical' +import { backtestFixture } from './fixtures/backtest' +import { summarizeBacktestResult } from '../report-summary' +import { pageStageTraces } from '../trace-page' + +it('preserves all scores and the retained artifact hash while omitting trace rows', () => { + const report = evaluateBacktest(backtestFixture()) + const contentHash = dataHash(report) + const descriptor = { report, available: true, contentHash } + const summary = summarizeBacktestResult(descriptor) + if (!summary) throw new Error('Expected a summary') + expect(summary).toMatchObject({ contentHash, contentHashScope: 'complete_retained_report', projection: 'scores_without_stage_trace_rows_v1' }) + const expected = structuredClone(report) as unknown as { nights: Array<{ evaluations: Array<{ windows: Array<{ blocks?: unknown }> }> }> } + for (const night of expected.nights) { + for (const evaluation of night.evaluations) { + for (const window of evaluation.windows) delete window.blocks + } + } + expect(summary.report).toEqual(expected) + expect(dataHash(report)).toBe(contentHash) + expect(pageStageTraces(report, { id: '11111111-1111-4111-8111-111111111111', expectedContentHash: contentHash, configuration: 'default' }).totalItems).toBeGreaterThan(0) + expect(JSON.stringify(summary.report).length).toBeLessThan(JSON.stringify(report).length) +}) +it('distinguishes missing and released reports without manufacturing content', () => { + expect(summarizeBacktestResult(null)).toBeNull() + expect(summarizeBacktestResult({ available: false, report: null, contentHash: 'retained' })).toMatchObject({ report: null, available: false, contentHash: 'retained' }) +}) +it('rejects unknown versions and oversize scores instead of silently truncating them', () => { + expect(() => summarizeBacktestResult({ available: true, report: { version: 'future' } })).toThrow('summary format') + expect(() => summarizeBacktestResult({ available: true, report: { version: 'processed-backtest-report-v1', nights: [], extra: 'x'.repeat(2 * 1024 * 1024) } })).toThrow('transport limit') +}) diff --git a/src/biometrics/backtest/tests/runner-waveform.test.ts b/src/biometrics/backtest/tests/runner-waveform.test.ts new file mode 100644 index 000000000..e0ee6b174 --- /dev/null +++ b/src/biometrics/backtest/tests/runner-waveform.test.ts @@ -0,0 +1,120 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { execFile } from 'node:child_process' +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { promisify } from 'node:util' +import { afterAll, afterEach, beforeAll, beforeEach, expect, it, vi } from 'vitest' +import * as schema from '@/src/db/biometrics-schema' +import { configHash, dataHash } from '../../algorithms/canonical' +import { defaultConfig, manifestHash } from '../../algorithms/registry' +import { workerBuildHash } from '../child' +import { datasetManifestSchema } from '../contracts' +import { DatasetStore } from '../datasets' +import { BacktestJobStore } from '../jobs' +import { executeNextBacktest } from '../runner' +import { prepareWaveformDataset } from '../waveform-dataset' +import { executeWaveformInputs, verifyWaveformJobBuild } from '../waveform-execution' +import { stageObjectiveTestPolicy } from './fixtures/stage-objective' + +// The Python replay child is exercised by waveform-execution.test.ts; here the +// runner's signal-replay orchestration is driven through the real scoring worker. +vi.mock('../waveform-execution', () => ({ executeWaveformInputs: vi.fn(), verifyWaveformJobBuild: vi.fn() })) + +type Work = Parameters[1] +const python = 'c'.repeat(64), digest = 'a'.repeat(64), start = 1791673200000 +const resolvedHash = configHash(defaultConfig) +const protocol = { version: 'descriptive-evaluation-v1' as const, mode: 'signal_replay' as const, assumedStageCalibrationQuality: 0, pointToleranceMs: 30000, clockOffsetMs: 0 } +let sqlite: Database.Database +let db: ReturnType> +let jobs: BacktestJobStore +let directory: string +let evaluatorBuildHash: string +let datasetId: string +beforeAll(async () => { + directory = await mkdtemp(join(tmpdir(), 'biometric-runner-waveform-')) + await promisify(execFile)(process.execPath, ['scripts/build/biometrics-worker.mjs', directory]) + evaluatorBuildHash = await workerBuildHash(join(directory, 'worker.cjs')) +}) +afterAll(async () => rm(directory, { recursive: true, force: true })) +beforeEach(() => { + sqlite = new Database(':memory:') + db = drizzle(sqlite, { schema }) + migrate(db, { migrationsFolder: 'src/db/biometrics-migrations' }) + db.insert(schema.sleepRecords).values({ id: 1, side: 'left', enteredBedAt: new Date(start), leftBedAt: new Date(start + 600000), sleepDurationSeconds: 600, timesExitedBed: 0 }).run() + for (let minute = 0; minute < 10; minute++) db.insert(schema.vitals).values({ side: 'left', timestamp: new Date(start + minute * 60000), heartRate: 60, hrv: 30, breathingRate: 14 }).run() + const datasets = new DatasetStore(db) + const source = datasets.read(datasets.create({ side: 'left', subjectId: 'A', sessionIds: [1], referenceImportIds: [], timezone: 'UTC', idempotencyKey: 'source' }).id) + const cohort = datasetManifestSchema.parse(source.manifest).splitProtocol.assignments[0] + const prepared = prepareWaveformDataset({ ...source, manifest: datasetManifestSchema.parse(source.manifest) }, [{ cohortId: cohort.cohortId, + input: { version: 'retained-waveform-input-v1', retained_start_ms: cohort.startMs, score_start_ms: cohort.startMs, score_end_ms: cohort.endMs, contexts: [], + artifacts: [{ artifact: { sha256: digest, name: `${digest}.RAW.gz`, compressed_bytes: 1, expanded_bytes: 2 }, first_event_ms: cohort.startMs, last_event_ms: cohort.endMs - 1 }] } }], 'raw') + db.insert(schema.biometricDatasets).values({ ...prepared, byteLength: Buffer.byteLength(JSON.stringify(prepared.manifest)) + Buffer.byteLength(JSON.stringify(prepared.snapshot)) }).run() + datasetId = prepared.id + jobs = new BacktestJobStore(db) + vi.mocked(verifyWaveformJobBuild).mockResolvedValue(undefined) + vi.mocked(executeWaveformInputs).mockImplementation(async (_db, work: Work) => { + if (work.dataset.snapshot.version !== 'waveform-dataset-v1' || !work.job.waveform) throw new Error('Processed work reached the waveform executor') + const assignments = datasetManifestSchema.parse(work.dataset.manifest).splitProtocol.assignments.filter(row => row.split === work.job.split) + const configs = [...new Map(work.job.configs.map(config => [config.resolvedHash, config])).values()] + const outputs = assignments.flatMap(cohort => configs.map((config) => { + const input = work.dataset.snapshot.version === 'waveform-dataset-v1' ? work.dataset.snapshot.waveformInputs.find(row => row.cohortId === cohort.cohortId)?.input : undefined + if (!input) throw new Error('Missing retained cohort') + const readings = Array.from({ length: 10 }, (_, minute) => ({ side: 'left' as const, timestampMs: cohort.startMs + minute * 60000, heartRate: 60, hrvIndex: 30, + hrvMetric: 'window_ibi_variability_v1' as const, breathingRate: 14, qualityScore: 1, flags: [], hrRaw: 60, sourceSide: 'left' as const, configHash: config.resolvedHash, limitations: [] })) + return { cohortId: cohort.cohortId, configHash: config.resolvedHash, readings, run: { version: 'retained-waveform-run-v1' as const, build_hash: python, + decoded_records: 10, raw_records: 10, counts: { records: 10, accepted_frames: 10, empty_channel_frames: 0, pump_gated_frames: 0, preroll_outputs: 0, scored_outputs: readings.length }, + limitations: ['missing_pump_context'], config_hashes: { left: config.resolvedHash, right: work.job.waveform?.otherSide.resolvedHash ?? resolvedHash }, artifact_hashes: input.artifacts.map(row => row.artifact.sha256) } } + })) + return { hash: dataHash(outputs), outputs } + }) +}) +afterEach(async () => { + vi.clearAllMocks() + sqlite.close() + await rm(join(directory, 'promotion-qualifications.json'), { force: true }) +}) +const options = { owner: 'waveform-test', get workerPath() { + return join(directory, 'worker.cjs') +}, admit: async () => true } +const config = (label: 'default' | 'current' | 'candidate', revisionId: string | null = null) => ({ label, revisionId, resolved: defaultConfig, resolvedHash, manifestHash }) +const enqueue = (experimentProtocol: typeof protocol & { waveformStages?: boolean, stageObjectivePolicy?: typeof stageObjectiveTestPolicy }, configs = [config('default'), config('current')]) => { + const experiment = jobs.createExperiment(datasetId, experimentProtocol) + return jobs.enqueue({ evaluatorBuildHash, experimentId: experiment.id, idempotencyKey: 'waveform', split: 'tuning', configs, + waveform: { pythonBuildHash: python, otherSide: { manifestHash, resolvedHash, resolved: defaultConfig } } }).id +} + +it('replays retained waveforms through Python, re-verifies the pinned build and publishes a descriptive waveform report', async () => { + const jobId = enqueue(protocol) + expect(await executeNextBacktest(db, options)).toEqual({ state: 'succeeded', jobId }) + expect(executeWaveformInputs).toHaveBeenCalledTimes(1) + expect(executeWaveformInputs).toHaveBeenCalledWith(db, expect.objectContaining({ job: expect.objectContaining({ datasetId }) }), expect.objectContaining({ owner: 'waveform-test', signal: expect.any(AbortSignal) })) + expect(verifyWaveformJobBuild).toHaveBeenCalledTimes(1) + expect(jobs.result(jobId)?.report).toMatchObject({ version: 'waveform-backtest-report-v1', status: 'insufficient_evidence', qualificationWarning: null, + waveform: { pythonBuildHash: python, otherSideConfigHash: resolvedHash }, nights: [{ uniqueObservationCount: 10 }] }) +}) + +it('fails the job instead of publishing when the waveform build drifts after scoring', async () => { + const jobId = enqueue(protocol) + vi.mocked(verifyWaveformJobBuild).mockRejectedValueOnce(new Error('Waveform worker build changed since enqueue')) + expect(await executeNextBacktest(db, options)).toEqual({ state: 'stopped', jobId }) + expect(jobs.get(jobId)).toMatchObject({ state: 'failed' }) + expect(jobs.result(jobId)).toBeNull() +}) + +it.each([ + [['missing_pump_context'], null], + [[], 'Replay context is outside the installed qualification; this report is descriptive.'], +] as const)('keeps an installed waveform qualification only when every replay limitation %j was qualified', async (allowedLimitations, qualificationWarning) => { + const qualified = { ...protocol, waveformStages: true, stageObjectivePolicy: stageObjectiveTestPolicy } + const jobId = enqueue(qualified, [config('default'), config('current'), config('candidate', '33333333-3333-4333-8333-333333333333')]) + await writeFile(join(directory, 'promotion-qualifications.json'), JSON.stringify({ version: 'biometric-promotion-qualifications-v1', entries: [{ + qualificationId: stageObjectiveTestPolicy.qualificationId, manifestHash, evaluatorBuildHash, pythonBuildHash: python, protocol: qualified, + fixtureEvidenceHash: 'a'.repeat(64), podEvidenceHash: 'b'.repeat(64), waveformContext: { allowedLimitations: [...allowedLimitations] }, + }] })) + expect(await executeNextBacktest(db, options)).toEqual({ state: 'succeeded', jobId }) + expect(jobs.result(jobId)?.report).toMatchObject({ version: 'waveform-backtest-report-v1', status: 'no_material_change', qualificationWarning }) +}) diff --git a/src/biometrics/backtest/tests/runner.test.ts b/src/biometrics/backtest/tests/runner.test.ts new file mode 100644 index 000000000..f8eb58949 --- /dev/null +++ b/src/biometrics/backtest/tests/runner.test.ts @@ -0,0 +1,241 @@ +import { ReferenceStore } from '../../references/store' +import { previewReferenceImport } from '../../references/normalize' +import { configHash } from '../../algorithms/canonical' +import { defaultConfig, manifestHash } from '../../algorithms/registry' +import { stageObjectiveTestPolicy } from './fixtures/stage-objective' +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { execFile } from 'node:child_process' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { promisify } from 'node:util' +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest' +import * as schema from '@/src/db/biometrics-schema' +import { DatasetStore } from '../datasets' +import { BacktestJobStore } from '../jobs' +import { workerBuildHash } from '../child' +import { executeNextBacktest } from '../runner' +import { backtestFixture } from './fixtures/backtest' +let sqlite: Database.Database +let db: ReturnType> +let jobs: BacktestJobStore +let jobId: string +let directory: string +let evaluatorBuildHash: string +beforeAll(async () => { + directory = await mkdtemp(join(tmpdir(), 'biometric-runner-test-')) + await promisify(execFile)(process.execPath, ['scripts/build/biometrics-worker.mjs', directory]) + evaluatorBuildHash = await workerBuildHash(join(directory, 'worker.cjs')) +}) +afterAll(async () => rm(directory, { recursive: true, force: true })) +beforeEach(() => { + sqlite = new Database(':memory:') + db = drizzle(sqlite, { schema }) + migrate(db, { migrationsFolder: 'src/db/biometrics-migrations' }) + const fixture = backtestFixture() + const session = fixture.dataset.snapshot.sessions[0] + db.insert(schema.sleepRecords).values({ id: 1, side: 'left', enteredBedAt: new Date(session.startMs), leftBedAt: new Date(session.endMs), sleepDurationSeconds: 600, timesExitedBed: 0 }).run() + for (const row of session.vitals) db.insert(schema.vitals).values({ side: 'left', timestamp: new Date(row.timestampMs), heartRate: row.heartRate, hrv: row.hrv, breathingRate: row.breathingRate }).run() + const dataset = new DatasetStore(db).create({ side: 'left', subjectId: 'A', sessionIds: [1], referenceImportIds: [], timezone: 'UTC', idempotencyKey: 'run' }) + jobs = new BacktestJobStore(db) + const experiment = jobs.createExperiment(dataset.id, fixture.job.protocol) + jobId = jobs.enqueue({ evaluatorBuildHash, experimentId: experiment.id, configs: fixture.job.configs, + idempotencyKey: 'run', split: 'tuning' }).id +}) +afterEach(() => sqlite.close()) +const options = { owner: 'integration-test', get workerPath() { + return join(directory, 'worker.cjs') +}, admit: async () => true } + +describe('durable replay execution', () => { + it('runs retained data through a child and publishes an integrity-checked result without changing canonical observations', async () => { + const before = sqlite.prepare('SELECT * FROM vitals').all() + expect(await executeNextBacktest(db, options)).toEqual({ state: 'succeeded', jobId }) + expect(jobs.result(jobId)?.report).toMatchObject({ version: 'processed-backtest-report-v1', status: 'insufficient_evidence', nights: [{ uniqueObservationCount: 10 }] }) + expect(jobs.result(jobId)?.report).toMatchObject({ jobOperations: { ingestion: { allChecksMeasured: false, samples: 0, maximumObservedAgeMs: null } } }) + expect(sqlite.prepare('SELECT * FROM vitals').all()).toEqual(before) + expect(await executeNextBacktest(db, options)).toEqual({ state: 'idle' }) + }) + it('retains supervisor observations in the committed report and marks legacy boolean admission unmeasured', async () => { + let checks = 0 + expect(await executeNextBacktest(db, { ...options, admit: async () => { + checks++ + return true + }, + admissionReceipt: () => ({ allowed: true, sample: { qualificationId: 'test-policy', checkedAtMs: Date.now(), ingestionAgeMs: checks * 100 } }), + })).toEqual({ state: 'succeeded', jobId }) + expect(jobs.result(jobId)?.report).toMatchObject({ jobOperations: { version: 'backtest-job-operations-v1', + ingestion: { allChecksMeasured: true, checks: 2, samples: 2, maximumObservedAgeMs: 200, qualificationId: 'test-policy' } } }) + }) + it('finalizes retained stage objectives after the real scoring child completes', async () => { + const datasetId = jobs.getExperiment(jobs.get(jobId).experimentId).datasetId + jobs.cancel(jobId) + const fixture = backtestFixture() + const experiment = jobs.createExperiment(datasetId, { ...fixture.job.protocol, stageObjectivePolicy: stageObjectiveTestPolicy }) + const staged = jobs.enqueue({ evaluatorBuildHash, experimentId: experiment.id, idempotencyKey: 'stage-objective', split: 'tuning', + configs: [...fixture.job.configs, { ...fixture.job.configs[0], label: 'candidate' }] }) + expect(await executeNextBacktest(db, options)).toEqual({ state: 'succeeded', jobId: staged.id }) + expect(jobs.result(staged.id)?.report).toMatchObject({ status: 'no_material_change', stageObjectiveAssessments: [ + { baseline: 'default', status: 'insufficient_evidence', evidence: { operational: null } }, + { baseline: 'current', status: 'insufficient_evidence', evidence: { operational: null } }, + ] }) + }) + it.each([['wake', 'improved'], ['deep', 'regressed']] as const)('publishes %s candidate evidence as %s through the real scoring worker', async (stage, status) => { + jobs.cancel(jobId) + const first = 1791673200000 + const ids = Array.from({ length: 12 }, (_, index) => index + 10) + const rows = ids.flatMap((id, index) => { + const start = first + id * 86400000, end = start + 600000 + db.insert(schema.sleepRecords).values({ id, side: 'left', enteredBedAt: new Date(start), leftBedAt: new Date(end), sleepDurationSeconds: 600, timesExitedBed: 0 }).run() + for (let minute = 0; minute < 10; minute++) db.insert(schema.vitals).values({ side: 'left', timestamp: new Date(start + minute * 60000), heartRate: 60, hrv: 30, breathingRate: 14 }).run() + const middle = start + 360000 + index % 3 * 60000 + return [{ recordKey: `${id}-wake`, metric: 'sleep_stages', value: 'wake', unit: 'stage', startMs: start, endMs: middle }, + { recordKey: `${id}-light`, metric: 'sleep_stages', value: 'light', unit: 'stage', startMs: middle, endMs: end }] + }) + const input = { side: 'left' as const, source: 'synthetic-runner-test', subjectId: 'fixture', rows } + const reference = new ReferenceStore(db).commit({ import: input, previewHash: previewReferenceImport(input).summary.contentHash }) + const dataset = new DatasetStore(db).create({ side: 'left', subjectId: 'fixture', sessionIds: ids, referenceImportIds: [reference.id], timezone: 'UTC', idempotencyKey: 'qualified-test' }) + const policy = { ...stageObjectiveTestPolicy, requiredStages: ['wake' as const], maximumMacroF1Regression: 0.5, + operational: { maximumWallMs: 60000, maximumPeakRssBytes: 1000000000, maximumIngestionAgeMs: 1000, + observations: { version: 'sampled-process-group-v1' as const, maximumMemoryGapMs: 60000, maximumIngestionGapMs: 60000 } } } + const protocol = { ...backtestFixture().job.protocol, assumedStageCalibrationQuality: 1, stageObjectivePolicy: policy, + nightUncertainty: { version: 'paired-night-percentile-v1' as const, seed: 'runner-qualified', resamples: 200, minimumNights: 3 } } + const experiment = jobs.createExperiment(dataset.id, protocol) + const candidate = structuredClone(defaultConfig) + candidate.sleep_stages.ruleset.rules = [{ stage, when: [] }] + const configurations = [...backtestFixture().job.configs, { label: 'candidate' as const, revisionId: '33333333-3333-4333-8333-333333333333', + resolved: candidate, resolvedHash: configHash(candidate), manifestHash }] + const request = { evaluatorBuildHash, experimentId: experiment.id, configs: configurations, split: 'holdout' as const } + const staged = jobs.enqueue({ ...request, idempotencyKey: 'qualified' }) + const registry = join(directory, 'promotion-qualifications.json') + await writeFile(registry, JSON.stringify({ version: 'biometric-promotion-qualifications-v1', entries: [{ + qualificationId: policy.qualificationId, manifestHash, evaluatorBuildHash, protocol, + fixtureEvidenceHash: 'a'.repeat(64), podEvidenceHash: 'b'.repeat(64), waveformContext: null, + }] })) + const before = sqlite.prepare('SELECT * FROM vitals').all() + const measuredOptions = { ...options, admissionReceipt: () => ({ allowed: true, + sample: { qualificationId: policy.qualificationId, checkedAtMs: Date.now(), ingestionAgeMs: 10 }, + memory: { version: 'process-group-rss-observations-v1' as const, scope: 'sum_rss_of_replay_process_group' as const, + attempts: 2, samples: 2, allSamplesAvailable: true, peakObservedRssBytes: 1000, maximumObservationGapMs: 1, intervalMs: 1000 as const, exactKernelPeak: false as const } }) } + try { + expect(await executeNextBacktest(db, measuredOptions)).toEqual({ state: 'succeeded', jobId: staged.id }) + expect(jobs.result(staged.id)?.report).toMatchObject({ status, + stageObjectiveAssessments: [{ status, designated: true }, { status, designated: true }] }) + if (status === 'improved') expect(jobs.result(staged.id)?.report).toMatchObject({ evidence: 'qualified_reference_agreement' }) + expect(sqlite.prepare('SELECT * FROM vitals').all()).toEqual(before) + const reused = jobs.enqueue({ ...request, idempotencyKey: 'reused' }) + expect(await executeNextBacktest(db, measuredOptions)).toEqual({ state: 'succeeded', jobId: reused.id }) + expect(jobs.result(reused.id)?.report).toMatchObject({ status: 'insufficient_evidence' }) + } + finally { await rm(registry, { force: true }) } + }) + + it('refuses to execute a different worker bundle after enqueue', async () => { + sqlite.prepare('UPDATE biometric_backtest_jobs SET input = json_set(input, \'$.evaluatorBuildHash\', ?)').run('0'.repeat(64)) + expect(await executeNextBacktest(db, options)).toEqual({ state: 'stopped', jobId }) + expect(jobs.get(jobId).state).toBe('failed') + expect(jobs.result(jobId)).toBeNull() + }) + + it('defers work before claim when resource admission is denied or shutdown arrives', async () => { + expect(await executeNextBacktest(db, { ...options, admit: async () => false })).toEqual({ state: 'deferred' }) + expect(jobs.get(jobId).attempt).toBe(0) + const controller = new AbortController() + expect(await executeNextBacktest(db, { ...options, signal: controller.signal, admit: async () => { + controller.abort() + return true + } })).toEqual({ state: 'deferred' }) + expect(jobs.get(jobId).attempt).toBe(0) + }) + it('interrupts a running child when health declines and never publishes its result', async () => { + const stalledPath = join(directory, 'stalled.cjs') + await writeFile(stalledPath, 'process.stdin.resume(); setInterval(() => {}, 1000)') + const hash = await workerBuildHash(stalledPath) + sqlite.prepare('UPDATE biometric_backtest_jobs SET input = json_set(input, \'$.evaluatorBuildHash\', ?)').run(hash) + let checks = 0 + const result = await executeNextBacktest(db, { ...options, workerPath: stalledPath, admit: async () => ++checks === 1 }) + expect(result).toEqual({ state: 'stopped', jobId }) + expect(checks).toBe(2) + expect(jobs.get(jobId)).toMatchObject({ state: 'interrupted', error: 'Resource admission withdrawn' }) + expect(jobs.result(jobId)).toBeNull() + }, 10000) + + it('withholds publication when the final resource check fails', async () => { + let checks = 0 + expect(await executeNextBacktest(db, { ...options, admit: async () => { + if (++checks === 1) return true + throw new Error('Health collector unavailable') + } })).toEqual({ state: 'stopped', jobId }) + expect(checks).toBe(2) + expect(jobs.get(jobId)).toMatchObject({ state: 'interrupted', error: 'Resource admission withdrawn' }) + expect(jobs.result(jobId)).toBeNull() + }) +}) + +describe('lease, cancellation and worker protocol failures', () => { + /** A scripted worker standing in for the scoring bundle; the job is re-pinned to its hash. */ + async function scriptedWorker(name: string, body: string) { + const path = join(directory, name) + await writeFile(path, `let input='';process.stdin.on('data',c=>input+=c);process.stdin.on('end',()=>{const work=JSON.parse(input);const emit=e=>process.stdout.write(JSON.stringify(e)+'\\n');${body}});`) + sqlite.prepare('UPDATE biometric_backtest_jobs SET input = json_set(input, \'$.evaluatorBuildHash\', ?)').run(await workerBuildHash(path)) + return path + } + const progress = (completedNights: number, totalNights = 1) => `emit({type:'progress',completedNights:${completedNights},totalNights:${totalNights}});` + const identity = 'const report={version:"processed-backtest-report-v1",datasetHash:work.job.datasetHash,protocolHash:work.job.protocolHash,evaluatorBuildHash:work.job.evaluatorBuildHash,split:work.job.split,nights:[{}]};' + + it('interrupts a running replay when the host shuts down and leaves it for the next claimant', async () => { + const workerPath = await scriptedWorker('shutdown.cjs', 'setInterval(()=>{},1000)') + const controller = new AbortController() + setTimeout(() => controller.abort(), 300) + expect(await executeNextBacktest(db, { ...options, workerPath, signal: controller.signal })).toEqual({ state: 'stopped', jobId }) + expect(jobs.get(jobId)).toMatchObject({ state: 'interrupted', error: 'Worker host stopped' }) + expect(jobs.result(jobId)).toBeNull() + }) + + it('acknowledges a cancellation observed through a progress heartbeat', async () => { + const workerPath = await scriptedWorker('cancel.cjs', `setInterval(()=>{${progress(0)}},50)`) + setTimeout(() => jobs.cancel(jobId), 300) + expect(await executeNextBacktest(db, { ...options, workerPath })).toEqual({ state: 'stopped', jobId }) + expect(jobs.get(jobId)).toMatchObject({ state: 'cancelled', leaseOwner: null }) + expect(jobs.result(jobId)).toBeNull() + }) + + it('stops without touching job state once its lease is no longer authoritative', async () => { + const workerPath = await scriptedWorker('fenced.cjs', `setInterval(()=>{${progress(0)}},50)`) + setTimeout(() => sqlite.prepare('UPDATE biometric_backtest_jobs SET fencing_token = fencing_token + 1').run(), 300) + expect(await executeNextBacktest(db, { ...options, workerPath })).toEqual({ state: 'stopped', jobId }) + expect(jobs.get(jobId)).toMatchObject({ state: 'running', error: null }) + }) + + it.each([ + ['a different night count', progress(0, 5)], + ['regressing completion', progress(1) + progress(0)], + ['completion beyond the night count', progress(2)], + ])('fails a worker reporting %s', async (_, body) => { + const workerPath = await scriptedWorker('progress.cjs', `${body}setInterval(()=>{},1000)`) + expect(await executeNextBacktest(db, { ...options, workerPath })).toEqual({ state: 'stopped', jobId }) + expect(jobs.get(jobId)).toMatchObject({ state: 'failed', error: 'Replay failed validation, resources, or execution' }) + }) + + it.each([ + ['evaluator build', 'report.evaluatorBuildHash="0".repeat(64);'], + ['dataset', 'report.datasetHash="0".repeat(64);'], + ['protocol', 'report.protocolHash="0".repeat(64);'], + ['split', 'report.split="holdout";'], + ['night count', 'report.nights=[];'], + ])('rejects a report whose %s identity differs from the leased job', async (_, mutate) => { + const workerPath = await scriptedWorker('identity.cjs', `${identity}${mutate}${progress(1)}emit({type:'result',report});`) + expect(await executeNextBacktest(db, { ...options, workerPath })).toEqual({ state: 'stopped', jobId }) + expect(jobs.get(jobId)).toMatchObject({ state: 'failed' }) + expect(jobs.result(jobId)).toBeNull() + }) + + it('rejects a report published before every night was reported complete', async () => { + const workerPath = await scriptedWorker('incomplete.cjs', `${identity}emit({type:'result',report});`) + expect(await executeNextBacktest(db, { ...options, workerPath })).toEqual({ state: 'stopped', jobId }) + expect(jobs.get(jobId)).toMatchObject({ state: 'failed' }) + }) +}) diff --git a/src/biometrics/backtest/tests/splits.test.ts b/src/biometrics/backtest/tests/splits.test.ts new file mode 100644 index 000000000..0e430f95a --- /dev/null +++ b/src/biometrics/backtest/tests/splits.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from 'vitest' +import { assignWholeNightSplits } from '../splits' +import { describeCoverage } from '../coverage' +const session = (id: number, start: string, end: string) => ({ id, startMs: Date.parse(start), endMs: Date.parse(end) }) + +describe('frozen whole-night split assignment', () => { + it('keeps naps and overnight sessions in the same local-noon cohort across DST', () => { + const result = assignWholeNightSplits([ + session(1, '2026-11-01T00:00:00-07:00', '2026-11-01T07:00:00-08:00'), + session(2, '2026-10-31T14:00:00-07:00', '2026-10-31T15:00:00-07:00'), + session(3, '2026-11-02T00:00:00-08:00', '2026-11-02T08:00:00-08:00'), + session(4, '2026-11-03T00:00:00-08:00', '2026-11-03T08:00:00-08:00'), + ], 'America/Los_Angeles') + expect(result.assignments.map(group => [group.split, group.sessionIds])).toEqual([ + ['tuning', [2, 1]], ['validation', [3]], ['holdout', [4]], + ]) + expect(result.assignments[0].nightLabels).toEqual(['2026-10-31']) + expect(result.evidence).toBe('partitioned_not_validated') + }) + it('joins sessions bridging adjacent nights instead of leaking correlated data', () => { + const result = assignWholeNightSplits([ + session(1, '2026-10-01T23:00:00Z', '2026-10-02T13:00:00Z'), + session(2, '2026-10-02T12:00:00Z', '2026-10-02T15:00:00Z'), + session(3, '2026-10-02T23:00:00Z', '2026-10-03T08:00:00Z'), + ], 'UTC') + expect(result.assignments).toHaveLength(1) + expect(result.assignments[0].sessionIds).toEqual([1, 2, 3]) + expect(result.evidence).toBe('descriptive_only') + }) + it('uses all small datasets descriptively and rejects malformed or duplicate sessions', () => { + const rows = [session(1, '2026-10-01T23:00:00Z', '2026-10-02T08:00:00Z')] + expect(assignWholeNightSplits(rows, 'UTC').assignments[0].split).toBe('tuning') + expect(() => assignWholeNightSplits([...rows, ...rows], 'UTC')).toThrow('Duplicate') + expect(() => assignWholeNightSplits(rows, 'not-a-timezone')).toThrow() + expect(() => assignWholeNightSplits([{ ...rows[0], endMs: rows[0].startMs }], 'UTC')).toThrow() + }) +}) + +describe('coverage gaps', () => { + it('reports leading, internal and trailing gaps without substituting zeros', () => { + expect(describeCoverage([100, 100, 200, 700], 0, 1000, 100)).toMatchObject({ count: 3, duplicateTimestamps: 1, + firstMs: 100, lastMs: 700, gapDurationMs: 700, gaps: [{ startMs: 0, endMs: 100 }, { startMs: 300, endMs: 700 }, { startMs: 800, endMs: 1000 }] }) + expect(describeCoverage([], 0, 1000, 100)).toMatchObject({ firstMs: null, lastMs: null, gapDurationMs: 1000 }) + }) +}) diff --git a/src/biometrics/backtest/tests/stage-comparison.test.ts b/src/biometrics/backtest/tests/stage-comparison.test.ts new file mode 100644 index 000000000..f32f4fb8c --- /dev/null +++ b/src/biometrics/backtest/tests/stage-comparison.test.ts @@ -0,0 +1,40 @@ +// @vitest-environment node +import { expect, it } from 'vitest' +import { previewReferenceImport } from '../../references/normalize' +import { compareStageOutputs } from '../stage-comparison' +const references = previewReferenceImport({ side: 'left', subjectId: 'fixture', source: 'test', rows: [ + { recordKey: 'first', metric: 'sleep_stages', value: 'light', unit: 'stage', startMs: 1791673200000, endMs: 1791673210000 }, + { recordKey: 'second', metric: 'sleep_stages', value: 'deep', unit: 'stage', startMs: 1791673210000, endMs: 1791673220000 }, +] }).measurements +it('exposes apparent gains from dropping difficult intervals', () => { + const comparison = compareStageOutputs([{ start: 1791673200000, end: 1791673220000, stage: 'light' }], + [{ start: 1791673200000, end: 1791673210000, stage: 'light' }], references, 1791673200000, 1791673220000) + expect(comparison.ownSupport.baseline.agreement).toBe(0.5) + expect(comparison.ownSupport.candidate.agreement).toBe(1) + expect(comparison.differences).toMatchObject({ ownAgreement: 0.5, commonAgreement: 0, referenceCoverage: -0.5, outputCoverage: -0.5, pairedDurationMs: -10000 }) + expect(comparison.coverageReduced).toBe(true) + expect(comparison.commonSupport.baseline.pairedDurationMs).toBe(10000) + expect(comparison.commonSupport.candidate.pairedDurationMs).toBe(10000) +}) +it('measures genuine paired improvement without penalizing unchanged support', () => { + const comparison = compareStageOutputs([{ start: 1791673200000, end: 1791673220000, stage: 'light' }], + [{ start: 1791673200000, end: 1791673210000, stage: 'light' }, { start: 1791673210000, end: 1791673220000, stage: 'deep' }], references, 1791673200000, 1791673220000) + expect(comparison.differences).toMatchObject({ commonAgreement: 0.5, referenceCoverage: 0, outputCoverage: 0 }) + expect(comparison.coverageReduced).toBe(false) + expect(comparison.interpretation).toBe('descriptive_agreement_not_validated_improvement') +}) +it('keeps no common support unknown instead of reporting zero error', () => { + const comparison = compareStageOutputs([{ start: 1791673200000, end: 1791673210000, stage: 'light' }], + [{ start: 1791673210000, end: 1791673220000, stage: 'deep' }], references, 1791673200000, 1791673220000) + expect(comparison.commonSupport.baseline.pairedDurationMs).toBe(0) + expect(comparison.differences.commonAgreement).toBeNull() + expect(comparison.differences.commonMacroF1).toBeNull() +}) +it('excludes conflicting references identically on both sides and never fills gaps', () => { + const conflict = { ...references[0], recordKey: 'conflict', stage: 'rem' as const } + const comparison = compareStageOutputs([{ start: 1791673200000, end: 1791673220000, stage: 'light' }], + [{ start: 1791673200000, end: 1791673205000, stage: 'light' }, { start: 1791673215000, end: 1791673220000, stage: 'deep' }], [...references, conflict], 1791673200000, 1791673220000) + expect(comparison.commonSupport.baseline.pairedDurationMs).toBe(5000) + expect(comparison.commonSupport.candidate.pairedDurationMs).toBe(5000) + expect(comparison.differences.commonAgreement).toBe(1) +}) diff --git a/src/biometrics/backtest/tests/stage-objective.test.ts b/src/biometrics/backtest/tests/stage-objective.test.ts new file mode 100644 index 000000000..536f31ccc --- /dev/null +++ b/src/biometrics/backtest/tests/stage-objective.test.ts @@ -0,0 +1,135 @@ +import { stageObjectiveTestPolicy as policy } from './fixtures/stage-objective' +// @vitest-environment node +import { expect, it } from 'vitest' +import { assessStageObjective, type StageObjectiveEvidence } from '../stage-objective' +import { experimentProtocolSchema } from '../contracts' +import { configHash } from '../../algorithms/canonical' +import { finalizeStageObjectiveReport } from '../objective-finalization' +import { evaluateBacktest } from '../evaluator' +import { backtestFixture } from './fixtures/backtest' +const evidence = (): StageObjectiveEvidence => ({ split: 'holdout', holdoutUse: 1, intervalLower: 0.125, + operational: { wallMs: 1000, peakRssBytes: 1000, maximumIngestionAgeMs: 1000 }, nights: ['a', 'b', 'c'].map(cohortId => ({ + cohortId, pairedDurationMs: 800, agreementDifference: 0.125, macroF1Difference: 0.125, candidateOutputCoverage: 0.9, + baselineOutputMs: 1000, commonOutputMs: 900, + perStage: policy.requiredStages.map(stage => ({ stage, referenceMs: 200, f1Difference: 0.125 })), + })) }) +it('awards reference agreement improvement only with designated independent supported evidence at exact boundaries', () => { + expect(assessStageObjective(policy, evidence(), policy).status).toBe('improved') + expect(assessStageObjective(policy, evidence(), null).missingEvidence).toEqual(['policy_not_designated']) + for (const patch of [{ holdoutUse: 2 }, { split: 'validation' as const }, { intervalLower: null }, { operational: null }]) { + expect(assessStageObjective(policy, { ...evidence(), ...patch }, policy).status).toBe('insufficient_evidence') + } + expect(assessStageObjective(policy, { ...evidence(), intervalLower: 0.124 }, policy).status).toBe('no_material_change') +}) +it('does not hide coverage, macro-F1 or required-stage regressions behind improved overall agreement', () => { + const data = evidence() + data.nights[0].commonOutputMs = 899 + data.nights[0].candidateOutputCoverage = 0.89 + data.nights[0].macroF1Difference = -0.126 + data.nights[0].perStage[3].f1Difference = -0.126 + expect(assessStageObjective(policy, data, policy)).toMatchObject({ status: 'regressed', failedThresholds: ['output_coverage', 'coverage_loss', 'macro_f1_regression', 'stage_regression:rem'] }) +}) +it('requires supported stages on every night and bounds night-level regressions', () => { + const data = evidence() + data.nights[0].perStage[3].referenceMs-- + data.nights[0].pairedDurationMs-- + expect(assessStageObjective(policy, data, policy).missingEvidence).toContain('stage_support:rem') + const regressed = evidence() + regressed.nights[0].agreementDifference = -0.126 + regressed.nights[1].agreementDifference = -0.01 + expect(assessStageObjective(policy, regressed, policy).failedThresholds).toEqual(['night_regression', 'regressed_night_fraction']) +}) +it('rejects duplicate or inconsistent support and freezes stage thresholds only with stage replay', () => { + const data = evidence() + data.nights.push(data.nights[0]) + expect(() => assessStageObjective(policy, data, policy)).toThrow('evidence') + const invalid = evidence() + invalid.nights[0].commonOutputMs = 1001 + expect(() => assessStageObjective(policy, invalid, policy)).toThrow('evidence') + const protocol = { ...backtestFixture().job.protocol, stageObjectivePolicy: policy } + expect(experimentProtocolSchema.safeParse(protocol).success).toBe(true) + expect(experimentProtocolSchema.safeParse({ ...protocol, mode: 'comparison' }).success).toBe(false) + expect(experimentProtocolSchema.safeParse({ ...protocol, mode: 'signal_replay', waveformStages: false }).success).toBe(false) + expect(configHash(protocol)).not.toBe(configHash({ ...protocol, stageObjectivePolicy: { ...policy, minimumAgreementImprovement: 0.25 } })) +}) +it('scores stage evidence in retained replay and replaces claimed operational evidence at publication', () => { + const work = backtestFixture() + const protocol = { ...work.job.protocol, stageObjectivePolicy: policy } + const report = evaluateBacktest({ ...work, job: { ...work.job, protocol, protocolHash: configHash(protocol), + configs: [...work.job.configs, { ...work.job.configs[0], label: 'candidate' }] } }) + expect(report.stageObjectiveAssessments).toHaveLength(2) + expect(report.stageObjectiveAssessments[0]).toMatchObject({ status: 'insufficient_evidence', designated: false, + evidence: { nights: [{ cohortId: 'night', commonOutputMs: 600000 }], operational: null } }) + const claimed = { stageObjectiveAssessments: ['default', 'current'].map(baseline => ({ baseline, candidate: 'candidate', baselineHash: baseline, candidateHash: 'candidate', + ...assessStageObjective(policy, evidence(), policy) })) } + expect(finalizeStageObjectiveReport(claimed, policy, null, policy).stageObjectiveAssessments).toEqual(expect.arrayContaining([expect.objectContaining({ status: 'insufficient_evidence', missingEvidence: ['operational_evidence_unavailable'] })])) +}) + +it('retains specific missing evidence for sparse nights and unavailable metrics', () => { + const cases: [string, (data: StageObjectiveEvidence) => void][] = [ + ['independent_holdout_required', (data) => { data.holdoutUse = 2 }], + ['uncertainty_unavailable', (data) => { data.intervalLower = null }], + ['coverage_unavailable', (data) => { data.nights[0].candidateOutputCoverage = null }], + ['macro_f1_unavailable', (data) => { data.nights[0].macroF1Difference = null }], + ['stage_support:rem', (data) => { data.nights[0].perStage[3].f1Difference = null }], + ] + for (const [reason, change] of cases) { + const data = evidence() + change(data) + expect(assessStageObjective(policy, data, policy)).toMatchObject({ status: 'insufficient_evidence', missingEvidence: [reason], failedThresholds: [] }) + } + for (const change of [ + (data: StageObjectiveEvidence) => { data.nights[0].agreementDifference = null }, + (data: StageObjectiveEvidence) => { + data.nights[0].pairedDurationMs-- + data.nights[0].perStage[0].referenceMs-- + }, + ]) { + const data = evidence() + change(data) + expect(assessStageObjective(policy, data, policy).missingEvidence).toEqual(expect.arrayContaining(['insufficient_supported_nights', 'unsupported_nights'])) + } + const empty = evidence() + empty.nights = [] + expect(assessStageObjective(policy, empty, policy)).toMatchObject({ status: 'insufficient_evidence', equalNightMeanAgreementDifference: null, missingEvidence: ['insufficient_supported_nights'] }) +}) + +it('requires actual mean improvement and preserves inclusive regression limits', () => { + const data = evidence() + data.nights[0].agreementDifference = -0.125 + data.nights[1].agreementDifference = 0 + data.nights[2].agreementDifference = 0.5 + data.nights[0].macroF1Difference = -0.125 + data.nights[0].perStage[3].f1Difference = -0.125 + const exact = { ...policy, maximumRegressedNightFraction: 1 / 3 } + expect(assessStageObjective(exact, data, exact)).toMatchObject({ status: 'improved', failedThresholds: [], equalNightMeanAgreementDifference: 0.125 }) + data.nights[2].agreementDifference = 0.499 + expect(assessStageObjective(exact, data, exact).status).toBe('no_material_change') +}) + +it('rejects duplicated stages, inconsistent paired durations, and altered designation', () => { + const duplicate = evidence() + duplicate.nights[0].perStage[3].stage = 'wake' + expect(() => assessStageObjective(policy, duplicate, policy)).toThrow('Invalid stage objective evidence') + const unpaired = evidence() + unpaired.nights[0].commonOutputMs = 799 + expect(() => assessStageObjective(policy, unpaired, policy)).toThrow('Invalid stage objective evidence') + const mismatch = evidence() + mismatch.nights[0].perStage[0].referenceMs++ + expect(() => assessStageObjective(policy, mismatch, policy)).toThrow('Invalid stage objective evidence') + expect(() => assessStageObjective({ ...policy, requiredStages: ['wake', 'wake'] }, evidence(), policy)).toThrow('Duplicate required stages') + expect(assessStageObjective(policy, evidence(), { ...policy, qualificationId: 'other' }).missingEvidence).toEqual(['policy_not_designated']) +}) + +it('reports absent stage support and zero baseline coverage without inventing a score', () => { + const data = evidence() + data.nights[0].baselineOutputMs = 0 + data.nights[0].commonOutputMs = 0 + data.nights[0].pairedDurationMs = 0 + data.nights[0].agreementDifference = null + data.nights[0].macroF1Difference = null + data.nights[0].perStage = [] + expect(assessStageObjective(policy, data, policy)).toMatchObject({ status: 'insufficient_evidence', failedThresholds: [], + missingEvidence: ['insufficient_supported_nights', 'unsupported_nights', 'coverage_unavailable', 'macro_f1_unavailable', + 'stage_support:wake', 'stage_support:light', 'stage_support:deep', 'stage_support:rem'] }) +}) diff --git a/src/biometrics/backtest/tests/submission-build.test.ts b/src/biometrics/backtest/tests/submission-build.test.ts new file mode 100644 index 000000000..1c9a6970e --- /dev/null +++ b/src/biometrics/backtest/tests/submission-build.test.ts @@ -0,0 +1,17 @@ +// @vitest-environment node +import { beforeEach, expect, it, vi } from 'vitest' +import { packagedWorkerPath } from '../location' +import { waveformEvaluatorBuildIdentity } from '../waveform-build' +import { resolveInstalledReplayBuild } from '../submission' + +vi.mock('../location', () => ({ packagedWorkerPath: vi.fn(), packagedCoordinatorPath: vi.fn() })) +vi.mock('../waveform-build', () => ({ waveformEvaluatorBuildIdentity: vi.fn() })) +beforeEach(() => vi.resetAllMocks()) + +it('pins the coordinator bundle that ships beside the worker, matching execution', async () => { + vi.mocked(packagedWorkerPath).mockReturnValue('/standalone/biometrics-worker/worker.cjs') + const build = { hash: 'a'.repeat(64), python: { hash: 'b'.repeat(64) } } as Awaited> + vi.mocked(waveformEvaluatorBuildIdentity).mockResolvedValue(build) + await expect(resolveInstalledReplayBuild('signal_replay')).resolves.toEqual({ evaluatorBuildHash: build.hash, pythonBuildHash: build.python.hash }) + expect(vi.mocked(waveformEvaluatorBuildIdentity).mock.calls[0]?.slice(0, 2)).toEqual(['/standalone/biometrics-worker/coordinator.cjs', '/standalone/biometrics-worker/worker.cjs']) +}) diff --git a/src/biometrics/backtest/tests/submission.test.ts b/src/biometrics/backtest/tests/submission.test.ts new file mode 100644 index 000000000..a278807e0 --- /dev/null +++ b/src/biometrics/backtest/tests/submission.test.ts @@ -0,0 +1,127 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import * as configSchema from '@/src/db/schema' +import * as bioSchema from '@/src/db/biometrics-schema' +import { configHash } from '../../algorithms/canonical' +import { defaultConfig, manifestHash } from '../../algorithms/registry' +import { ProfileStore } from '../../profiles/store' +import { backtestJobSnapshotSchema } from '../contracts' +import { DatasetStore } from '../datasets' +import { BacktestJobStore } from '../jobs' +import { resolveInstalledReplayBuild, resolveSubmissionBuild, retryBacktestSubmission, submitBacktest } from '../submission' +import { freezeWaveformDataset } from './fixtures/waveform' + +const mocks = vi.hoisted(() => ({ + workerPath: vi.fn<() => string>(), coordinatorPath: vi.fn<() => string>(), + workerBuildHash: vi.fn<(path: string) => Promise>(), waveformBuild: vi.fn<() => Promise<{ hash: string, python: { hash: string } }>>(), +})) +vi.mock('../location', () => ({ packagedWorkerPath: mocks.workerPath, packagedCoordinatorPath: mocks.coordinatorPath })) +vi.mock('../child', () => ({ workerBuildHash: mocks.workerBuildHash })) +vi.mock('../waveform-build', () => ({ waveformEvaluatorBuildIdentity: mocks.waveformBuild })) + +let configRaw: Database.Database, bioRaw: Database.Database +let config: ReturnType>, bio: ReturnType> +const now = 1791673200000 +const protocol = { version: 'descriptive-evaluation-v1' as const, mode: 'stage_replay' as const, assumedStageCalibrationQuality: 0, pointToleranceMs: 30000, clockOffsetMs: 0 } +beforeEach(() => { + vi.clearAllMocks() + mocks.workerPath.mockReturnValue('/packaged/worker.cjs') + mocks.coordinatorPath.mockReturnValue('/packaged/coordinator.cjs') + mocks.workerBuildHash.mockResolvedValue('e'.repeat(64)) + mocks.waveformBuild.mockResolvedValue({ hash: 'd'.repeat(64), python: { hash: 'b'.repeat(64) } }) + configRaw = new Database(':memory:') + bioRaw = new Database(':memory:') + config = drizzle(configRaw, { schema: configSchema }) + bio = drizzle(bioRaw, { schema: bioSchema }) + migrate(config, { migrationsFolder: 'src/db/migrations' }) + migrate(bio, { migrationsFolder: 'src/db/biometrics-migrations' }) + for (let id = 1; id <= 3; id++) bio.insert(bioSchema.sleepRecords).values({ id, side: 'left', enteredBedAt: new Date(now + id * 86400000), + leftBedAt: new Date(now + id * 86400000 + 3600000), sleepDurationSeconds: 3600, timesExitedBed: 0 }).run() +}) +afterEach(() => { + configRaw.close() + bioRaw.close() +}) + +function fixture(mode: 'stage_replay' | 'signal_replay' = 'stage_replay') { + const revision = new ProfileStore(config).createRevision({ side: 'left', origin: 'manual', requester: 'fixture', idempotencyKey: 'candidate', + overrides: [{ path: 'heart_rate.trackingToleranceBpm', value: 20 }] }) + const dataset = new DatasetStore(bio).create({ side: 'left', subjectId: 'fixture', sessionIds: [1, 2, 3], referenceImportIds: [], timezone: 'UTC', idempotencyKey: 'dataset' }) + if (mode === 'signal_replay') freezeWaveformDataset(bio, dataset.id) + const experiment = new BacktestJobStore(bio).createExperiment(dataset.id, { ...protocol, mode }) + const request = { experimentId: experiment.id, candidateRevisionId: revision.id, expectedBindingGeneration: 0, split: 'tuning' as const, idempotencyKey: 'submission' } + return { revision, dataset, experiment, request } +} + +describe('backtest submission', () => { + it('freezes default, current and candidate snapshots and retries idempotently', () => { + const { revision, request } = fixture() + expect(retryBacktestSubmission(bio, request)).toBeNull() + const job = submitBacktest(config, bio, 'f'.repeat(64), request) + const snapshot = backtestJobSnapshotSchema.parse(job.input) + expect(snapshot.configs.map(row => [row.label, row.revisionId, row.resolvedHash])).toEqual([ + ['default', null, configHash(defaultConfig)], ['current', null, configHash(defaultConfig)], ['candidate', revision.id, revision.resolvedHash]]) + expect(snapshot.waveform).toBeUndefined() + expect(snapshot.submissionHash).toBe(configHash(request)) + expect(retryBacktestSubmission(bio, request)?.id).toBe(job.id) + expect(submitBacktest(config, bio, 'f'.repeat(64), request).id).toBe(job.id) + expect(() => submitBacktest(config, bio, 'f'.repeat(64), { ...request, split: 'validation' })).toThrow('different submission') + expect(() => retryBacktestSubmission(bio, { ...request, split: 'validation' })).toThrow('different submission') + }) + it('rejects unknown experiments, stale bindings and waveform submissions without a Python identity', () => { + const { request } = fixture() + expect(() => submitBacktest(config, bio, 'f'.repeat(64), { ...request, experimentId: '00000000-0000-4000-8000-000000000000' })).toThrow('Experiment not found') + expect(() => submitBacktest(config, bio, 'f'.repeat(64), { ...request, expectedBindingGeneration: 1 })).toThrow('binding changed') + expect(bio.select().from(bioSchema.biometricBacktestJobs).all()).toEqual([]) + const signal = fixture('signal_replay') + expect(() => submitBacktest(config, bio, 'f'.repeat(64), { ...signal.request, idempotencyKey: 'signal' })).toThrow('requires the Python build identity') + }) + it('pins both sides of a waveform submission from the acknowledged baseline', () => { + const { request, revision } = fixture('signal_replay') + const job = submitBacktest(config, bio, 'f'.repeat(64), request, 'b'.repeat(64)) + const snapshot = backtestJobSnapshotSchema.parse(job.input) + expect(snapshot.waveform).toMatchObject({ pythonBuildHash: 'b'.repeat(64), + otherSide: { manifestHash, resolvedHash: configHash(defaultConfig) }, + baselineProvenance: { current: { source: 'unacknowledged_defaults' }, otherSide: { source: 'unacknowledged_defaults' } } }) + expect(snapshot.configs[2]).toMatchObject({ label: 'candidate', revisionId: revision.id }) + }) +}) + +describe('installed replay build resolution', () => { + it('returns an existing submission before probing any build identity', async () => { + const { request } = fixture() + const job = submitBacktest(config, bio, 'f'.repeat(64), request) + expect(await resolveSubmissionBuild(bio, request)).toEqual({ existing: job }) + expect(mocks.workerBuildHash).not.toHaveBeenCalled() + expect(await resolveSubmissionBuild(bio, { ...request, idempotencyKey: 'fresh' })).toEqual({ evaluatorBuildHash: 'e'.repeat(64) }) + expect(mocks.workerBuildHash).toHaveBeenCalledWith('/packaged/worker.cjs') + }) + it('coalesces concurrent waveform build probes and never caches a finished identity', async () => { + const [first, second] = await Promise.all([resolveInstalledReplayBuild('signal_replay'), resolveInstalledReplayBuild('signal_replay')]) + expect(first).toEqual({ evaluatorBuildHash: 'd'.repeat(64), pythonBuildHash: 'b'.repeat(64) }) + expect(second).toEqual(first) + expect(mocks.waveformBuild).toHaveBeenCalledTimes(1) + expect(mocks.waveformBuild).toHaveBeenCalledWith('/packaged/coordinator.cjs', '/packaged/worker.cjs', expect.objectContaining({ pythonPath: expect.any(String), modulesPath: expect.any(String) })) + await resolveInstalledReplayBuild('signal_replay') + expect(mocks.waveformBuild).toHaveBeenCalledTimes(2) + mocks.coordinatorPath.mockReturnValue('/elsewhere/coordinator.cjs') + const pending = resolveInstalledReplayBuild('signal_replay') + const other = resolveInstalledReplayBuild('signal_replay') + await Promise.all([pending, other]) + expect(mocks.waveformBuild).toHaveBeenCalledTimes(3) + }) + it('reports unreadable packaged builds as worker unavailability without leaking paths', async () => { + mocks.workerPath.mockImplementation(() => { + throw new Error('/secret/path missing') + }) + await expect(resolveInstalledReplayBuild('comparison')).rejects.toThrow('Packaged replay build identity cannot be read') + mocks.workerPath.mockReturnValue('/packaged/worker.cjs') + mocks.waveformBuild.mockRejectedValue(new Error('python exploded')) + await expect(resolveInstalledReplayBuild('signal_replay')).rejects.toThrow('Packaged replay build identity cannot be read') + mocks.workerBuildHash.mockRejectedValue(new Error('ENOENT')) + await expect(resolveInstalledReplayBuild('stage_replay')).rejects.toThrow('Packaged replay build identity cannot be read') + }) +}) diff --git a/src/biometrics/backtest/tests/traces.test.ts b/src/biometrics/backtest/tests/traces.test.ts new file mode 100644 index 000000000..2225a445b --- /dev/null +++ b/src/biometrics/backtest/tests/traces.test.ts @@ -0,0 +1,108 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import * as schema from '@/src/db/biometrics-schema' +import { configHash } from '../../algorithms/canonical' +import { defaultConfig, manifestHash } from '../../algorithms/registry' +import { datasetManifestSchema } from '../contracts' +import { DatasetStore } from '../datasets' +import { BacktestJobStore } from '../jobs' +import { pageStageTraces, readNumericTraces, readReferenceTraces, readStageTraces } from '../traces' +const request = { id: '11111111-1111-4111-8111-111111111111', expectedContentHash: 'a'.repeat(64), configuration: 'candidate' as const, limit: 2 } +const report = { version: 'processed-backtest-report-v1', protocol: { mode: 'stage_replay', clockOffsetMs: 60000 }, nights: ['one', 'two'].map((cohortId, index) => ({ + cohortId, evaluations: [{ label: 'candidate', configHash: 'b'.repeat(64), revisionId: null, windows: [{ startMs: index * 300, endMs: index * 300 + 300, + blocks: [0, 1, 2].map(i => ({ start: index * 300 + i * 100, end: index * 300 + (i + 1) * 100, stage: 'light' })) }] }], +})) } +it('pages every retained interval exactly once with a stable hash cursor and no timestamp offset', () => { + const before = JSON.stringify(report) + const first = pageStageTraces(report, request) + const second = pageStageTraces(report, { ...request, offset: first.nextCursor?.offset }) + const third = pageStageTraces(report, { ...request, offset: second.nextCursor?.offset }) + expect(first).toMatchObject({ totalItems: 6, downsampled: false, clockOffsetMs: 60000, nextCursor: { offset: 2, contentHash: request.expectedContentHash } }) + expect([...first.items, ...second.items, ...third.items].map(row => row.startMs)).toEqual([0, 100, 200, 300, 400, 500]) + expect(third.nextCursor).toBeNull() + expect(pageStageTraces(report, { ...request, offset: 6 }).items).toEqual([]) + expect(JSON.stringify(report)).toBe(before) +}) +it('filters the cohort before counting and rejects out-of-range or missing selections', () => { + expect(pageStageTraces(report, { ...request, cohortId: 'two' })).toMatchObject({ totalItems: 3, items: [{ cohortId: 'two' }, { cohortId: 'two' }] }) + expect(() => pageStageTraces(report, { ...request, cohortId: 'missing' })).toThrow('not evaluated') + expect(() => pageStageTraces(report, { ...request, configuration: 'default' })).toThrow('not evaluated') + expect(() => pageStageTraces(report, { ...request, offset: 7 })).toThrow('cursor') + expect(() => pageStageTraces(report, { ...request, limit: 501 })).toThrow() + expect(() => pageStageTraces({ ...report, protocol: { ...report.protocol, mode: 'comparison' } }, request)).toThrow('did not produce') +}) +it('serves waveform stage traces only when the protocol replayed stages and rejects unreadable reports', () => { + expect(pageStageTraces({ ...report, version: 'waveform-backtest-report-v1', protocol: { ...report.protocol, mode: 'signal_replay', waveformStages: true } }, request)).toMatchObject({ totalItems: 6 }) + expect(() => pageStageTraces({ ...report, protocol: { ...report.protocol, mode: 'signal_replay', waveformStages: false } }, request)).toThrow('did not produce') + expect(() => pageStageTraces({ ...report, protocol: { ...report.protocol, mode: 'signal_replay' } }, request)).toThrow('did not produce') + expect(() => pageStageTraces({ ...report, nights: null }, request)).toThrow('format is unavailable') +}) + +const start = 1791673200000 +const numericReport = () => { + const hash = 'b'.repeat(64) + const reading = { side: 'left', sourceSide: 'left', configHash: hash, timestampMs: 1000, heartRate: 60, breathingRate: null, hrvIndex: null, + hrvMetric: 'window_ibi_variability_v1', qualityScore: 0, flags: null, limitations: [], hrRaw: 65 } + return { version: 'waveform-backtest-report-v1', side: 'left', protocol: { mode: 'signal_replay', clockOffsetMs: 0 }, + nights: [{ cohortId: 'one', evaluations: [{ label: 'candidate', configHash: hash, revisionId: null, windows: [{ startMs: 1000, endMs: 5000 }] }] }], + waveform: { outputs: [{ cohortId: 'one', configHash: hash, readings: [reading] }] } } +} +describe('durable trace reads', () => { + let sqlite: Database.Database + let db: ReturnType> + let jobs: BacktestJobStore + let dataset: ReturnType + beforeEach(() => { + sqlite = new Database(':memory:') + db = drizzle(sqlite, { schema }) + migrate(db, { migrationsFolder: 'src/db/biometrics-migrations' }) + db.insert(schema.sleepRecords).values({ id: 1, side: 'left', enteredBedAt: new Date(start), leftBedAt: new Date(start + 600000), sleepDurationSeconds: 600, timesExitedBed: 0 }).run() + const datasets = new DatasetStore(db) + dataset = datasets.read(datasets.create({ side: 'left', subjectId: 'A', sessionIds: [1], referenceImportIds: [], timezone: 'UTC', idempotencyKey: 'traces' }).id) + jobs = new BacktestJobStore(db) + }) + afterEach(() => sqlite.close()) + const config = { label: 'default' as const, revisionId: null, resolved: defaultConfig, resolvedHash: configHash(defaultConfig), manifestHash } + const enqueue = (idempotencyKey: string, experimentId = jobs.createExperiment(dataset.id, { version: 'descriptive-evaluation-v1', mode: 'stage_replay', assumedStageCalibrationQuality: 0, pointToleranceMs: 30000, clockOffsetMs: 0 }, idempotencyKey).id) => + jobs.enqueue({ experimentId, evaluatorBuildHash: 'f'.repeat(64), idempotencyKey, split: 'tuning', configs: [config] }) + const publish = (idempotencyKey: string, mode: 'stage_replay' | 'signal_replay', published: object) => { + const job = enqueue(idempotencyKey) + const claimed = jobs.claim('reader') + if (!claimed) throw new Error('Missing claim') + const result = jobs.complete({ id: job.id, owner: 'reader', token: claimed.fencingToken }, { ...published, protocol: { mode, clockOffsetMs: 0, pointToleranceMs: 30000 } }) + return { id: job.id, expectedContentHash: result.contentHash } + } + it('reads retained stage traces only from a verified, unreleased successful result', () => { + const published = publish('stage', 'stage_replay', report) + expect(readStageTraces(db, { ...published, configuration: 'candidate', limit: 2 })).toMatchObject({ totalItems: 6, clockOffsetMs: 0, items: [{ cohortId: 'one' }, { cohortId: 'one' }] }) + expect(() => readStageTraces(db, { ...published, expectedContentHash: '0'.repeat(64), configuration: 'candidate' })).toThrow('hash changed') + const queued = enqueue('queued', jobs.get(published.id).experimentId) + expect(() => readStageTraces(db, { ...published, id: queued.id, configuration: 'candidate' })).toThrow('successful replay') + jobs.releaseResult(published.id, published.expectedContentHash) + expect(() => readStageTraces(db, { ...published, configuration: 'candidate' })).toThrow('released') + }) + it('reads retained numeric traces with the same lifecycle checks', () => { + const published = publish('numeric', 'signal_replay', numericReport()) + expect(readNumericTraces(db, { ...published, configuration: 'candidate' })).toMatchObject({ totalItems: 1, side: 'left', items: [{ timestampMs: 1000 }] }) + expect(() => readNumericTraces(db, { ...published, expectedContentHash: '0'.repeat(64), configuration: 'candidate' })).toThrow('hash changed') + const queued = enqueue('queued', jobs.get(published.id).experimentId) + expect(() => readNumericTraces(db, { ...published, id: queued.id, configuration: 'candidate' })).toThrow('successful replay') + jobs.releaseResult(published.id, published.expectedContentHash) + expect(() => readNumericTraces(db, { ...published, configuration: 'candidate' })).toThrow('released') + }) + it('reads frozen reference traces for the retained dataset the report identifies', () => { + const cohortId = datasetManifestSchema.parse(dataset.manifest).splitProtocol.assignments[0].cohortId + const identity = { version: 'processed-backtest-report-v1', datasetId: dataset.id, datasetHash: dataset.contentHash, + nights: [{ cohortId, evaluations: [{ label: 'default', configHash: 'b'.repeat(64), revisionId: null, windows: [{ startMs: start, endMs: start + 600000, blocks: [] }] }] }] } + const published = publish('reference', 'stage_replay', identity) + expect(readReferenceTraces(db, { ...published, cohortId, metric: 'heart_rate' })).toMatchObject({ totalItems: 0, side: 'left', datasetHash: dataset.contentHash, provenance: 'frozen_dataset_reference' }) + expect(() => readReferenceTraces(db, { ...published, expectedContentHash: '0'.repeat(64), cohortId, metric: 'heart_rate' })).toThrow('hash changed') + const queued = enqueue('queued', jobs.get(published.id).experimentId) + expect(() => readReferenceTraces(db, { ...published, id: queued.id, cohortId, metric: 'heart_rate' })).toThrow('successful replay') + jobs.releaseResult(published.id, published.expectedContentHash) + expect(() => readReferenceTraces(db, { ...published, cohortId, metric: 'heart_rate' })).toThrow('released') + }) +}) diff --git a/src/biometrics/backtest/tests/uncertainty-core.test.ts b/src/biometrics/backtest/tests/uncertainty-core.test.ts new file mode 100644 index 000000000..39e13fb40 --- /dev/null +++ b/src/biometrics/backtest/tests/uncertainty-core.test.ts @@ -0,0 +1,29 @@ +// @vitest-environment node +import { createHash } from 'node:crypto' +import { expect, it } from 'vitest' +import { createNightUncertainty } from '../uncertainty-core' + +const digest = (text: string) => new Uint8Array(createHash('sha256').update(text).digest()) +const policy = { version: 'paired-night-percentile-v1' as const, seed: 'fixture', resamples: 200, minimumNights: 3 } +const night = (cohortId: string, difference: number | null) => ({ cohortId, difference }) + +it('orders nights by cohort before resampling so input order never changes the interval', () => { + const uncertainty = createNightUncertainty(digest) + const nights = [night('c', 0.3), night('a', -0.1), night('b', 0.2), night('d', null)] + const result = uncertainty(nights, policy) + expect(result).toMatchObject({ status: 'estimated', scoredNights: 3, omittedNights: 1, statistic: 'equal_night_mean_common_agreement_difference' }) + expect(result.estimate).toBeCloseTo(0.4 / 3) + expect(uncertainty([...nights].reverse(), policy)).toEqual(result) + expect(uncertainty(nights)).toMatchObject({ status: 'not_requested', policy: null, interval: null }) +}) + +it('rejects oversized, duplicated and out-of-range paired differences per statistic', () => { + const agreement = createNightUncertainty(digest) + expect(() => agreement(Array.from({ length: 61 }, (_, i) => night(String(i), 0)), policy)).toThrow('Invalid paired-night uncertainty input') + expect(() => agreement([night('a', 0), night('a', 0)], policy)).toThrow('Invalid paired-night uncertainty input') + expect(() => agreement([night('a', 1.5)], policy)).toThrow('Invalid paired-night uncertainty input') + expect(() => agreement([night('a', Number.NaN)], policy)).toThrow('Invalid paired-night uncertainty input') + const mae = createNightUncertainty(digest, 'equal_night_mean_common_mae_difference') + expect(mae([night('a', 1.5), night('b', 2.5), night('c', 40)], policy)).toMatchObject({ status: 'estimated', statistic: 'equal_night_mean_common_mae_difference' }) + expect(() => mae([night('a', Number.POSITIVE_INFINITY)], policy)).toThrow('Invalid paired-night uncertainty input') +}) diff --git a/src/biometrics/backtest/tests/uncertainty.test.ts b/src/biometrics/backtest/tests/uncertainty.test.ts new file mode 100644 index 000000000..a617374e0 --- /dev/null +++ b/src/biometrics/backtest/tests/uncertainty.test.ts @@ -0,0 +1,42 @@ +// @vitest-environment node +import { expect, it } from 'vitest' +import { nightUncertainty, type NightBootstrapPolicy } from '../uncertainty' +import { experimentProtocolSchema } from '../contracts' +import { configHash } from '../../algorithms/canonical' +const policy: NightBootstrapPolicy = { version: 'paired-night-percentile-v1', seed: 'fixture', resamples: 200, minimumNights: 3 } +const nights = [{ cohortId: 'a', difference: -0.5 }, { cohortId: 'b', difference: 0.2 }, { cohortId: 'c', difference: 0.8 }] +it('resamples paired nights reproducibly irrespective of input ordering', () => { + const result = nightUncertainty(nights, policy) + expect(result.status).toBe('estimated') + expect(result).toEqual(nightUncertainty([...nights].reverse(), policy)) + expect(result.interval?.lower).toBeLessThan(result.estimate ?? 0) + expect(result.interval?.upper).toBeGreaterThan(result.estimate ?? 0) + expect(result).toMatchObject({ scoredNights: 3, omittedNights: 0, samplingUnit: 'whole_night_cohort' }) +}) +it('does not fabricate precision from missing or identical nights', () => { + expect(nightUncertainty(nights)).toMatchObject({ status: 'not_requested', interval: null }) + expect(nightUncertainty([...nights.slice(0, 2), { cohortId: 'missing', difference: null }], policy)) + .toMatchObject({ status: 'insufficient_nights', scoredNights: 2, omittedNights: 1, interval: null }) + expect(nightUncertainty(nights.map(row => ({ ...row, difference: 0.5 })), policy)).toMatchObject({ status: 'degenerate', interval: null }) +}) +it('bounds computation and rejects invalid or duplicated evidence', () => { + expect(() => nightUncertainty([...nights, nights[0]], policy)).toThrow('Invalid paired-night') + expect(() => nightUncertainty([{ cohortId: 'bad', difference: NaN }], policy)).toThrow('Invalid paired-night') + expect(() => nightUncertainty(nights, { ...policy, resamples: 5001 })).toThrow() + expect(() => nightUncertainty(nights, { ...policy, minimumNights: 2 })).toThrow() +}) +it('pins uncertainty parameters in the experiment protocol hash', () => { + const base = { version: 'descriptive-evaluation-v1', mode: 'stage_replay', assumedStageCalibrationQuality: 0, pointToleranceMs: 30000, clockOffsetMs: 0 } + const first = experimentProtocolSchema.parse({ ...base, nightUncertainty: policy }) + const second = experimentProtocolSchema.parse({ ...base, nightUncertainty: { ...policy, seed: 'different' } }) + expect(configHash(first)).not.toBe(configHash(second)) + expect(experimentProtocolSchema.parse(base)).toEqual(base) +}) + +it('matches a separately computed Python SHA-256/quantile golden', () => { + const fixture = Array.from({ length: 12 }, (_, i) => ({ cohortId: String(i).padStart(2, '0'), difference: ((i * 7) % 11 - 5) / 10 })) + const result = nightUncertainty(fixture, policy) + expect(result.estimate).toBeCloseTo(-0.041666666666666664, 14) + expect(result.interval?.lower).toBeCloseTo(-0.22520833333333334, 14) + expect(result.interval?.upper).toBeCloseTo(0.125, 14) +}) diff --git a/src/biometrics/backtest/tests/waveform-build.test.ts b/src/biometrics/backtest/tests/waveform-build.test.ts new file mode 100644 index 000000000..87a780718 --- /dev/null +++ b/src/biometrics/backtest/tests/waveform-build.test.ts @@ -0,0 +1,39 @@ +// @vitest-environment node +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' +import { afterEach, beforeEach, expect, it } from 'vitest' +import { probeWaveformBuild, waveformEvaluatorBuildIdentity } from '../waveform-build' +let directory: string +const location = { pythonPath: resolve('modules/piezo-processor/.venv/bin/python'), modulesPath: resolve('modules') } +beforeEach(async () => { + directory = await mkdtemp(join(tmpdir(), 'waveform-build-')) +}) +afterEach(() => rm(directory, { recursive: true, force: true })) +it('verifies a real Python identity and binds both shipped Node bundles', async () => { + const coordinator = join(directory, 'coordinator.cjs'), worker = join(directory, 'worker.cjs') + await writeFile(coordinator, 'orchestration-v1') + await writeFile(worker, 'scoring-v1') + const first = await waveformEvaluatorBuildIdentity(coordinator, worker, location) + expect(first.python).toEqual(await probeWaveformBuild(location)) + expect(first.python.identity.sources).toHaveProperty('common/biometrics/replay_worker.py') + expect(first).toEqual(await waveformEvaluatorBuildIdentity(coordinator, worker, location)) + await writeFile(coordinator, 'orchestration-v2') + const second = await waveformEvaluatorBuildIdentity(coordinator, worker, location) + expect(second.hash).not.toBe(first.hash) + expect(second.python.hash).toBe(first.python.hash) + await writeFile(worker, 'scoring-v2') + expect((await waveformEvaluatorBuildIdentity(coordinator, worker, location)).hash).not.toBe(second.hash) +}) +it('rejects missing interpreters, invalid output and forged build hashes', async () => { + await expect(probeWaveformBuild({ ...location, pythonPath: join(directory, 'missing') })).rejects.toThrow('unavailable') + const fixture = join(directory, 'fixture') + await writeFile(fixture, `#!${process.execPath}\nprocess.stdout.write('{}');`, { mode: 0o700 }) + await expect(probeWaveformBuild({ ...location, pythonPath: fixture })).rejects.toThrow() + const build = await probeWaveformBuild(location) + await writeFile(fixture, `#!${process.execPath}\nprocess.stdout.write(${JSON.stringify(JSON.stringify({ ...build, hash: '0'.repeat(64) }))});`, { mode: 0o700 }) + await expect(probeWaveformBuild({ ...location, pythonPath: fixture })).rejects.toThrow('integrity mismatch') + const controller = new AbortController() + controller.abort() + await expect(probeWaveformBuild(location, controller.signal)).rejects.toThrow('unavailable') +}) diff --git a/src/biometrics/backtest/tests/waveform-child.test.ts b/src/biometrics/backtest/tests/waveform-child.test.ts new file mode 100644 index 000000000..184e3a4d0 --- /dev/null +++ b/src/biometrics/backtest/tests/waveform-child.test.ts @@ -0,0 +1,161 @@ +// @vitest-environment node +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import { runWaveformChild } from '../waveform-child' + +let directory: string +let executable: string +const hash = 'a'.repeat(64) +const reading = { side: 'left', timestampMs: 1000, heartRate: 60, hrvIndex: null, hrvMetric: 'window_ibi_variability_v1', + breathingRate: 12, qualityScore: 0.8, flags: null, hrRaw: 60, sourceSide: 'left', configHash: hash, limitations: [] } +const result = { version: 'retained-waveform-run-v1', build_hash: hash, decoded_records: 1, raw_records: 1, + counts: { records: 1, accepted_frames: 1, empty_channel_frames: 0, pump_gated_frames: 0, preroll_outputs: 0, scored_outputs: 1 }, + limitations: [], config_hashes: { left: hash, right: hash }, artifact_hashes: [hash] } +const events = [{ type: 'reading', reading }, { type: 'result', result }] +const location = () => ({ pythonPath: executable, modulesPath: directory, retainedDirectory: directory }) +beforeEach(async () => { + directory = await mkdtemp(join(tmpdir(), 'waveform-child-')) + executable = join(directory, 'fixture-worker') +}) +afterEach(() => rm(directory, { recursive: true, force: true })) +async function worker(body: string) { + await writeFile(executable, `#!${process.execPath}\nsetTimeout(() => process.exit(91), 3000).unref(); process.stdin.resume(); process.stdin.on('end', () => { ${body} });\n`, { mode: 0o700 }) +} +const emit = (value: unknown) => `process.stdout.write(${JSON.stringify(JSON.stringify(value) + '\n')});` + +it('passes a trusted CPU limit to the fixed worker and rejects invalid limits before spawning', async () => { + await worker(`if (process.argv.slice(-2).join(' ') !== '--cpu-seconds 1800') process.exit(1); ${events.map(emit).join('')}`) + expect(await runWaveformChild(location(), {}, { cpuSeconds: 1800 })).toEqual({ readings: [reading], run: result }) + for (const cpuSeconds of [0, -1, 1.5, NaN, Infinity, 3601]) { + await expect(runWaveformChild(location(), {}, { cpuSeconds })).rejects.toThrow('Invalid waveform CPU budget') + } +}) + +it('returns readings only after a valid success event and zero exit with a clean environment', async () => { + await worker(`if (process.env.WAVEFORM_TEST_SECRET + || JSON.stringify(process.argv.slice(2)) !== ${JSON.stringify(JSON.stringify(['-B', '-m', 'common.biometrics.replay_worker', '--retained-directory', directory]))} + || process.env.PYTHONPATH !== ${JSON.stringify(directory)} || process.env.NODE_ENV !== 'production' + || process.env.LANG !== 'C.UTF-8' || process.env.TZ !== 'UTC' + || ['OPENBLAS_NUM_THREADS','OMP_NUM_THREADS','MKL_NUM_THREADS','NUMEXPR_NUM_THREADS'].some(key => process.env[key] !== '1')) process.exit(1); + ${events.map(emit).join('')}`) + process.env.WAVEFORM_TEST_SECRET = 'must not reach worker' + try { + expect(await runWaveformChild(location(), {})).toEqual({ readings: [reading], run: result }) + } + finally { + delete process.env.WAVEFORM_TEST_SECRET + } +}) + +it.each([ + [events[0]], + [...events, events[1]], + [...events, events[0]], + [events[0], { type: 'result', result: { ...result, counts: { ...result.counts, scored_outputs: 2 } } }], + [{ type: 'reading', reading: { ...reading, configHash: 'b'.repeat(64) } }, events[1]], + [{ type: 'reading', reading: { ...reading, hrvMetric: 'rmssd' } }, events[1]], + [events[0], { type: 'result', result: { ...result, decoded_records: 2 } }], + [events[0], { type: 'result', result: { ...result, counts: { ...result.counts, records: 2 } } }], + [events[0], { type: 'reading', reading: { ...reading, configHash: 'b'.repeat(64) } }, + { type: 'result', result: { ...result, counts: { ...result.counts, scored_outputs: 2 } } }], +].map(output => ({ output })))('rejects incomplete, inconsistent, or out-of-order protocol output %#', async ({ output }) => { + await worker(output.map(emit).join('')) + await expect(runWaveformChild(location(), {})).rejects.toThrow(/invalid event|incomplete result/) +}) + +it('rejects trailing partial data and nonzero exits even after a success event', async () => { + await worker(`${events.map(emit).join('')} process.stdout.write('partial');`) + await expect(runWaveformChild(location(), {})).rejects.toThrow('incomplete result') + await worker(`${events.map(emit).join('')} process.stdout.write(Buffer.from([0xc3]));`) + await expect(runWaveformChild(location(), {})).rejects.toThrow('incomplete result') + await worker(`${events.map(emit).join('')} process.exitCode = 1;`) + await expect(runWaveformChild(location(), {})).rejects.toThrow('incomplete result') +}) + +it('bounds individual output events, diagnostics and serialized input', async () => { + await worker('process.stdout.write(\'x\'.repeat(70000));') + await expect(runWaveformChild(location(), {})).rejects.toThrow('event exceeds') + await worker('process.stderr.write(\'x\'.repeat(17000));') + await expect(runWaveformChild(location(), {})).rejects.toThrow('diagnostic output') + await expect(runWaveformChild(location(), 'x'.repeat(8 * 1024 * 1024))).rejects.toThrow('input exceeds') +}) + +it('kills overdue and cancelled work and handles missing executables', async () => { + await worker('setInterval(() => {}, 1000);') + await expect(runWaveformChild(location(), {}, { timeoutMs: 30 })).rejects.toThrow('time budget') + const controller = new AbortController() + const pending = runWaveformChild(location(), {}, { signal: controller.signal }) + controller.abort() + await expect(pending).rejects.toThrow('cancelled') + await expect(runWaveformChild(location(), {}, { signal: controller.signal })).rejects.toThrow('cancelled') + await expect(runWaveformChild({ ...location(), pythonPath: join(directory, 'missing') }, {})).rejects.toThrow('could not start') +}) + +it('escalates cancellation when a running worker ignores SIGTERM', async () => { + const ready = join(directory, 'ready') + await worker(`process.on('SIGTERM', () => {}); require('node:fs').writeFileSync(${JSON.stringify(ready)}, String(process.pid)); setInterval(() => {}, 1000);`) + const controller = new AbortController() + const pending = runWaveformChild(location(), {}, { signal: controller.signal }) + const rejected = expect(pending).rejects.toThrow('cancelled') + await vi.waitFor(async () => expect(Number(await readFile(ready, 'utf8'))).toBeGreaterThan(0), { timeout: 1500 }) + const pid = Number(await readFile(ready, 'utf8')) + const started = performance.now() + controller.abort() + await rejected + expect(performance.now() - started).toBeLessThan(1500) + expect(() => process.kill(pid, 0)).toThrow() +}) + +it('enforces time-budget bounds while accepting their exact endpoints', async () => { + await worker(events.map(emit).join('')) + for (const timeoutMs of [0, 9, 1.5, Infinity, NaN, 3600001]) { + await expect(runWaveformChild(location(), {}, { timeoutMs })).rejects.toThrow('Invalid waveform time budget') + } + expect(await runWaveformChild(location(), {}, { timeoutMs: 3600000 })).toEqual({ readings: [reading], run: result }) + await worker('setInterval(() => {}, 1000);') + await expect(runWaveformChild(location(), {}, { timeoutMs: 10 })).rejects.toThrow('exceeded its time budget') +}) + +it('accepts exact input and diagnostic byte budgets', async () => { + await worker(`process.stderr.write('x'.repeat(16384)); ${events.map(emit).join('')}`) + expect(await runWaveformChild(location(), 'x'.repeat(8 * 1024 * 1024 - 2))).toEqual({ readings: [reading], run: result }) +}) + +it('rejects empty event lines without treating them as absent output', async () => { + await worker(`process.stdout.write('\\n'); ${events.map(emit).join('')}`) + await expect(runWaveformChild(location(), {})).rejects.toThrow('invalid event') +}) + +it('rejects excessive valid readings before exposing a result', async () => { + await worker(`for (let i=0; i<20001; i++) { ${emit(events[0])} } + ${emit({ type: 'result', result: { ...result, counts: { ...result.counts, scored_outputs: 20001 } } })}`) + await expect(runWaveformChild(location(), {})).rejects.toThrow('invalid event') +}) + +it('bounds aggregate output even when every event is valid and small enough', async () => { + const large = { type: 'reading', reading: { ...reading, limitations: Array.from({ length: 40 }, () => 'x'.repeat(200)) } } + await worker(`for (let i=0; i<2500; i++) { ${emit(large)} } + ${emit({ type: 'result', result: { ...result, counts: { ...result.counts, scored_outputs: 2500 } } })}`) + await expect(runWaveformChild(location(), {})).rejects.toThrow('output exceeds 16 MiB') +}) + +it('pins the requested Python build in the envelope and rejects mismatched completion', async () => { + await writeFile(executable, `#!${process.execPath}\nlet input='';process.stdin.on('data',chunk=>input+=chunk);process.stdin.on('end',()=>{if(JSON.parse(input).expected_build_hash!==${JSON.stringify(hash)})process.exitCode=1;${events.map(emit).join('')}});`, { mode: 0o700 }) + expect((await runWaveformChild(location(), {}, { expectedBuildHash: hash })).run.build_hash).toBe(hash) + await worker(events.map(emit).join('')) + await expect(runWaveformChild(location(), {}, { expectedBuildHash: 'b'.repeat(64) })).rejects.toThrow('incomplete result') + expect(() => runWaveformChild(location(), {}, { expectedBuildHash: 'invalid' })).toThrow() +}) + +it('retains valid measured process resources and rejects fabricated scopes or negative counts', async () => { + const resources = { version: 'waveform-process-resources-v1', wallMs: 120, cpuMs: 80, peakRssBytes: 102400, + scope: 'python_process_until_result', memoryScope: 'process_lifetime_high_water_mark' } + await worker([events[0], { type: 'result', result: { ...result, resources } }].map(emit).join('')) + expect((await runWaveformChild(location(), {})).run.resources).toEqual(resources) + for (const patch of [{ wallMs: -1 }, { cpuMs: -1 }, { peakRssBytes: 0 }, { scope: 'whole_job' }]) { + await worker([events[0], { type: 'result', result: { ...result, resources: { ...resources, ...patch } } }].map(emit).join('')) + await expect(runWaveformChild(location(), {})).rejects.toThrow('invalid event') + } +}) diff --git a/src/biometrics/backtest/tests/waveform-contracts.test.ts b/src/biometrics/backtest/tests/waveform-contracts.test.ts new file mode 100644 index 000000000..1f78d7c36 --- /dev/null +++ b/src/biometrics/backtest/tests/waveform-contracts.test.ts @@ -0,0 +1,56 @@ +// @vitest-environment node +import { describe, expect, it } from 'vitest' +import { rawArtifactDescriptorSchema, retainedWaveformInputSchema, waveformDatasetInputsSchema } from '../waveform-contracts' + +const hash = (char: string) => char.repeat(64) +const artifact = (char: string, compressed = 10) => ({ name: `${hash(char)}.RAW.gz`, sha256: hash(char), compressed_bytes: compressed, expanded_bytes: 20 }) +const source = (char: string, first: number, last: number, compressed = 10) => ({ artifact: artifact(char, compressed), first_event_ms: first, last_event_ms: last }) +const input = (changes: Record = {}) => ({ version: 'retained-waveform-input-v1', retained_start_ms: 1000, score_start_ms: 2000, score_end_ms: 5000, + artifacts: [source('a', 1000, 3000), source('b', 3500, 6000)], contexts: [], ...changes }) +const context = (start: number, end: number, changes: Record = {}) => ({ start_ms: start, end_ms: end, active_sides: null, left_occupied: null, right_occupied: null, pump_context_known: false, ...changes }) + +describe('retained waveform input contract', () => { + it('accepts an ordered envelope with ordered nonoverlapping contexts and either sleeper mode', () => { + const parsed = retainedWaveformInputSchema.parse(input({ contexts: [context(1000, 2000, { active_sides: ['left'] }), context(2000, 2500, { active_sides: ['left', 'right'], left_occupied: true, right_occupied: false, pump_context_known: true }), context(4000, 5000, { active_sides: [] })] })) + expect(parsed.contexts).toHaveLength(3) + }) + it('requires artifact names to match their content hash', () => { + expect(() => rawArtifactDescriptorSchema.parse({ ...artifact('a'), name: `${hash('b')}.RAW.gz` })).toThrow('content hash') + expect(() => rawArtifactDescriptorSchema.parse({ ...artifact('a'), sha256: 'short' })).toThrow() + }) + it('rejects reversed, swapped and overlapping replay windows and event bounds', () => { + expect(() => retainedWaveformInputSchema.parse(input({ retained_start_ms: 2500 }))).toThrow('Invalid retained replay window') + expect(() => retainedWaveformInputSchema.parse(input({ score_end_ms: 2000 }))).toThrow('Invalid retained replay window') + expect(() => retainedWaveformInputSchema.parse(input({ artifacts: [source('a', 3000, 1000)] }))).toThrow('no events in the retained window') + expect(() => retainedWaveformInputSchema.parse(input({ artifacts: [source('a', 100, 900)] }))).toThrow('no events in the retained window') + expect(() => retainedWaveformInputSchema.parse(input({ artifacts: [source('a', 5000, 6000)] }))).toThrow('no events in the retained window') + expect(() => retainedWaveformInputSchema.parse(input({ artifacts: [source('a', 1000, 3000), source('a', 3500, 6000)] }))).toThrow('distinct with nonoverlapping ordered') + expect(() => retainedWaveformInputSchema.parse(input({ artifacts: [source('a', 1000, 3000), source('b', 3000, 6000)] }))).toThrow('distinct with nonoverlapping ordered') + expect(() => retainedWaveformInputSchema.parse(input({ artifacts: [source('a', 1000, 3000, 1024 * 1024 * 1024 + 1)] }))).toThrow('compressed input budget') + expect(() => retainedWaveformInputSchema.parse(input({ artifacts: [] }))).toThrow() + }) + it('rejects contexts that start before the retained window, run backwards, overlap or outlive scoring', () => { + expect(() => retainedWaveformInputSchema.parse(input({ contexts: [context(500, 1500)] }))).toThrow('Context intervals') + expect(() => retainedWaveformInputSchema.parse(input({ contexts: [context(1500, 1500)] }))).toThrow('Context intervals') + expect(() => retainedWaveformInputSchema.parse(input({ contexts: [context(1000, 2000), context(1500, 2500)] }))).toThrow('Context intervals') + expect(() => retainedWaveformInputSchema.parse(input({ contexts: [context(4000, 5001)] }))).toThrow('Context intervals') + expect(() => retainedWaveformInputSchema.parse(input({ contexts: [context(1000, 2000, { active_sides: ['right', 'left'] })] }))).toThrow('sleeper mode') + expect(() => retainedWaveformInputSchema.parse(input({ contexts: [context(1000, 2000, { active_sides: ['left', 'left'] })] }))).toThrow('sleeper mode') + }) +}) + +describe('waveform dataset inputs contract', () => { + const cohort = (cohortId: string, changes: Record = {}) => ({ cohortId, input: input(changes) }) + it('accepts shared artifacts with identical evidence and counts shared bytes once', () => { + expect(waveformDatasetInputsSchema.parse([cohort('one'), cohort('two')])).toHaveLength(2) + const half = 600 * 1024 * 1024 + expect(waveformDatasetInputsSchema.parse([cohort('one', { artifacts: [source('a', 1000, 3000, half)] }), cohort('two', { artifacts: [source('a', 1000, 3000, half)] })])).toHaveLength(2) + }) + it('rejects repeated cohorts, conflicting shared artifact evidence and storage over budget', () => { + expect(() => waveformDatasetInputsSchema.parse([cohort('one'), cohort('one')])).toThrow('Select each waveform cohort once') + expect(() => waveformDatasetInputsSchema.parse([cohort('one'), cohort('two', { artifacts: [source('a', 1000, 2999)] })])).toThrow('Conflicting evidence') + const half = 600 * 1024 * 1024 + expect(() => waveformDatasetInputsSchema.parse([cohort('one', { artifacts: [source('a', 1000, 3000, half)] }), cohort('two', { artifacts: [source('b', 1000, 3000, half)] })])).toThrow('RAW storage budget') + expect(() => waveformDatasetInputsSchema.parse([])).toThrow() + }) +}) diff --git a/src/biometrics/backtest/tests/waveform-dataset.test.ts b/src/biometrics/backtest/tests/waveform-dataset.test.ts new file mode 100644 index 000000000..48f61a75f --- /dev/null +++ b/src/biometrics/backtest/tests/waveform-dataset.test.ts @@ -0,0 +1,111 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { afterEach, beforeEach, expect, it } from 'vitest' +import * as schema from '@/src/db/biometrics-schema' +import { dataHash } from '../../algorithms/canonical' +import { DatasetStore } from '../datasets' +import { datasetManifestSchema } from '../contracts' +import { prepareWaveformDataset } from '../waveform-dataset' +import { retainedWaveformInputSchema, waveformDatasetInputsSchema } from '../waveform-contracts' + +let sqlite: Database.Database +let db: ReturnType> +let datasets: DatasetStore +let base: ReturnType +const start = 1791673200000 +beforeEach(() => { + sqlite = new Database(':memory:') + db = drizzle(sqlite, { schema }) + migrate(db, { migrationsFolder: 'src/db/biometrics-migrations' }) + for (const id of [1, 2, 3]) db.insert(schema.sleepRecords).values({ id, side: 'left', + enteredBedAt: new Date(start + id * 86400000), leftBedAt: new Date(start + id * 86400000 + 3600000), + sleepDurationSeconds: 3600, timesExitedBed: 0 }).run() + datasets = new DatasetStore(db) + base = datasets.read(datasets.create({ side: 'left', subjectId: 'fixture', sessionIds: [1, 2, 3], + referenceImportIds: [], timezone: 'UTC', idempotencyKey: 'base' }).id) +}) +afterEach(() => sqlite.close()) +function inputs() { + return datasetManifestSchema.parse(base.manifest).splitProtocol.assignments.map((cohort, index) => ({ + cohortId: cohort.cohortId, + input: { version: 'retained-waveform-input-v1' as const, retained_start_ms: cohort.startMs - 300000, + score_start_ms: cohort.startMs, score_end_ms: cohort.endMs, contexts: [], + artifacts: [{ artifact: { name: `${String(index + 1).repeat(64)}.RAW.gz`, sha256: String(index + 1).repeat(64), compressed_bytes: 100, expanded_bytes: 200 }, + first_event_ms: cohort.startMs - 300000, last_event_ms: cohort.endMs - 1 }] }, + })) +} +function source() { + return { ...base, manifest: datasetManifestSchema.parse(base.manifest) } +} + +it('creates a separate immutable snapshot with stable request/content hashes and original observations', () => { + const before = structuredClone(base) + const first = prepareWaveformDataset(source(), inputs().reverse(), 'raw', 1000) + const retry = prepareWaveformDataset(source(), inputs(), 'raw', 2000) + expect(first.requestHash).toBe(retry.requestHash) + expect(first.contentHash).toBe(retry.contentHash) + expect(first.id).not.toBe(retry.id) + expect(base).toEqual(before) + expect(first.snapshot.sessions).toEqual(base.snapshot.sessions) + expect(first.manifest.parentDataset).toEqual({ id: base.id, contentHash: base.contentHash }) + expect(first.manifest.capabilities.signalReplay).toBe(true) + expect(first.contentHash).toBe(dataHash({ manifest: first.manifest, snapshot: first.snapshot })) + expect(datasets.list('left')).toHaveLength(1) // Preparation performs no publication. +}) + +it('preserves waveform inputs through the shared dataset reader and detects later corruption', () => { + const prepared = prepareWaveformDataset(source(), inputs(), 'raw') + db.insert(schema.biometricDatasets).values({ ...prepared, + byteLength: Buffer.byteLength(JSON.stringify(prepared.manifest)) + Buffer.byteLength(JSON.stringify(prepared.snapshot)) }).run() + const retained = datasets.read(prepared.id) + expect(retained.snapshot).toEqual(prepared.snapshot) + sqlite.prepare('UPDATE biometric_datasets SET snapshot = json_set(snapshot, \'$.waveformInputs[0].input.artifacts[0].artifact.expanded_bytes\', 999) WHERE id = ?').run(prepared.id) + expect(() => datasets.read(prepared.id)).toThrow('integrity') +}) + +it('rejects missing, duplicated and mismatched whole-night inputs and forged source identity', () => { + expect(() => prepareWaveformDataset(source(), inputs().slice(1), 'raw')).toThrow('every whole-night') + const duplicated = inputs() + duplicated[1] = duplicated[0] + expect(() => prepareWaveformDataset(source(), duplicated, 'raw')).toThrow('each waveform cohort once') + const wrongWindow = inputs() + wrongWindow[0].input.score_start_ms++ + expect(() => prepareWaveformDataset(source(), wrongWindow, 'raw')).toThrow('whole-night cohort') + expect(() => prepareWaveformDataset({ ...source(), contentHash: '0'.repeat(64) }, inputs(), 'raw')).toThrow('integrity') +}) + +it('rejects overlapping source/context windows, invalid modes and inconsistent shared evidence', () => { + const input = inputs()[0].input + expect(() => retainedWaveformInputSchema.parse({ ...input, artifacts: [...input.artifacts, input.artifacts[0]] })).toThrow('distinct') + const context = { start_ms: input.retained_start_ms, end_ms: input.score_end_ms, + active_sides: ['right', 'left'], left_occupied: null, right_occupied: null, pump_context_known: false } + expect(() => retainedWaveformInputSchema.parse({ ...input, contexts: [context] })).toThrow('sleeper mode') + expect(() => retainedWaveformInputSchema.parse({ ...input, contexts: [{ ...context, active_sides: null }, { ...context, active_sides: null }] })).toThrow('nonoverlapping') + const shared = [{ cohortId: 'one', input }, { cohortId: 'two', input: structuredClone(input) }] + expect(waveformDatasetInputsSchema.parse(shared)).toEqual(shared) + shared[1].input.artifacts[0].artifact.expanded_bytes++ + expect(() => waveformDatasetInputsSchema.parse(shared)).toThrow('Conflicting evidence') +}) + +it('accepts only processed sources and bounds the retained dataset size', () => { + const prepared = prepareWaveformDataset(source(), inputs(), 'raw') + expect(() => prepareWaveformDataset(prepared, inputs(), 'again')).toThrow('processed source dataset') + expect(() => prepareWaveformDataset({ ...source(), manifest: prepared.manifest }, inputs(), 'again')).toThrow('processed source dataset') + const oversized = source() + oversized.snapshot.sessions[0].presentIntervals = Array.from({ length: 130 }, () => 'x'.repeat(65536)) + oversized.contentHash = dataHash({ manifest: oversized.manifest, snapshot: oversized.snapshot }) + expect(() => prepareWaveformDataset(oversized, inputs(), 'raw')).toThrow('byte budget') +}) + +it('detaches opaque historical context from caller-owned objects before hashing', () => { + const mutable = source() + const interval = { start: start + 1, end: start + 2 } + mutable.snapshot.sessions[0].presentIntervals = [interval] + mutable.contentHash = dataHash({ manifest: mutable.manifest, snapshot: mutable.snapshot }) + const prepared = prepareWaveformDataset(mutable, inputs(), 'raw') + interval.end++ + expect(prepared.snapshot.sessions[0].presentIntervals).toEqual([{ start: start + 1, end: start + 2 }]) + expect(dataHash({ manifest: prepared.manifest, snapshot: prepared.snapshot })).toBe(prepared.contentHash) +}) diff --git a/src/biometrics/backtest/tests/waveform-evaluator.test.ts b/src/biometrics/backtest/tests/waveform-evaluator.test.ts new file mode 100644 index 000000000..9b0208730 --- /dev/null +++ b/src/biometrics/backtest/tests/waveform-evaluator.test.ts @@ -0,0 +1,150 @@ +import { previewReferenceImport } from '../../references/normalize' +// @vitest-environment node +import { expect, it } from 'vitest' +import { backtestFixture } from './fixtures/backtest' +import { dataHash, configHash } from '../../algorithms/canonical' +import { evaluateBacktest } from '../evaluator' +import { summarizeBacktestResult } from '../report-summary' +import { defaultConfig, manifestHash } from '../../algorithms/registry' + +function fixture() { + const base = backtestFixture() + const cohort = base.dataset.manifest.splitProtocol.assignments[0] + const digest = 'a'.repeat(64), build = 'b'.repeat(64), resolvedHash = configHash(defaultConfig) + const dataset = { ...base.dataset, snapshot: { ...base.dataset.snapshot, version: 'waveform-dataset-v1' as const, + waveformInputs: [{ cohortId: cohort.cohortId, input: { version: 'retained-waveform-input-v1' as const, + retained_start_ms: cohort.startMs, score_start_ms: cohort.startMs, score_end_ms: cohort.endMs, contexts: [], + artifacts: [{ artifact: { sha256: digest, name: `${digest}.RAW.gz`, compressed_bytes: 1, expanded_bytes: 2 }, + first_event_ms: cohort.startMs, last_event_ms: cohort.endMs - 1 }] } }] } } + const protocol = { ...base.job.protocol, mode: 'signal_replay' as const, waveformStages: false } + const readings = [{ side: 'left' as const, timestampMs: cohort.startMs, heartRate: 62, hrvIndex: null, + hrvMetric: 'window_ibi_variability_v1' as const, breathingRate: 12, qualityScore: 1, flags: [], hrRaw: 62, + sourceSide: 'left' as const, configHash: resolvedHash, limitations: ['unknown_context'] }] + const outputs = [{ cohortId: cohort.cohortId, configHash: resolvedHash, readings, run: { + version: 'retained-waveform-run-v1' as const, build_hash: build, decoded_records: 1, raw_records: 1, + counts: { records: 1, accepted_frames: 1, empty_channel_frames: 0, pump_gated_frames: 0, preroll_outputs: 0, scored_outputs: 1 }, + limitations: ['unknown_context'], config_hashes: { left: resolvedHash, right: resolvedHash }, artifact_hashes: [digest], + } }] + return { dataset, job: { ...base.job, datasetHash: dataHash(dataset), protocol, protocolHash: configHash(protocol), + waveform: { pythonBuildHash: build, otherSide: { manifestHash, resolvedHash, resolved: defaultConfig } } }, + waveform: { hash: dataHash(outputs), outputs } } +} +it('scores derived readings for matching labels without replacing immutable dataset identity', () => { + const work = fixture() + const before = structuredClone(work) + const report = evaluateBacktest(work) + expect(report.version).toBe('waveform-backtest-report-v1') + expect(report.datasetHash).toBe(work.job.datasetHash) + expect(report.nights[0].uniqueObservationCount).toBe(1) + expect(report.nights[0].evaluations[0].windows[0].observationCount).toBe(1) + expect(report.nights[0].evaluations[0].windows[0].numeric[0]).toMatchObject({ pairedCount: 1, metrics: { mae: 0 } }) + expect(report.nights[0].evaluations[1].windows).toEqual(report.nights[0].evaluations[0].windows) + expect(report.nights[0].evaluations[0].windows[0].blocks).toEqual([]) + expect(report.nights[0].numericComparisons?.[0].metrics).toEqual(expect.arrayContaining([ + expect.objectContaining({ metric: 'heart_rate', baseline: 'default', candidate: 'current', + commonObservationCount: 1, differences: expect.objectContaining({ commonMae: 0 }), coverageReduced: false }), + ])) + const summary = summarizeBacktestResult({ available: true, report }) + expect(summary?.report).toMatchObject({ nights: [expect.objectContaining({ numericComparisons: report.nights[0].numericComparisons })] }) + expect(report.numericNightSummary?.[0]).toMatchObject({ metric: 'heart_rate', scoredNights: 1, equalNightMeanMaeDifference: 0, uncertainty: { statistic: 'equal_night_mean_common_mae_difference', status: 'not_requested' } }) + expect(work).toEqual(before) +}) +it('rejects incomplete, mismatched or extraneous derived waveform evidence', () => { + for (const mutate of [ + (work: ReturnType) => { work.waveform.hash = '0'.repeat(64) }, + (work: ReturnType) => { work.waveform.outputs = [] }, + (work: ReturnType) => { work.waveform.outputs.push(structuredClone(work.waveform.outputs[0])) }, + (work: ReturnType) => { work.waveform.outputs[0].run.build_hash = '0'.repeat(64) }, + (work: ReturnType) => { work.waveform.outputs[0].run.config_hashes.right = '0'.repeat(64) }, + (work: ReturnType) => { work.waveform.outputs[0].run.artifact_hashes = ['0'.repeat(64)] }, + (work: ReturnType) => { work.waveform.outputs[0].readings[0].timestampMs-- }, + (work: ReturnType) => { work.waveform.outputs[0].readings[0].configHash = '0'.repeat(64) }, + ]) { + const work = fixture() + mutate(work) + if (work.waveform.hash !== '0'.repeat(64)) work.waveform.hash = dataHash(work.waveform.outputs) + expect(() => evaluateBacktest(work)).toThrow() + } +}) +it('rejects waveform evidence supplied to processed evaluation and missing evidence in signal replay', () => { + expect(() => evaluateBacktest({ ...backtestFixture(), waveform: fixture().waveform })).toThrow('Unexpected waveform') + const work = fixture() + expect(() => evaluateBacktest({ ...work, waveform: undefined })).toThrow('Missing or corrupt') +}) + +it('retains objective thresholds but cannot promote without qualified measured evidence', () => { + const work = fixture() + const protocol = { ...work.job.protocol, objectivePolicy: { version: 'numeric-objective-v1' as const, qualificationId: 'test-only', metric: 'heart_rate' as const, + minimumNights: 3, minimumPairedReferencesPerNight: 10, maximumLostObservationFraction: 0, minimumMaeImprovement: 2, + maximumNightMaeRegression: 1, maximumRegressedNightFraction: 0, maximumSubgroupMaeRegression: 1, requiredSubgroups: [], + operational: { maximumWallMs: 1000, maximumPeakRssBytes: 1000, maximumIngestionAgeMs: 1000 } } } + const report = evaluateBacktest({ ...work, job: { ...work.job, protocol, protocolHash: configHash(protocol), + configs: [...work.job.configs, { ...work.job.configs[0], label: 'candidate' }] } }) + expect(report.objectiveAssessments).toHaveLength(2) + expect(report.objectiveAssessments[0]).toMatchObject({ status: 'insufficient_evidence', designated: false, + missingEvidence: expect.arrayContaining(['policy_not_designated', 'operational_evidence_unavailable']) }) + expect(report.protocol.objectivePolicy).toEqual(protocol.objectivePolicy) +}) + +it('retains measured waveform costs through compact report projection without claiming whole-job health', () => { + const work = fixture() + const outputs = work.waveform.outputs.map(output => ({ ...output, run: { ...output.run, resources: { + version: 'waveform-process-resources-v1' as const, wallMs: 120, cpuMs: 80, peakRssBytes: 102400, + scope: 'python_process_until_result' as const, memoryScope: 'process_lifetime_high_water_mark' as const, + } } })) + const report = evaluateBacktest({ ...work, waveform: { hash: dataHash(outputs), outputs } }) + const summary = summarizeBacktestResult({ available: true, report }) + expect(summary?.report).toMatchObject({ waveform: { resources: { + expectedProcesses: outputs.length, measuredProcesses: outputs.length, totalWallMs: outputs.length * 120, + totalCpuMs: outputs.length * 80, largestProcessPeakRssBytes: 102400, ingestionHealthMeasured: false, + } } }) +}) + +it('exposes a late-window regression hidden by lower whole-night error', () => { + const work = fixture(), start = work.dataset.snapshot.sessions[0].startMs + const baseline = work.waveform.outputs[0] + baseline.readings = [80, 60].map((heartRate, index) => ({ ...baseline.readings[0], heartRate, timestampMs: start + index * 300000 })) + baseline.run.counts.scored_outputs = 2 + const candidate = structuredClone(defaultConfig) + candidate.heart_rate.trackingToleranceBpm = 20 + const candidateHash = configHash(candidate) + const output = { ...baseline, configHash: candidateHash, + readings: baseline.readings.map((row, index) => ({ ...row, heartRate: index ? 65 : 60, configHash: candidateHash })), + run: { ...baseline.run, config_hashes: { ...baseline.run.config_hashes, left: candidateHash } } } + work.dataset.snapshot.references[0].measurements = previewReferenceImport({ side: 'left', source: 'fixture', subjectId: 'A', rows: [ + { recordKey: 'early', metric: 'heart_rate', timestampMs: start, value: 60, unit: 'bpm' }, + { recordKey: 'late', metric: 'heart_rate', timestampMs: start + 300000, value: 60, unit: 'bpm' }, + ] }).measurements + const protocol = { ...work.job.protocol, objectivePolicy: { version: 'numeric-objective-v1' as const, qualificationId: 'test-only', metric: 'heart_rate' as const, + minimumNights: 3, minimumPairedReferencesPerNight: 1, maximumLostObservationFraction: 0, minimumMaeImprovement: 2, + maximumNightMaeRegression: 1, maximumRegressedNightFraction: 0, maximumSubgroupMaeRegression: 1, requiredSubgroups: ['window_second_half'], + operational: { maximumWallMs: 1000, maximumPeakRssBytes: 1000, maximumIngestionAgeMs: 1000 } } } + const outputs = [baseline, output] + const report = evaluateBacktest({ ...work, waveform: { outputs, hash: dataHash(outputs) }, job: { ...work.job, + datasetHash: dataHash(work.dataset), protocol, protocolHash: configHash(protocol), + configs: [...work.job.configs, { ...work.job.configs[0], label: 'candidate', resolved: candidate, resolvedHash: candidateHash }] } }) + expect(report.numericNightSummary?.find(row => row.baseline === 'default' && row.candidate === 'candidate' && row.metric === 'heart_rate')?.equalNightMeanMaeDifference).toBe(-7.5) + expect(report.numericSubgroupSummary?.find(row => row.name === 'window_second_half' && row.summary.baseline === 'default' && row.summary.candidate === 'candidate')?.summary.equalNightMeanMaeDifference).toBe(5) + expect(report.objectiveAssessments[0].evidence.subgroups).toContainEqual({ name: 'window_second_half', pairedNights: 1, maeDifference: 5 }) + expect(report.objectiveAssessments[0].evidence.subgroups).toContainEqual({ name: 'reference_intervals', pairedNights: 0, maeDifference: null }) +}) + +it('evaluates each numeric policy against both baselines without mixing metric subgroups', () => { + const work = fixture() + const objectivePolicies = (['heart_rate', 'breathing_rate', 'window_ibi_variability_v1'] as const).map(metric => ({ + version: 'numeric-objective-v1' as const, qualificationId: 'multi-test', metric, minimumNights: 3, + minimumPairedReferencesPerNight: 1, maximumLostObservationFraction: 0, minimumMaeImprovement: 1, + maximumNightMaeRegression: 1, maximumRegressedNightFraction: 0, maximumSubgroupMaeRegression: 1, + requiredSubgroups: ['reference_points'], operational: { maximumWallMs: 1000, maximumPeakRssBytes: 1000, maximumIngestionAgeMs: 1000 }, + })) + const protocol = { ...work.job.protocol, objectivePolicies } + const report = evaluateBacktest({ ...work, job: { ...work.job, protocol, protocolHash: configHash(protocol), + configs: [...work.job.configs, { ...work.job.configs[0], label: 'candidate' }] } }) + expect(report.objectiveAssessments).toHaveLength(6) + for (const policy of objectivePolicies) { + const assessments = report.objectiveAssessments.filter(row => row.policy.metric === policy.metric) + expect(assessments.map(row => row.baseline).sort()).toEqual(['current', 'default']) + expect(assessments.every(row => row.status === 'insufficient_evidence' && !row.designated)).toBe(true) + expect(report.numericSubgroupSummary?.filter(row => row.summary.metric === policy.metric)).toHaveLength(12) + } +}) diff --git a/src/biometrics/backtest/tests/waveform-execution.test.ts b/src/biometrics/backtest/tests/waveform-execution.test.ts new file mode 100644 index 000000000..38b350dac --- /dev/null +++ b/src/biometrics/backtest/tests/waveform-execution.test.ts @@ -0,0 +1,114 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { mkdtemp, realpath, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import * as schema from '@/src/db/biometrics-schema' +import { configHash, dataHash } from '../../algorithms/canonical' +import { defaultConfig, manifestHash } from '../../algorithms/registry' +import { datasetManifestSchema } from '../contracts' +import { DatasetStore } from '../datasets' +import { podRawInspectionLocation } from '../raw-inspection-child' +import { prepareWaveformDataset } from '../waveform-dataset' +import { waveformEvaluatorBuildIdentity } from '../waveform-build' +import { runWaveformChild } from '../waveform-child' +import { executeWaveformInputs, verifyWaveformJobBuild } from '../waveform-execution' +import { backtestFixture } from './fixtures/backtest' + +vi.mock('../waveform-build', () => ({ waveformEvaluatorBuildIdentity: vi.fn() })) +vi.mock('../waveform-child', () => ({ runWaveformChild: vi.fn() })) + +const build = 'b'.repeat(64), python = 'c'.repeat(64), digest = 'a'.repeat(64), resolvedHash = configHash(defaultConfig) +let retained: ReturnType +const start = 1791673200000 +function fixture(side: 'left' | 'right' = 'left', labels: Array<'default' | 'current' | 'candidate'> = ['default', 'current']) { + const base = backtestFixture() + const dataset = structuredClone({ manifest: retained.manifest, snapshot: { ...retained.snapshot, side } }) + const protocol = { ...base.job.protocol, mode: 'signal_replay' as const } + return { dataset, job: { ...base.job, evaluatorBuildHash: build, datasetHash: dataHash(dataset), protocol, protocolHash: configHash(protocol), + configs: labels.map(label => ({ ...base.job.configs[0], label })), + waveform: { pythonBuildHash: python, otherSide: { manifestHash, resolvedHash, resolved: defaultConfig } } } } +} +const reading = (timestampMs: number) => ({ side: 'left' as const, timestampMs, heartRate: 62, hrvIndex: null, hrvMetric: 'window_ibi_variability_v1' as const, + breathingRate: 12, qualityScore: 1, flags: [], hrRaw: 62, sourceSide: 'left' as const, configHash: resolvedHash, limitations: [] }) +const run = { version: 'retained-waveform-run-v1' as const, build_hash: python, decoded_records: 1, raw_records: 1, + counts: { records: 1, accepted_frames: 1, empty_channel_frames: 0, pump_gated_frames: 0, preroll_outputs: 0, scored_outputs: 1 }, + limitations: [], config_hashes: { left: resolvedHash, right: resolvedHash }, artifact_hashes: [digest] } +const rawLocation = { pythonPath: '/python', modulesPath: '/modules', archiveDirectory: '/archive' } +let directory: string, sqlite: Database.Database, db: ReturnType> +beforeEach(async () => { + directory = await realpath(await mkdtemp(join(tmpdir(), 'waveform-execution-'))) + sqlite = new Database(join(directory, 'biometrics.db')) + db = drizzle(sqlite, { schema }) + migrate(db, { migrationsFolder: 'src/db/biometrics-migrations' }) + db.insert(schema.sleepRecords).values({ id: 1, side: 'left', enteredBedAt: new Date(start), leftBedAt: new Date(start + 600000), sleepDurationSeconds: 600, timesExitedBed: 0 }).run() + const datasets = new DatasetStore(db) + const source = datasets.read(datasets.create({ side: 'left', subjectId: 'A', sessionIds: [1], referenceImportIds: [], timezone: 'UTC', idempotencyKey: 'source' }).id) + const cohort = datasetManifestSchema.parse(source.manifest).splitProtocol.assignments[0] + retained = prepareWaveformDataset({ ...source, manifest: datasetManifestSchema.parse(source.manifest) }, [{ cohortId: cohort.cohortId, + input: { version: 'retained-waveform-input-v1', retained_start_ms: cohort.startMs, score_start_ms: cohort.startMs, score_end_ms: cohort.endMs, contexts: [], + artifacts: [{ artifact: { sha256: digest, name: `${digest}.RAW.gz`, compressed_bytes: 1, expanded_bytes: 2 }, first_event_ms: cohort.startMs, last_event_ms: cohort.endMs - 1 }] } }], 'raw') + vi.mocked(waveformEvaluatorBuildIdentity).mockResolvedValue({ hash: build, python: { hash: python } } as Awaited>) + vi.mocked(runWaveformChild).mockResolvedValue({ readings: [reading(start)], run }) +}) +afterEach(async () => { + vi.clearAllMocks() + sqlite.close() + await rm(directory, { recursive: true, force: true }) +}) +const options = () => ({ workerPath: join(directory, 'worker.cjs'), signal: new AbortController().signal }) + +it('pins the coordinator beside the worker and rejects any build drift since enqueue', async () => { + const work = fixture() + await verifyWaveformJobBuild(work, options()) + expect(waveformEvaluatorBuildIdentity).toHaveBeenLastCalledWith(join(directory, 'coordinator.cjs'), join(directory, 'worker.cjs'), podRawInspectionLocation, expect.any(AbortSignal)) + await verifyWaveformJobBuild(work, { ...options(), rawLocation }) + expect(waveformEvaluatorBuildIdentity).toHaveBeenLastCalledWith(expect.any(String), expect.any(String), rawLocation, expect.any(AbortSignal)) + vi.mocked(waveformEvaluatorBuildIdentity).mockResolvedValueOnce({ hash: '0'.repeat(64), python: { hash: python } } as Awaited>) + await expect(verifyWaveformJobBuild(work, options())).rejects.toThrow('build changed') + vi.mocked(waveformEvaluatorBuildIdentity).mockResolvedValueOnce({ hash: build, python: { hash: '0'.repeat(64) } } as Awaited>) + await expect(verifyWaveformJobBuild(work, options())).rejects.toThrow('build changed') +}) + +it('runs each cohort and distinct configuration through Python beside the persistent database', async () => { + const work = fixture('left', ['default', 'current', 'candidate']) + const result = await executeWaveformInputs(db, work, { ...options(), rawLocation, timeoutMs: 5000 }) + expect(runWaveformChild).toHaveBeenCalledTimes(1) + expect(runWaveformChild).toHaveBeenCalledWith({ ...rawLocation, retainedDirectory: join(directory, 'biometric-raw') }, + { version: 'retained-waveform-work-v1', input: expect.objectContaining({ version: 'retained-waveform-input-v1', score_start_ms: start }), configs: { left: defaultConfig, right: defaultConfig } }, + { signal: expect.any(AbortSignal), timeoutMs: 5000, cpuSeconds: undefined, expectedBuildHash: python }) + expect(result.outputs).toEqual([{ cohortId: retained.manifest.splitProtocol.assignments[0].cohortId, configHash: resolvedHash, readings: [reading(start)], run }]) + expect(result.hash).toBe(dataHash(result.outputs)) + expect(waveformEvaluatorBuildIdentity).toHaveBeenCalledTimes(2) +}) + +it('routes the retained side to the candidate configuration and the replay budget to the child', async () => { + const work = fixture('right') + const replayBudget = { jobWallMs: 60000, waveformWallMs: 30000, waveformCpuSeconds: 20 } + vi.mocked(runWaveformChild).mockResolvedValueOnce({ readings: [{ ...reading(start), side: 'right', sourceSide: 'right' }], run }) + await executeWaveformInputs(db, work, { ...options(), replayBudget }) + expect(runWaveformChild).toHaveBeenCalledWith({ ...podRawInspectionLocation, retainedDirectory: join(directory, 'biometric-raw') }, + expect.objectContaining({ configs: { left: defaultConfig, right: defaultConfig } }), + expect.objectContaining({ timeoutMs: 30000, cpuSeconds: 20 })) +}) + +it('rejects processed jobs, in-memory databases, missing cohorts, cancellation and reading budget overruns', async () => { + const processed = backtestFixture() + await expect(executeWaveformInputs(db, processed, options())).rejects.toThrow('Missing retained waveform job inputs') + await expect(executeWaveformInputs(db, { ...fixture(), job: { ...fixture().job, waveform: undefined } }, options())).rejects.toThrow('Missing retained waveform job inputs') + await expect(executeWaveformInputs(db, { ...fixture(), dataset: processed.dataset }, options())).rejects.toThrow('Missing retained waveform job inputs') + const memory = drizzle(new Database(':memory:'), { schema }) + await expect(executeWaveformInputs(memory, fixture(), options())).rejects.toThrow('persistent database') + const orphan = fixture() + if (orphan.dataset.snapshot.version === 'waveform-dataset-v1') orphan.dataset.snapshot.waveformInputs[0].cohortId = 'other' + await expect(executeWaveformInputs(db, orphan, options())).rejects.toThrow('Missing retained waveform cohort') + const controller = new AbortController() + controller.abort() + await expect(executeWaveformInputs(db, fixture(), { ...options(), signal: controller.signal })).rejects.toThrow('cancelled') + expect(runWaveformChild).not.toHaveBeenCalled() + vi.mocked(runWaveformChild).mockResolvedValueOnce({ readings: Array.from({ length: 20001 }, (_, index) => reading(start + index)), run }) + await expect(executeWaveformInputs(db, fixture(), options())).rejects.toThrow('reading budget exceeded') +}) diff --git a/src/biometrics/backtest/tests/waveform-publication.integration.test.ts b/src/biometrics/backtest/tests/waveform-publication.integration.test.ts new file mode 100644 index 000000000..f2f966676 --- /dev/null +++ b/src/biometrics/backtest/tests/waveform-publication.integration.test.ts @@ -0,0 +1,230 @@ +// @vitest-environment node +import { execFileSync } from 'node:child_process' +import { mkdtempSync, readdirSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { expect, it } from 'vitest' +import * as schema from '@/src/db/biometrics-schema' +import { configHash } from '../../algorithms/canonical' +import { BacktestJobStore } from '../jobs' +import { defaultConfig, manifestHash } from '../../algorithms/registry' +import { DatasetStore } from '../datasets' +import { datasetManifestSchema } from '../contracts' +import { prepareSelectedWaveformDataset } from '../waveform-selection' +import { RawSourceCatalog } from '../raw-catalog' +import { waveformEvaluatorBuildIdentity } from '../waveform-build' +import { runCoordinatorTask } from '../coordinator' +import { runWaveformChild } from '../waveform-child' +import type { z } from 'zod' +import type { retainedWaveformInputSchema } from '../waveform-contracts' + +type Input = z.infer +const python = resolve('modules/piezo-processor/.venv/bin/python') +const environment: NodeJS.ProcessEnv = { PYTHONPATH: resolve('modules'), OPENBLAS_NUM_THREADS: '1', LANG: 'C.UTF-8', TZ: 'UTC', NODE_ENV: 'test' } + +it('publishes, verifies, replays, releases and collects a retained waveform dataset without changing observations', async () => { + const directory = mkdtempSync(join(tmpdir(), 'waveform-publication-integration-')) + const database = join(directory, 'biometrics.db') + const archive = join(directory, 'archive') + const retained = join(directory, 'biometric-raw') + const sqlite = new Database(database) + try { + // Keep the kernel lock real but isolate its inode from /run and other tests. + // Production has no CLI/environment override for the host execution lock. + const fixturePython = join(directory, 'fixture-python') + writeFileSync(fixturePython, `#!${python} +import runpy, sys +args = sys.argv[1:] +if args[0] == '-B': + args = args[1:] +assert args[0] == '-m' +module = args[1] +sys.argv = [module, *args[2:]] +if module == 'common.biometrics.replay_worker': + from common.biometrics import replay_worker + from common.biometrics.execution_lock import acquire_execution_lock + replay_worker.acquire_execution_lock = lambda: acquire_execution_lock(${JSON.stringify(join(directory, 'execution.lock'))}) + sys.exit(replay_worker.main()) +runpy.run_module(module, run_name='__main__') +`, { mode: 0o700 }) + const db = drizzle(sqlite, { schema }) + migrate(db, { migrationsFolder: 'src/db/biometrics-migrations' }) + const fixture = JSON.parse(execFileSync(python, ['-B', '-c', ` +import json, gzip, sys +from dataclasses import asdict +from pathlib import Path +import cbor2 +import numpy as np +from common.biometrics.raw_sources import ArchiveSource +root=Path(sys.argv[1]); root.mkdir() +start=1700000000000 +records=[{'type':'frzHealth','ts':start/1000,'left':{'pumpRpm':0},'right':{'pumpRpm':0}}] +for second in range(150): + samples=(np.sin((np.arange(500)+second*500)/500*2*np.pi*1.2)*1000000).astype(' now) + const base = datasets.read(datasets.create({ side: 'left', subjectId: 'fixture', sessionIds: [1], + referenceImportIds: [], timezone: 'UTC', idempotencyKey: 'processed' }).id) + const manifest = datasetManifestSchema.parse(base.manifest) + const catalog = new RawSourceCatalog(db) + const sourceIds = fixture.inspections.map(inspection => catalog.record(inspection)) + const request = { parentDatasetId: base.id, expectedContentHash: base.contentHash, + cohorts: [{ cohortId: manifest.splitProtocol.assignments[0].cohortId, sourceIds }], idempotencyKey: 'waveform' } + const { prepared } = prepareSelectedWaveformDataset(db, request, now) + execFileSync(process.execPath, ['scripts/build/biometrics-worker.mjs', directory]) + const options = { replayBudget: { jobWallMs: 300000, waveformWallMs: 200000, waveformCpuSeconds: 180 }, databasePath: database, coordinatorPath: join(directory, 'coordinator.cjs'), workerPath: join(directory, 'worker.cjs'), + modulePath: resolve('node_modules'), rawLocation: { pythonPath: fixturePython, modulesPath: resolve('modules'), archiveDirectory: archive } } + expect(await runCoordinatorTask(options, { type: 'createWaveformDataset', input: request })).toEqual({ status: 'deferred' }) + let admissionChecks = 0 + expect(await runCoordinatorTask(options, { type: 'createWaveformDataset', input: request }, { admit: async () => ++admissionChecks === 1 })).toEqual({ status: 'deferred' }) + expect(admissionChecks).toBe(2) + expect(readdirSync(directory)).not.toContain('biometric-raw') + const slowPublisher = join(directory, 'slow-publisher') + writeFileSync(slowPublisher, `#!${process.execPath}\nprocess.stdin.resume();setInterval(()=>{},1000);setTimeout(()=>process.exit(91),10000).unref();`, { mode: 0o700 }) + let resourceChecks = 0 + expect(await runCoordinatorTask({ ...options, rawLocation: { ...options.rawLocation, pythonPath: slowPublisher } }, + { type: 'createWaveformDataset', input: request }, { admit: async () => ++resourceChecks <= 2 })).toEqual({ status: 'deferred' }) + expect(resourceChecks).toBe(3) + expect(sqlite.prepare('SELECT count(*) n FROM biometric_datasets').get()).toEqual({ n: 1 }) + const publish = async (input = request) => { + const result = await runCoordinatorTask(options, { type: 'createWaveformDataset', input }, { admit: async () => true }) + if (result.status !== 'published') throw new Error('Publication was deferred') + return { type: 'result', id: result.dataset.id, contentHash: result.dataset.contentHash, available: result.dataset.available } + } + const published = await publish() + expect(published).toEqual({ type: 'result', id: expect.any(String), contentHash: prepared.contentHash, available: true }) + const frozen = datasets.read(published.id) + expect(frozen.snapshot).toEqual(prepared.snapshot) // Includes shared cross-language content-hash verification. + expect(sqlite.prepare('SELECT count(*) AS count FROM biometric_dataset_raw_artifacts').get()).toEqual({ count: 2 }) + rmSync(archive, { recursive: true }) + for (const id of sourceIds) catalog.markUnavailable(id) + expect(await publish()).toEqual(published) + const retryWithoutAdmission = await runCoordinatorTask(options, { type: 'createWaveformDataset', input: { ...request, cohorts: request.cohorts.map(item => ({ ...item, sourceIds: [...item.sourceIds].reverse() })) } }) + expect(retryWithoutAdmission).toMatchObject({ status: 'published', dataset: { id: published.id } }) + if (frozen.snapshot.version !== 'waveform-dataset-v1') throw new Error('Missing waveform inputs') + const build = await waveformEvaluatorBuildIdentity(options.coordinatorPath, options.workerPath, options.rawLocation) + const replay = await runWaveformChild({ pythonPath: fixturePython, modulesPath: resolve('modules'), retainedDirectory: retained }, { + version: 'retained-waveform-work-v1', input: frozen.snapshot.waveformInputs[0].input, + configs: { left: defaultConfig, right: defaultConfig }, + }, { expectedBuildHash: build.python.hash }) + expect(replay.run.build_hash).toBe(build.python.hash) + expect(replay.readings.length).toBeGreaterThan(0) + expect(replay.run.raw_records).toBe(151) + expect(sqlite.prepare('SELECT heart_rate FROM vitals').all()).toEqual([{ heart_rate: 65 }]) + const jobs = new BacktestJobStore(db) + const experiment = jobs.createExperiment(published.id, { version: 'descriptive-evaluation-v1', mode: 'signal_replay', waveformStages: true, + assumedStageCalibrationQuality: 0, pointToleranceMs: 30000, clockOffsetMs: 0, + objectivePolicy: { version: 'numeric-objective-v1', qualificationId: 'integration-only', metric: 'heart_rate', + minimumNights: 3, minimumPairedReferencesPerNight: 10, maximumLostObservationFraction: 0.1, minimumMaeImprovement: 2, + maximumNightMaeRegression: 1, maximumRegressedNightFraction: 0.34, maximumSubgroupMaeRegression: 1, requiredSubgroups: [], + operational: { maximumWallMs: 30000, maximumPeakRssBytes: 536870912, maximumIngestionAgeMs: 5000, + observations: { version: 'sampled-process-group-v1', maximumMemoryGapMs: 2000, maximumIngestionGapMs: 7000 } } } }) + const candidate = structuredClone(defaultConfig) + candidate.heart_rate.trackingToleranceBpm = 20 + const jobInput = { experimentId: experiment.id, evaluatorBuildHash: build.hash, split: 'tuning' as const, + waveform: { pythonBuildHash: build.python.hash, otherSide: { manifestHash, resolvedHash: configHash(defaultConfig), resolved: defaultConfig } }, + configs: [ + { label: 'default' as const, revisionId: null, manifestHash, resolvedHash: configHash(defaultConfig), resolved: defaultConfig }, + { label: 'current' as const, revisionId: null, manifestHash, resolvedHash: configHash(defaultConfig), resolved: defaultConfig }, + { label: 'candidate' as const, revisionId: null, manifestHash, resolvedHash: configHash(candidate), resolved: candidate }, + ] } + expect(() => jobs.enqueue({ ...jobInput, waveform: undefined, idempotencyKey: 'unpinned-signal' })).toThrow('pinned Python') + expect(() => jobs.enqueue({ ...jobInput, waveform: { ...jobInput.waveform, otherSide: { ...jobInput.waveform.otherSide, resolvedHash: '0'.repeat(64) } }, idempotencyKey: 'wrong-other-side' })).toThrow('Other-side configuration') + const job = jobs.enqueue({ ...jobInput, idempotencyKey: 'signal-job' }) + expect(await runCoordinatorTask(options, { type: 'execute', owner: 'signal-integration' }, { admit: async () => true })).toEqual({ state: 'succeeded', jobId: job.id }) + const report = jobs.result(job.id)?.report + expect(report).toMatchObject({ version: 'waveform-backtest-report-v1', datasetHash: published.contentHash, + evaluatorBuildHash: build.hash, protocol: { mode: 'signal_replay' }, waveform: { outputs: [{}, {}] }, + nights: [{ evaluations: [{ label: 'default' }, { label: 'current' }, { label: 'candidate' }] }] }) + expect(report).toMatchObject({ objectiveAssessments: [ + { baseline: 'default', evidence: { operational: null }, status: 'insufficient_evidence' }, + { baseline: 'current', evidence: { operational: null }, status: 'insufficient_evidence' }, + ] }) + const summary = await runCoordinatorTask(options, { type: 'readSummary', input: { id: job.id } }) + expect(JSON.stringify(summary?.report)).not.toContain('"readings"') + expect(JSON.stringify(report)).toContain('"readings"') + if (!summary) throw new Error('Missing retained report') + const referenceTraceInput = { id: job.id, expectedContentHash: summary.contentHash, + cohortId: prepared.manifest.splitProtocol.assignments[0].cohortId, metric: 'heart_rate' as const } + const references = await runCoordinatorTask(options, { type: 'readReferenceTrace', input: referenceTraceInput }) + expect(references).toMatchObject({ provenance: 'frozen_dataset_reference', datasetHash: published.contentHash, items: [], totalItems: 0 }) + await expect(runCoordinatorTask(options, { type: 'readReferenceTrace', input: { + ...referenceTraceInput, expectedContentHash: '0'.repeat(64), + } })).rejects.toMatchObject({ code: 'REVISION_CONFLICT' }) + const numericTrace = await runCoordinatorTask(options, { type: 'readNumericTrace', input: { + id: job.id, expectedContentHash: summary.contentHash, configuration: 'candidate', limit: 2, + } }) + expect(numericTrace).toMatchObject({ provenance: 'retained_waveform_recomputation', side: 'left', downsampled: false }) + expect(numericTrace.totalItems).toBeGreaterThan(0) + expect(numericTrace.items.every(row => row.side === 'left')).toBe(true) + await expect(runCoordinatorTask(options, { type: 'readNumericTrace', input: { + id: job.id, expectedContentHash: '0'.repeat(64), configuration: 'candidate', + } })).rejects.toMatchObject({ code: 'REVISION_CONFLICT' }) + const wrongBuild = jobs.enqueue({ ...jobInput, evaluatorBuildHash: '0'.repeat(64), idempotencyKey: 'wrong-signal-build' }) + expect(await runCoordinatorTask(options, { type: 'execute', owner: 'wrong-build' }, { admit: async () => true })).toEqual({ state: 'stopped', jobId: wrongBuild.id }) + expect(jobs.get(wrongBuild.id).state).toBe('failed') + expect(jobs.result(wrongBuild.id)).toBeNull() + jobs.releaseResult(job.id, summary.contentHash) + await expect(runCoordinatorTask(options, { type: 'readReferenceTrace', input: referenceTraceInput })).rejects.toMatchObject({ code: 'RESULT_RELEASED' }) + await expect(runCoordinatorTask(options, { type: 'readNumericTrace', input: { + id: job.id, expectedContentHash: summary.contentHash, configuration: 'candidate', + } })).rejects.toMatchObject({ code: 'RESULT_RELEASED' }) + expect(sqlite.prepare('SELECT heart_rate FROM vitals').all()).toEqual([{ heart_rate: 65 }]) + const reused = await publish({ ...request, idempotencyKey: 'reuse-retained' }) + expect(reused.contentHash).toBe(published.contentHash) + expect(reused.id).not.toBe(published.id) + datasets.release(reused.id) + const lostReply = join(directory, 'lost-reply') + writeFileSync(lostReply, `#!${process.execPath}\nconst {spawnSync}=require('node:child_process');const input=require('node:fs').readFileSync(0);const result=spawnSync(${JSON.stringify(python)},process.argv.slice(2),{input,env:process.env});process.exit(result.status===0?42:1);`, { mode: 0o700 }) + const recovered = await runCoordinatorTask({ ...options, rawLocation: { ...options.rawLocation, pythonPath: lostReply } }, + { type: 'createWaveformDataset', input: { ...request, idempotencyKey: 'lost-reply' } }, { admit: async () => true }) + expect(recovered.status).toBe('published') + if (recovered.status !== 'published') throw new Error('Lost committed response was not recovered') + expect(recovered.dataset.contentHash).toBe(published.contentHash) + datasets.release(recovered.dataset.id) + datasets.release(published.id) + expect(await publish()).toEqual({ ...published, available: false }) + const collected = JSON.parse(execFileSync(python, ['-B', '-c', ` +import json, sqlite3, sys +from common.biometrics.raw_artifacts import RawArtifactStore +connection=sqlite3.connect(sys.argv[1]) +print(json.dumps(RawArtifactStore(sys.argv[2]).collect_released(connection,now_ms=int(sys.argv[3]),minimum_age_ms=0))) +connection.close() +`, database, retained, String(now)], { env: environment, encoding: 'utf8', timeout: 20000 })) as string[] + expect(collected).toHaveLength(2) + expect(readdirSync(retained).filter(name => name.endsWith('.RAW.gz'))).toEqual([]) + expect(datasets.read(base.id)).toEqual(base) + datasets.release(base.id) + expect(await publish()).toEqual({ ...published, available: false }) + await expect(publish({ ...request, expectedContentHash: '0'.repeat(64) })).rejects.toMatchObject({ code: 'REVISION_CONFLICT' }) + } + finally { + sqlite.close() + rmSync(directory, { recursive: true, force: true }) + } +}, 30000) diff --git a/src/biometrics/backtest/tests/waveform-publication.test.ts b/src/biometrics/backtest/tests/waveform-publication.test.ts new file mode 100644 index 000000000..e493ebf0f --- /dev/null +++ b/src/biometrics/backtest/tests/waveform-publication.test.ts @@ -0,0 +1,174 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { mkdtempSync, realpathSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import * as schema from '@/src/db/biometrics-schema' +import { publishSelectedWaveformDataset } from '../waveform-publication' +import { runRawPublicationChild } from '../raw-publication-child' +import { podRawInspectionLocation } from '../raw-inspection-child' +import { prepareSelectedWaveformDataset, selectedWaveformRetry } from '../waveform-selection' +import type * as Selection from '../waveform-selection' + +vi.mock('../raw-publication-child', () => ({ runRawPublicationChild: vi.fn() })) +vi.mock('../waveform-selection', async original => ({ ...await original(), prepareSelectedWaveformDataset: vi.fn(), selectedWaveformRetry: vi.fn() })) + +type Dataset = NonNullable> +type Work = ReturnType +const hash = 'a'.repeat(64), id = '11111111-1111-4111-8111-111111111111' +const request = { parentDatasetId: id, expectedContentHash: hash, cohorts: [{ cohortId: 'night', sourceIds: [hash] }], idempotencyKey: 'waveform' } +const dataset = { id, contentHash: hash, available: true } as unknown as Dataset +const work = { prepared: { contentHash: hash }, selections: [] } as unknown as Work +const childResult = { type: 'result' as const, id, contentHash: hash, available: true } +let directory: string, sqlite: Database.Database, db: ReturnType> +let controller: AbortController +const publish = (options: Partial[2]> = {}, raw: unknown = request) => + publishSelectedWaveformDataset(db, raw as typeof request, { signal: controller.signal, admit: async () => true, ...options }) + +beforeEach(() => { + vi.clearAllMocks() + directory = realpathSync(mkdtempSync(join(tmpdir(), 'waveform-publication-'))) + sqlite = new Database(join(directory, 'biometrics.db')) + db = drizzle(sqlite, { schema }) + controller = new AbortController() + vi.mocked(selectedWaveformRetry).mockReturnValue(null) + vi.mocked(prepareSelectedWaveformDataset).mockReturnValue(work) + vi.mocked(runRawPublicationChild).mockResolvedValue(childResult) +}) +afterEach(() => { + vi.useRealTimers() + sqlite.close() + rmSync(directory, { recursive: true, force: true }) +}) + +it('rejects malformed selections before touching the database', async () => { + await expect(publish({}, { ...request, cohorts: [] })).rejects.toThrow() + expect(selectedWaveformRetry).not.toHaveBeenCalled() +}) + +it('returns an already published dataset without admission or publication', async () => { + vi.mocked(selectedWaveformRetry).mockReturnValue(dataset) + const admit = vi.fn(async () => true) + expect(await publish({ admit })).toEqual({ status: 'published', dataset }) + expect(admit).not.toHaveBeenCalled() + expect(prepareSelectedWaveformDataset).not.toHaveBeenCalled() +}) + +it('defers when cancelled or denied before any work is prepared', async () => { + controller.abort() + const admit = vi.fn(async () => true) + expect(await publish({ admit })).toEqual({ status: 'deferred' }) + expect(admit).not.toHaveBeenCalled() + controller = new AbortController() + expect(await publish({ admit: async () => false })).toEqual({ status: 'deferred' }) + const unavailable = async () => { + throw new Error('admission unavailable') + } + expect(await publish({ admit: unavailable })).toEqual({ status: 'deferred' }) + const abortingAdmit = async () => { + controller.abort() + return true + } + expect(await publish({ admit: abortingAdmit })).toEqual({ status: 'deferred' }) + expect(prepareSelectedWaveformDataset).not.toHaveBeenCalled() + expect(runRawPublicationChild).not.toHaveBeenCalled() +}) + +it('requires a persistent database file', async () => { + sqlite.close() + sqlite = new Database(':memory:') + db = drizzle(sqlite, { schema }) + await expect(publish()).rejects.toMatchObject({ code: 'WORKER_UNAVAILABLE' }) + expect(prepareSelectedWaveformDataset).not.toHaveBeenCalled() +}) + +it('defers after preparation when admission is withdrawn or the request is cancelled', async () => { + let checks = 0 + expect(await publish({ admit: async () => ++checks === 1 })).toEqual({ status: 'deferred' }) + expect(checks).toBe(2) + expect(prepareSelectedWaveformDataset).toHaveBeenCalledTimes(1) + checks = 0 + const abortOnSecondCheck = async () => { + if (++checks === 2) controller.abort() + return true + } + expect(await publish({ admit: abortOnSecondCheck })).toEqual({ status: 'deferred' }) + expect(runRawPublicationChild).not.toHaveBeenCalled() +}) + +it('publishes through the pod location by default and verifies the committed identity', async () => { + vi.mocked(selectedWaveformRetry).mockReturnValueOnce(null).mockReturnValueOnce(dataset) + expect(await publish()).toEqual({ status: 'published', dataset }) + expect(runRawPublicationChild).toHaveBeenCalledWith({ ...podRawInspectionLocation, databasePath: join(directory, 'biometrics.db'), + retainedDirectory: join(directory, 'biometric-raw') }, work, { signal: expect.any(AbortSignal) }) + const rawLocation = { pythonPath: '/custom/python', modulesPath: '/custom/modules', archiveDirectory: '/custom/archive' } + vi.mocked(selectedWaveformRetry).mockReturnValueOnce(null).mockReturnValueOnce(dataset) + await publish({ rawLocation }) + expect(vi.mocked(runRawPublicationChild).mock.calls[1][0]).toMatchObject(rawLocation) +}) + +it.each([ + ['no committed row', [null, null]], + ['a different id', [{ ...dataset, id: '22222222-2222-4222-8222-222222222222' }, null]], + ['a different content hash', [{ ...dataset, contentHash: 'b'.repeat(64) }, null]], +])('fails when the child response leaves %s', async (_label, retries) => { + vi.mocked(selectedWaveformRetry).mockReturnValueOnce(null) + for (const retry of retries) vi.mocked(selectedWaveformRetry).mockReturnValueOnce(retry as Dataset | null) + await expect(publish()).rejects.toThrow('Published waveform identity mismatch') +}) + +it('returns a dataset that committed even though the child response was lost', async () => { + vi.mocked(runRawPublicationChild).mockRejectedValue(new Error('IPC lost')) + vi.mocked(selectedWaveformRetry).mockReturnValueOnce(null).mockReturnValueOnce(dataset) + expect(await publish()).toEqual({ status: 'published', dataset }) +}) + +it('rethrows child failures that left nothing committed', async () => { + vi.mocked(runRawPublicationChild).mockRejectedValue(new Error('publication crashed')) + await expect(publish()).rejects.toThrow('publication crashed') +}) + +it('cancels the child when the request is aborted mid-flight', async () => { + vi.mocked(runRawPublicationChild).mockImplementation((_location, _work, options) => new Promise((_resolve, reject) => { + options?.signal?.addEventListener('abort', () => reject(new Error('Waveform publication cancelled')), { once: true }) + })) + const pending = publish() + await vi.waitFor(() => expect(runRawPublicationChild).toHaveBeenCalled()) + controller.abort() + await expect(pending).rejects.toThrow('cancelled') +}) + +it('withdraws admission while the child runs and reports deferral', async () => { + vi.useFakeTimers() + vi.mocked(runRawPublicationChild).mockImplementation((_location, _work, options) => new Promise((_resolve, reject) => { + options?.signal?.addEventListener('abort', () => reject(new Error('Waveform publication cancelled')), { once: true }) + })) + let release!: (allowed: boolean) => void + const admit = vi.fn<() => Promise>().mockResolvedValueOnce(true).mockResolvedValueOnce(true) + .mockImplementationOnce(() => new Promise((resolve) => { release = resolve })) + const pending = publish({ admit }) + await vi.advanceTimersByTimeAsync(5000) + expect(admit).toHaveBeenCalledTimes(3) + await vi.advanceTimersByTimeAsync(5000) + expect(admit).toHaveBeenCalledTimes(3) + release(false) + expect(await pending).toEqual({ status: 'deferred' }) + expect(controller.signal.aborted).toBe(false) +}) + +it('keeps polling admission while the child runs when it stays granted', async () => { + vi.useFakeTimers() + let finish!: () => void + vi.mocked(runRawPublicationChild).mockImplementation(() => new Promise((resolve) => { + finish = () => resolve(childResult) + })) + const admit = vi.fn(async () => true) + vi.mocked(selectedWaveformRetry).mockReturnValueOnce(null).mockReturnValueOnce(dataset) + const pending = publish({ admit }) + await vi.advanceTimersByTimeAsync(10000) + expect(admit).toHaveBeenCalledTimes(4) + finish() + expect(await pending).toEqual({ status: 'published', dataset }) +}) diff --git a/src/biometrics/backtest/tests/waveform-resources.test.ts b/src/biometrics/backtest/tests/waveform-resources.test.ts new file mode 100644 index 000000000..4c6b79854 --- /dev/null +++ b/src/biometrics/backtest/tests/waveform-resources.test.ts @@ -0,0 +1,20 @@ +// @vitest-environment node +import { expect, it } from 'vitest' +import { summarizeWaveformResources } from '../waveform-resources' +const resources = { version: 'waveform-process-resources-v1' as const, wallMs: 100, cpuMs: 80, peakRssBytes: 500, + scope: 'python_process_until_result' as const, memoryScope: 'process_lifetime_high_water_mark' as const } +it('sums sequential work without summing process memory peaks or claiming live health', () => { + expect(summarizeWaveformResources([{ run: { resources } }, { run: { resources: { ...resources, peakRssBytes: 700 } } }])).toMatchObject({ + expectedProcesses: 2, measuredProcesses: 2, totalWallMs: 200, totalCpuMs: 160, largestProcessPeakRssBytes: 700, ingestionHealthMeasured: false, + }) +}) +it('does not turn absent legacy measurements or missing kernel telemetry into zero', () => { + expect(summarizeWaveformResources([])).toMatchObject({ totalWallMs: null, totalCpuMs: null, largestProcessPeakRssBytes: null }) + expect(summarizeWaveformResources([{ run: {} }, { run: { resources } }])).toMatchObject({ measuredProcesses: 1, totalWallMs: null, totalCpuMs: null, largestProcessPeakRssBytes: null }) + expect(summarizeWaveformResources([{ run: { resources: { ...resources, peakRssBytes: null } } }])).toMatchObject({ totalWallMs: 100, totalCpuMs: 80, largestProcessPeakRssBytes: null }) +}) +it('rejects invalid measurement units and nonfinite counters', () => { + for (const bad of [{ wallMs: -1 }, { cpuMs: Infinity }, { peakRssBytes: 0 }, { peakRssBytes: 2 ** 53 }, { scope: 'whole_job' }]) { + expect(() => summarizeWaveformResources([{ run: { resources: { ...resources, ...bad } as typeof resources } }])).toThrow() + } +}) diff --git a/src/biometrics/backtest/tests/waveform-selection.test.ts b/src/biometrics/backtest/tests/waveform-selection.test.ts new file mode 100644 index 000000000..3c2db2201 --- /dev/null +++ b/src/biometrics/backtest/tests/waveform-selection.test.ts @@ -0,0 +1,155 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { afterEach, beforeEach, expect, it } from 'vitest' +import * as schema from '@/src/db/biometrics-schema' +import { DatasetStore } from '../datasets' +import { RawSourceCatalog } from '../raw-catalog' +import { createWaveformDatasetSchema, prepareSelectedWaveformDataset, recordedWaveformContexts, selectedWaveformRetry, waveformSelectionRequestHash } from '../waveform-selection' +import { datasetManifestSchema } from '../contracts' + +let sqlite: Database.Database +let db: ReturnType> +let parent: ReturnType +const start = 1700000000000 +let sourceIds: string[] +beforeEach(() => { + sqlite = new Database(':memory:') + db = drizzle(sqlite, { schema }) + migrate(db, { migrationsFolder: 'src/db/biometrics-migrations' }) + db.insert(schema.sleepRecords).values({ id: 1, side: 'left', enteredBedAt: new Date(start), leftBedAt: new Date(start + 600000), + sleepDurationSeconds: 600, timesExitedBed: 1, presentIntervals: [[start / 1000, start / 1000 + 60]], + notPresentIntervals: [[start / 1000 + 60, start / 1000 + 120]] }).run() + const store = new DatasetStore(db) + parent = store.read(store.create({ side: 'left', subjectId: 'test', sessionIds: [1], referenceImportIds: [], timezone: 'UTC', idempotencyKey: 'parent' }).id) + sourceIds = [0, 1].map((index) => { + const sha256 = String(index + 1).repeat(64) + return new RawSourceCatalog(db).record({ source_name: `${index}.RAW.gz`, retained: false, + artifact: { name: `${sha256}.RAW.gz`, sha256, compressed_bytes: 100, expanded_bytes: 200 }, + inventory: { version: 'retained-raw-inventory-v1', artifact_sha256: sha256, decoded_records: 2, piezo_records: 1, pump_records: 1, + usable_dual_frames: 1, empty_channel_frames: 0, oversize_channel_frames: 0, invalid_timestamps: 0, timestamp_reversals: 0, + duplicate_timestamps: 0, first_event_ms: start - 300000 + index * 450000, last_event_ms: start + 149999 + index * 450000, + channels: { left1: { samples: 10, lost_markers: 0, truncated_tail_bytes: 0 }, right1: { samples: 10, lost_markers: 0, truncated_tail_bytes: 0 } }, + ordered_waveform_available: true, limitations: [] } }) + }) +}) +afterEach(() => sqlite.close()) +const request = () => ({ parentDatasetId: parent.id, expectedContentHash: parent.contentHash, idempotencyKey: 'waveform', + cohorts: [{ cohortId: datasetManifestSchema.parse(parent.manifest).splitProtocol.assignments[0].cohortId, sourceIds: [...sourceIds].reverse() }] }) + +it('resolves ordered immutable source identities and recorded occupancy without reading current state', () => { + sqlite.prepare('UPDATE sleep_records SET present_intervals = NULL, not_present_intervals = NULL').run() + const result = prepareSelectedWaveformDataset(db, request(), 123) + expect(result.selections.map(item => item.source_name)).toEqual(['0.RAW.gz', '1.RAW.gz']) + if (result.prepared.snapshot.version !== 'waveform-dataset-v1') throw new Error('Wrong snapshot') + expect(result.prepared.snapshot.waveformInputs[0].input).toMatchObject({ retained_start_ms: start - 300000, + score_start_ms: start, score_end_ms: start + 600000, contexts: [ + { start_ms: start, end_ms: start + 60000, left_occupied: true, right_occupied: null, active_sides: null, pump_context_known: false }, + { start_ms: start + 60000, end_ms: start + 120000, left_occupied: false, right_occupied: null, active_sides: null, pump_context_known: false }, + ] }) + const retry = request() + retry.cohorts[0].sourceIds.reverse() + expect(prepareSelectedWaveformDataset(db, retry, 456).prepared.contentHash).toBe(result.prepared.contentHash) + expect(sqlite.prepare('SELECT count(*) n FROM biometric_datasets').get()).toEqual({ n: 1 }) + expect(sqlite.prepare('SELECT count(*) n FROM biometric_raw_artifacts').get()).toEqual({ n: 0 }) +}) + +it('rejects stale, duplicate, incomplete and forged selections', () => { + expect(() => prepareSelectedWaveformDataset(db, { ...request(), expectedContentHash: '0'.repeat(64) })).toThrow('expected immutable') + expect(() => prepareSelectedWaveformDataset(db, { ...request(), cohorts: [...request().cohorts, ...request().cohorts] })).toThrow('each whole-night') + const duplicate = request() + duplicate.cohorts[0].sourceIds = [sourceIds[0], sourceIds[0]] + expect(() => prepareSelectedWaveformDataset(db, duplicate)).toThrow('each RAW source') + const missing = request() + missing.cohorts[0].cohortId = 'missing' + expect(() => prepareSelectedWaveformDataset(db, missing)).toThrow('Missing whole-night') + expect(() => createWaveformDatasetSchema.parse({ ...request(), archiveDirectory: '/tmp' })).toThrow() + new RawSourceCatalog(db).markUnavailable(sourceIds[0]) + expect(() => prepareSelectedWaveformDataset(db, request())).toThrow('unavailable') +}) + +it('retains unknown context for gaps, malformed evidence and contradictory intervals', () => { + const snapshot = structuredClone(parent.snapshot) + snapshot.sessions[0].presentIntervals = [[start / 1000 - 10, start / 1000 + 60], ['bad', 10], [Infinity, 50], [10, 0]] + snapshot.sessions[0].notPresentIntervals = [[start / 1000 + 30, start / 1000 + 90]] + expect(recordedWaveformContexts(snapshot, start - 300000, start + 600000)).toEqual([ + { start_ms: start, end_ms: start + 30000, left_occupied: true, right_occupied: null, active_sides: null, pump_context_known: false }, + { start_ms: start + 60000, end_ms: start + 90000, left_occupied: false, right_occupied: null, active_sides: null, pump_context_known: false }, + ]) + snapshot.side = 'right' + expect(recordedWaveformContexts(snapshot, start + 10000, start + 20000)).toEqual([ + { start_ms: start + 10000, end_ms: start + 20000, right_occupied: true, left_occupied: null, active_sides: null, pump_context_known: false }, + ]) + snapshot.sessions[0].presentIntervals = null + snapshot.sessions[0].notPresentIntervals = { unrecognized: true } + expect(recordedWaveformContexts(snapshot, start, start + 600000)).toEqual([]) +}) + +it('merges consistent overlapping intervals, preserves fractional seconds and bounds recorded evidence', () => { + const snapshot = structuredClone(parent.snapshot) + snapshot.sessions[0].presentIntervals = [[start / 1000 + 0.125, start / 1000 + 40], [start / 1000 + 30, start / 1000 + 60]] + snapshot.sessions[0].notPresentIntervals = null + expect(recordedWaveformContexts(snapshot, start, start + 600000)).toEqual([ + { start_ms: start + 125, end_ms: start + 60000, left_occupied: true, right_occupied: null, active_sides: null, pump_context_known: false }, + ]) + snapshot.sessions[0].presentIntervals = Array(20001).fill([1, 2]) + expect(() => recordedWaveformContexts(snapshot, start, start + 600000)).toThrow('budget') +}) + +const recordSource = (source_name: string, sha256: string, first_event_ms: number, last_event_ms: number) => new RawSourceCatalog(db).record({ source_name, retained: false, + artifact: { name: `${sha256}.RAW.gz`, sha256, compressed_bytes: 100, expanded_bytes: 200 }, + inventory: { version: 'retained-raw-inventory-v1', artifact_sha256: sha256, decoded_records: 2, piezo_records: 1, pump_records: 1, + usable_dual_frames: 1, empty_channel_frames: 0, oversize_channel_frames: 0, invalid_timestamps: 0, timestamp_reversals: 0, + duplicate_timestamps: 0, first_event_ms, last_event_ms, + channels: { left1: { samples: 10, lost_markers: 0, truncated_tail_bytes: 0 }, right1: { samples: 10, lost_markers: 0, truncated_tail_bytes: 0 } }, + ordered_waveform_available: true, limitations: [] } }) + +it('hashes a selection independently of cohort and source order', () => { + const base = { parentDatasetId: parent.id, expectedContentHash: parent.contentHash, idempotencyKey: 'order' } + const a = { cohortId: 'a', sourceIds: [sourceIds[1], sourceIds[0]] }, b = { cohortId: 'b', sourceIds: [sourceIds[0]] } + expect(waveformSelectionRequestHash({ ...base, cohorts: [b, a] })).toBe(waveformSelectionRequestHash({ ...base, cohorts: [{ ...a, sourceIds: [sourceIds[0], sourceIds[1]] }, b] })) + expect(waveformSelectionRequestHash({ ...base, cohorts: [a, a] })).toBe(waveformSelectionRequestHash({ ...base, cohorts: [a, a] })) + expect(waveformSelectionRequestHash({ ...base, cohorts: [a] })).not.toBe(waveformSelectionRequestHash({ ...base, cohorts: [b] })) +}) + +it('replays an identical published selection by idempotency key and rejects a reused key', () => { + expect(selectedWaveformRetry(db, request())).toBeNull() + const { prepared } = prepareSelectedWaveformDataset(db, request(), 123) + db.insert(schema.biometricDatasets).values({ ...prepared, byteLength: Buffer.byteLength(JSON.stringify(prepared.manifest)) + Buffer.byteLength(JSON.stringify(prepared.snapshot)) }).run() + expect(selectedWaveformRetry(db, request())).toMatchObject({ id: prepared.id, contentHash: prepared.contentHash }) + const changed = request() + changed.cohorts[0].sourceIds = [sourceIds[0]] + expect(() => selectedWaveformRetry(db, changed)).toThrow('already used') +}) + +it('reuses a retained artifact for an unavailable source only when its recorded descriptor matches', () => { + new RawSourceCatalog(db).markUnavailable(sourceIds[0]) + db.insert(schema.biometricRawArtifacts).values({ contentHash: '1'.repeat(64), name: `${'1'.repeat(64)}.RAW.gz`, compressedBytes: 100, expandedBytes: 999, createdAtMs: 1 }).run() + expect(() => prepareSelectedWaveformDataset(db, request())).toThrow('unavailable') + sqlite.prepare('UPDATE biometric_raw_artifacts SET expanded_bytes = 200').run() + expect(prepareSelectedWaveformDataset(db, request()).selections.map(item => item.source_name)).toEqual(['0.RAW.gz', '1.RAW.gz']) +}) + +it('rejects sources whose event bounds tie or overlap inside one cohort', () => { + const tie = recordSource('tie.RAW.gz', '4'.repeat(64), start - 300000, start + 149999) + const selection = request() + selection.cohorts[0].sourceIds = [sourceIds[0], tie] + expect(() => prepareSelectedWaveformDataset(db, selection)).toThrow('distinct') +}) + +it('requires a selection for every cohort and attributes a shared artifact to its earliest source name', () => { + db.insert(schema.sleepRecords).values({ id: 2, side: 'left', enteredBedAt: new Date(start + 86400000), leftBedAt: new Date(start + 86400000 + 600000), sleepDurationSeconds: 600, timesExitedBed: 0 }).run() + const store = new DatasetStore(db) + const twoNights = store.read(store.create({ side: 'left', subjectId: 'test', sessionIds: [1, 2], referenceImportIds: [], timezone: 'UTC', idempotencyKey: 'two-nights' }).id) + const cohorts = datasetManifestSchema.parse(twoNights.manifest).splitProtocol.assignments + expect(cohorts).toHaveLength(2) + const sha256 = '3'.repeat(64) + const [later, earlier] = ['z.RAW.gz', 'b.RAW.gz'].map(name => recordSource(name, sha256, start - 300000, start + 86400000 + 599999)) + const base = { parentDatasetId: twoNights.id, expectedContentHash: twoNights.contentHash, idempotencyKey: 'shared' } + expect(() => prepareSelectedWaveformDataset(db, { ...base, cohorts: [{ cohortId: cohorts[0].cohortId, sourceIds: [later] }] })).toThrow('every whole-night cohort') + for (const order of [[later, earlier], [earlier, later]]) { + const result = prepareSelectedWaveformDataset(db, { ...base, cohorts: cohorts.map((cohort, index) => ({ cohortId: cohort.cohortId, sourceIds: [order[index]] })) }) + expect(result.selections.map(item => item.source_name)).toEqual(['b.RAW.gz']) + } +}) diff --git a/src/biometrics/backtest/tests/worker-entry.test.ts b/src/biometrics/backtest/tests/worker-entry.test.ts new file mode 100644 index 000000000..88d59bdf3 --- /dev/null +++ b/src/biometrics/backtest/tests/worker-entry.test.ts @@ -0,0 +1,82 @@ +// @vitest-environment node +import { afterEach, expect, it, vi } from 'vitest' +import { backtestFixture } from './fixtures/backtest' + +type Listener = (chunk: Buffer) => void + +/** Drive the stdin protocol in-process so the entry module's own lines are instrumented. */ +async function runWorker(chunks: Buffer[]) { + vi.resetModules() + const listeners = new Map() + const stdout: string[] = [] + const stderr: string[] = [] + const stdin = vi.spyOn(process.stdin, 'on').mockImplementation(((event: string, listener: Listener) => { + listeners.set(event, listener) + return process.stdin + }) as typeof process.stdin.on) + const out = vi.spyOn(process.stdout, 'write').mockImplementation(((chunk: string | Uint8Array) => { + stdout.push(String(chunk)) + return true + }) as typeof process.stdout.write) + const err = vi.spyOn(process.stderr, 'write').mockImplementation(((chunk: string | Uint8Array) => { + stderr.push(String(chunk)) + return true + }) as typeof process.stderr.write) + const exit = vi.spyOn(process, 'exit').mockImplementation(((code?: number) => { + throw new Error(`exit ${code}`) + }) as typeof process.exit) + const previousExitCode = process.exitCode + process.exitCode = undefined + try { + await import('../worker-entry') + const data = listeners.get('data'), end = listeners.get('end') + if (!data || !end) throw new Error('Worker did not subscribe to stdin') + let failure: unknown + try { + for (const chunk of chunks) data(chunk) + end(Buffer.alloc(0)) + } + catch (error) { failure = error } + return { stdout, stderr, failure, exitCode: process.exitCode } + } + finally { + process.exitCode = previousExitCode + stdin.mockRestore() + out.mockRestore() + err.mockRestore() + exit.mockRestore() + } +} +afterEach(() => vi.restoreAllMocks()) + +it('streams progress then a single result line with operational measurements', async () => { + const serialized = JSON.stringify(backtestFixture()) + const half = Math.floor(serialized.length / 2) + const run = await runWorker([Buffer.from(serialized.slice(0, half)), Buffer.from(serialized.slice(half))]) + expect(run.failure).toBeUndefined() + expect(run.stderr).toEqual([]) + expect(run.exitCode).toBeUndefined() + const events = run.stdout.map(line => JSON.parse(line) as { type: string, report?: { version: string, operational: { runtimeMs: number, peakRssKiB: number, heapLimitMiB: number } } }) + expect(events.map(event => event.type)).toEqual(['progress', 'result']) + expect(events[0]).toEqual({ type: 'progress', completedNights: 1, totalNights: 1 }) + expect(events[1].report).toMatchObject({ version: 'processed-backtest-report-v1', operational: { heapLimitMiB: 128 } }) + expect(events[1].report?.operational.runtimeMs).toBeGreaterThanOrEqual(0) + expect(events[1].report?.operational.peakRssKiB).toBeGreaterThan(0) +}) + +it('reports invalid input without echoing it and exits nonzero', async () => { + const invalid = await runWorker([Buffer.from('{"job":')]) + expect(invalid.stdout).toEqual([]) + expect(invalid.stderr).toEqual(['Replay failed input validation or evaluation\n']) + expect(invalid.exitCode).toBe(1) + const rejected = await runWorker([Buffer.from(JSON.stringify({ job: {}, dataset: {} }))]) + expect(rejected.stderr).toEqual(['Replay failed input validation or evaluation\n']) + expect(rejected.exitCode).toBe(1) +}) + +it('aborts once input exceeds the worker byte limit before parsing', async () => { + const run = await runWorker([Buffer.alloc(6 * 1024 * 1024), Buffer.alloc(5 * 1024 * 1024)]) + expect(run.failure).toEqual(new Error('exit 2')) + expect(run.stderr).toEqual(['Replay input exceeds the worker byte limit\n']) + expect(run.stdout).toEqual([]) +}) diff --git a/src/biometrics/backtest/trace-contracts.ts b/src/biometrics/backtest/trace-contracts.ts new file mode 100644 index 000000000..06fd06bc7 --- /dev/null +++ b/src/biometrics/backtest/trace-contracts.ts @@ -0,0 +1,12 @@ +import { z } from 'zod' +export const traceRequestSchema = z.object({ id: z.string().uuid(), expectedContentHash: z.string().regex(/^[a-f0-9]{64}$/), + configuration: z.enum(['default', 'current', 'candidate']), cohortId: z.string().min(1).max(200).optional(), + offset: z.number().int().min(0).max(1000000).default(0), limit: z.number().int().min(1).max(500).default(200), +}).strict() +export const tracePageSchema = z.object({ jobId: z.string().uuid(), contentHash: z.string(), configuration: z.enum(['default', 'current', 'candidate']), + provenance: z.literal('versioned_stage_replay'), timestampBasis: z.literal('original_trace_before_reference_clock_offset'), clockOffsetMs: z.number(), + totalItems: z.number().int(), offset: z.number().int(), limit: z.number().int(), downsampled: z.literal(false), + items: z.array(z.object({ cohortId: z.string(), configurationHash: z.string(), revisionId: z.string().uuid().nullable(), + windowStartMs: z.number(), windowEndMs: z.number(), startMs: z.number(), endMs: z.number(), stage: z.enum(['wake', 'light', 'deep', 'rem']) })), + nextCursor: z.object({ offset: z.number().int(), contentHash: z.string() }).nullable(), +}) diff --git a/src/biometrics/backtest/trace-page.ts b/src/biometrics/backtest/trace-page.ts new file mode 100644 index 000000000..ba8825b7b --- /dev/null +++ b/src/biometrics/backtest/trace-page.ts @@ -0,0 +1,47 @@ +import { z } from 'zod' +import { ReferenceInputError } from '../references/time' +import { traceRequestSchema, tracePageSchema } from './trace-contracts' + +const reportSchema = z.object({ version: z.enum(['processed-backtest-report-v1', 'waveform-backtest-report-v1']), + protocol: z.object({ mode: z.enum(['comparison', 'stage_replay', 'signal_replay']), waveformStages: z.boolean().optional(), clockOffsetMs: z.number().int() }), + nights: z.array(z.object({ cohortId: z.string(), evaluations: z.array(z.object({ label: z.enum(['default', 'current', 'candidate']), + configHash: z.string(), revisionId: z.string().uuid().nullable(), windows: z.array(z.object({ startMs: z.number().int(), endMs: z.number().int(), + blocks: z.array(z.object({ start: z.number().int(), end: z.number().int(), stage: z.enum(['wake', 'light', 'deep', 'rem']) })), + })), + })).max(3) })).max(60), +}) + +/** Selection changes only returned detail, never the retained evaluation or + * its scoring. Offset cursors are pinned to an immutable verified report hash. */ +export function pageStageTraces(report: unknown, raw: z.input) { + const input = traceRequestSchema.parse(raw) + const parsed = reportSchema.safeParse(report) + if (!parsed.success) throw new ReferenceInputError('RESULT_CORRUPT', 'Report trace format is unavailable') + const data = parsed.data + if (data.protocol.mode !== 'stage_replay' && !(data.protocol.mode === 'signal_replay' && data.protocol.waveformStages)) throw new ReferenceInputError('INCOMPATIBLE_METRIC', 'This comparison did not produce stage replay traces') + const nights = input.cohortId === undefined ? data.nights : data.nights.filter(night => night.cohortId === input.cohortId) + if (!nights.length || !nights.some(night => night.evaluations.some(evaluation => evaluation.label === input.configuration))) { + throw new ReferenceInputError('NOT_FOUND', 'Requested cohort or configuration was not evaluated') + } + const items: z.infer['items'] = [] + let totalItems = 0 + for (const night of nights) { + for (const evaluation of night.evaluations.filter(row => row.label === input.configuration)) { + for (const window of evaluation.windows) { + for (const block of window.blocks) { + if (totalItems >= input.offset && items.length < input.limit) items.push({ cohortId: night.cohortId, configurationHash: evaluation.configHash, + revisionId: evaluation.revisionId, windowStartMs: window.startMs, windowEndMs: window.endMs, + startMs: block.start, endMs: block.end, stage: block.stage }) + totalItems++ + } + } + } + } + if (input.offset > totalItems) throw new ReferenceInputError('INVALID_CURSOR', 'Trace cursor exceeds the selected report') + const next = input.offset + items.length + return tracePageSchema.parse({ jobId: input.id, contentHash: input.expectedContentHash, configuration: input.configuration, + provenance: 'versioned_stage_replay', timestampBasis: 'original_trace_before_reference_clock_offset', clockOffsetMs: data.protocol.clockOffsetMs, + items, totalItems, offset: input.offset, limit: input.limit, downsampled: false, + nextCursor: next < totalItems ? { offset: next, contentHash: input.expectedContentHash } : null }) +} + diff --git a/src/biometrics/backtest/traces.ts b/src/biometrics/backtest/traces.ts new file mode 100644 index 000000000..0b5be7b0f --- /dev/null +++ b/src/biometrics/backtest/traces.ts @@ -0,0 +1,40 @@ +import { DatasetStore } from './datasets' +import { pageReferenceTraces, referenceTraceIdentity, referenceTraceRequestSchema } from './reference-trace-page' +import { pageNumericTraces } from './numeric-trace-page' +import type { z } from 'zod' +import { pageStageTraces } from './trace-page' +export { pageStageTraces } from './trace-page' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import type * as schema from '@/src/db/biometrics-schema' +import { ReferenceInputError } from '../references/time' +import { BacktestJobStore } from './jobs' +import { traceRequestSchema } from './trace-contracts' + +export function readStageTraces(db: BetterSQLite3Database, raw: z.input) { + const input = traceRequestSchema.parse(raw) + const result = new BacktestJobStore(db).result(input.id) + if (!result) throw new ReferenceInputError('INVALID_STATE', 'Wait for a successful replay before reading traces') + if (result.contentHash !== input.expectedContentHash) throw new ReferenceInputError('REVISION_CONFLICT', 'Trace report hash changed') + if (!result.available) throw new ReferenceInputError('RESULT_RELEASED', 'Replay report was released') + return pageStageTraces(result.report, input) +} + +export function readNumericTraces(db: BetterSQLite3Database, raw: z.input) { + const input = traceRequestSchema.parse(raw) + const result = new BacktestJobStore(db).result(input.id) + if (!result) throw new ReferenceInputError('INVALID_STATE', 'Wait for a successful replay before reading traces') + if (result.contentHash !== input.expectedContentHash) throw new ReferenceInputError('REVISION_CONFLICT', 'Trace report hash changed') + if (!result.available) throw new ReferenceInputError('RESULT_RELEASED', 'Replay report was released') + return pageNumericTraces(result.report, input) +} + +export function readReferenceTraces(db: BetterSQLite3Database, raw: z.input) { + const input = referenceTraceRequestSchema.parse(raw) + const result = new BacktestJobStore(db).result(input.id) + if (!result) throw new ReferenceInputError('INVALID_STATE', 'Wait for a successful replay before reading references') + if (result.contentHash !== input.expectedContentHash) throw new ReferenceInputError('REVISION_CONFLICT', 'Reference report hash changed') + if (!result.available) throw new ReferenceInputError('RESULT_RELEASED', 'Replay report was released') + const identity = referenceTraceIdentity(result.report) + const dataset = new DatasetStore(db).read(identity.datasetId) + return pageReferenceTraces(result.report, dataset, input) +} diff --git a/src/biometrics/backtest/uncertainty-core.ts b/src/biometrics/backtest/uncertainty-core.ts new file mode 100644 index 000000000..122b8b148 --- /dev/null +++ b/src/biometrics/backtest/uncertainty-core.ts @@ -0,0 +1,57 @@ +import { nightBootstrapSchema, type NightBootstrapPolicy } from './uncertainty-policy' +export type { NightBootstrapPolicy } from './uncertainty-policy' + +/** Versioned, platform-independent SHA-256 counter stream. Rejection sampling + * avoids modulo bias. This seed is for reproducibility, not security. */ +function indexStream(seed: string, size: number, digest: (text: string) => Uint8Array) { + let counter = 0, position = 32, bytes: Uint8Array = new Uint8Array(32) + const ceiling = 4294967296 - (4294967296 % size) + return () => { + for (;;) { + if (position === 32) { + bytes = digest(JSON.stringify(['paired-night-percentile-v1', seed, counter++])) + position = 0 + } + const value = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength).getUint32(position, false) + position += 4 + if (value < ceiling) return value % size + } + } +} +const average = (values: number[]) => values.reduce((sum, value) => sum + value, 0) / values.length +function quantile(sorted: number[], probability: number) { + const position = (sorted.length - 1) * probability + const lower = Math.floor(position), fraction = position - lower + return sorted[lower] + fraction * (sorted[Math.min(lower + 1, sorted.length - 1)] - sorted[lower]) +} + +/** Resample paired whole-night differences, never correlated epochs. An + * exploratory interval alone cannot award improvement or validate activation. */ +export function createNightUncertainty(digest: (text: string) => Uint8Array, statistic: 'equal_night_mean_common_agreement_difference' | 'equal_night_mean_common_mae_difference' = 'equal_night_mean_common_agreement_difference') { + return function nightUncertainty(input: Array<{ cohortId: string, difference: number | null }>, rawPolicy?: NightBootstrapPolicy) { + if (input.length > 60 || new Set(input.map(row => row.cohortId)).size !== input.length + || input.some(row => row.difference !== null && (!Number.isFinite(row.difference) || Math.abs(row.difference) > (statistic === 'equal_night_mean_common_agreement_difference' ? 1 : 2 * Number.MAX_SAFE_INTEGER)))) throw new Error('Invalid paired-night uncertainty input') + const policy = rawPolicy === undefined ? null : nightBootstrapSchema.parse(rawPolicy) + const ordered = [...input].sort((a, b) => a.cohortId < b.cohortId ? -1 : a.cohortId > b.cohortId ? 1 : 0) + const values = ordered.flatMap(row => row.difference === null ? [] : [row.difference]) + const metadata = { policy, confidenceLevel: 0.95, samplingUnit: 'whole_night_cohort' as const, + statistic, scoredNights: values.length, omittedNights: input.length - values.length, + estimate: values.length ? average(values) : null, + limitations: ['Exploratory interval assumes exchangeable nights; serial dependence may understate uncertainty.', + 'The requested minimum night count is a reporting threshold, not proof of sufficient evidence.', + 'Missing-reference nights are omitted and remain counted; the interval does not establish whole-history accuracy.', + 'This interval is not a promotion decision or clinical validation.'] } + if (!policy) return { ...metadata, status: 'not_requested' as const, interval: null } + if (values.length < policy.minimumNights) return { ...metadata, status: 'insufficient_nights' as const, interval: null } + if (values.every(value => value === values[0])) return { ...metadata, status: 'degenerate' as const, interval: null } + const index = indexStream(policy.seed, values.length, digest) + const samples: number[] = [] + for (let replicate = 0; replicate < policy.resamples; replicate++) { + let sum = 0 + for (let draw = 0; draw < values.length; draw++) sum += values[index()] + samples.push(sum / values.length) + } + samples.sort((a, b) => a - b) + return { ...metadata, status: 'estimated' as const, interval: { lower: quantile(samples, 0.025), upper: quantile(samples, 0.975) } } + } +} diff --git a/src/biometrics/backtest/uncertainty-policy.ts b/src/biometrics/backtest/uncertainty-policy.ts new file mode 100644 index 000000000..99102e78e --- /dev/null +++ b/src/biometrics/backtest/uncertainty-policy.ts @@ -0,0 +1,8 @@ +import { z } from 'zod' + +export const nightBootstrapSchema = z.object({ + version: z.literal('paired-night-percentile-v1'), seed: z.string().min(1).max(64), + resamples: z.number().int().min(200).max(5000), minimumNights: z.number().int().min(3).max(60), +}).strict() +export type NightBootstrapPolicy = z.infer + diff --git a/src/biometrics/backtest/uncertainty.ts b/src/biometrics/backtest/uncertainty.ts new file mode 100644 index 000000000..48182d826 --- /dev/null +++ b/src/biometrics/backtest/uncertainty.ts @@ -0,0 +1,4 @@ +import { createHash } from 'node:crypto' +import { createNightUncertainty } from './uncertainty-core' +export type { NightBootstrapPolicy } from './uncertainty-policy' +export const nightUncertainty = createNightUncertainty(text => createHash('sha256').update(text).digest()) diff --git a/src/biometrics/backtest/waveform-build.ts b/src/biometrics/backtest/waveform-build.ts new file mode 100644 index 000000000..d64852937 --- /dev/null +++ b/src/biometrics/backtest/waveform-build.ts @@ -0,0 +1,44 @@ +import { execFile } from 'node:child_process' +import { tmpdir } from 'node:os' +import { promisify } from 'node:util' +import { z } from 'zod' +import { configHash } from '../algorithms/canonical' +import { workerBuildHash } from './child' +import type { WaveformWorkerLocation } from './waveform-child' + +const hash = z.string().regex(/^[a-f0-9]{64}$/) +const label = z.string().min(1).max(1000) +const dependency = z.object({ version: label, record_sha256: hash, wheel_sha256: hash }).strict() +export const waveformPythonBuildSchema = z.object({ hash, identity: z.object({ + version: z.literal('waveform-python-build-v1'), + sources: z.record(z.string().regex(/^(common|piezo-processor)\/[A-Za-z0-9_./-]+$/), hash) + .refine(value => Object.keys(value).length > 0 && Object.keys(value).length <= 64), + dependencies: z.object({ numpy: dependency, scipy: dependency }).strict(), + runtime: z.object({ implementation: label, version: label, cache_tag: label.nullable(), soabi: label.nullable(), + machine: label, system: label, byteorder: z.enum(['little', 'big']) }).strict(), +}).strict() }).strict() + +/** Read-only bounded probe of the exact interpreter used for replay. Locations + * are trusted deployment configuration; no command or source comes from RPC. */ +export async function probeWaveformBuild(location: Pick, signal?: AbortSignal) { + const { stdout } = await promisify(execFile)(location.pythonPath, ['-B', '-m', 'common.biometrics.replay_worker', '--describe-build'], { + cwd: tmpdir(), env: { NODE_ENV: 'production', PYTHONPATH: location.modulesPath, LANG: 'C.UTF-8', TZ: 'UTC', + OPENBLAS_NUM_THREADS: '1', OMP_NUM_THREADS: '1', MKL_NUM_THREADS: '1', NUMEXPR_NUM_THREADS: '1' }, + encoding: 'utf8', maxBuffer: 65536, timeout: 10000, killSignal: 'SIGKILL', signal, + }).catch(() => { throw new Error('Waveform build identity is unavailable') }) + const result = waveformPythonBuildSchema.parse(JSON.parse(stdout)) + if (configHash(result.identity) !== result.hash) throw new Error('Waveform build identity integrity mismatch') + return result +} + +/** Pin orchestration, scoring, Python code and installed numerical wheel + * metadata together. Architecture/runtime changes deliberately invalidate it. */ +export async function waveformEvaluatorBuildIdentity(coordinatorPath: string, workerPath: string, + location: Pick, signal?: AbortSignal) { + const [coordinatorHash, evaluatorHash, python] = await Promise.all([ + workerBuildHash(coordinatorPath), workerBuildHash(workerPath), probeWaveformBuild(location, signal), + ]) + const identity = { version: 'waveform-evaluator-build-v1', coordinatorHash, evaluatorHash, pythonHash: python.hash, + node: { version: process.version, platform: process.platform, arch: process.arch, icu: process.versions.icu ?? null } } + return { hash: configHash(identity), identity, python } +} diff --git a/src/biometrics/backtest/waveform-child.ts b/src/biometrics/backtest/waveform-child.ts new file mode 100644 index 000000000..d0c901783 --- /dev/null +++ b/src/biometrics/backtest/waveform-child.ts @@ -0,0 +1,99 @@ +import { spawn, type ChildProcessWithoutNullStreams } from 'node:child_process' +import { tmpdir } from 'node:os' +import { StringDecoder } from 'node:string_decoder' +import { z } from 'zod' + +import { waveformReadingSchema, waveformRunSchema } from './waveform-output' +export { waveformReadingSchema, waveformRunSchema } from './waveform-output' +const hash = z.string().regex(/^[a-f0-9]{64}$/) +const eventSchema = z.discriminatedUnion('type', [ + z.object({ type: z.literal('reading'), reading: waveformReadingSchema }).strict(), + z.object({ type: z.literal('result'), result: waveformRunSchema }).strict(), +]) +export interface WaveformWorkerLocation { + pythonPath: string + modulesPath: string + retainedDirectory: string +} + +/** Locations are trusted server configuration, never fields accepted by RPC. */ +export function runWaveformChild(location: WaveformWorkerLocation, work: unknown, options: { + signal?: AbortSignal + timeoutMs?: number + expectedBuildHash?: string + cpuSeconds?: number +} = {}): Promise<{ readings: z.infer[], run: z.infer }> { + if (options.cpuSeconds !== undefined && (!Number.isSafeInteger(options.cpuSeconds) || options.cpuSeconds < 1 || options.cpuSeconds > 3600)) return Promise.reject(new Error('Invalid waveform CPU budget')) + if (options.expectedBuildHash !== undefined) hash.parse(options.expectedBuildHash) + const serialized = JSON.stringify(options.expectedBuildHash === undefined ? work : { ...z.record(z.string(), z.unknown()).parse(work), expected_build_hash: options.expectedBuildHash }) + if (!serialized || Buffer.byteLength(serialized) > 8 * 1024 * 1024) return Promise.reject(new Error('Waveform input exceeds 8 MiB')) + const timeoutMs = options.timeoutMs ?? 120000 + if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 10 || timeoutMs > 3600000) return Promise.reject(new Error('Invalid waveform time budget')) + if (options.signal?.aborted) return Promise.reject(new Error('Waveform replay cancelled')) + return new Promise((resolve, reject) => { + const child: ChildProcessWithoutNullStreams = spawn(location.pythonPath, ['-B', '-m', 'common.biometrics.replay_worker', '--retained-directory', location.retainedDirectory, ...(options.cpuSeconds === undefined ? [] : ['--cpu-seconds', String(options.cpuSeconds)])], { + cwd: tmpdir(), env: { NODE_ENV: 'production', PYTHONPATH: location.modulesPath, LANG: 'C.UTF-8', TZ: 'UTC', + OPENBLAS_NUM_THREADS: '1', OMP_NUM_THREADS: '1', MKL_NUM_THREADS: '1', NUMEXPR_NUM_THREADS: '1' }, + stdio: ['pipe', 'pipe', 'pipe'], + }) + const readings: z.infer[] = [] + let result: z.infer | undefined + const decoder = new StringDecoder('utf8') + let pending = '', outputBytes = 0, diagnosticBytes = 0 + let failure: Error | undefined, killTimer: ReturnType | undefined + const stop = (error: Error) => { + failure ??= error + child.kill('SIGTERM') + killTimer ??= setTimeout(() => child.kill('SIGKILL'), 500) + } + const timeout = setTimeout(() => stop(new Error('Waveform replay exceeded its time budget')), timeoutMs) + const abort = () => stop(new Error('Waveform replay cancelled')) + options.signal?.addEventListener('abort', abort, { once: true }) + if (options.signal?.aborted) abort() + child.on('error', () => stop(new Error('Waveform worker could not start'))) + child.stdin.on('error', () => stop(new Error('Waveform worker input pipe closed'))) + child.stderr.on('data', (chunk: Buffer) => { + diagnosticBytes += chunk.length + if (diagnosticBytes > 16384) stop(new Error('Waveform diagnostic output exceeded its bound')) + }) + child.stdout.on('data', (chunk: Buffer) => { + if (failure) return + outputBytes += chunk.length + if (outputBytes > 16 * 1024 * 1024) return stop(new Error('Waveform output exceeds 16 MiB')) + pending += decoder.write(chunk) + let newline: number + while ((newline = pending.indexOf('\n')) >= 0) { + const line = pending.slice(0, newline) + pending = pending.slice(newline + 1) + try { + if (result || Buffer.byteLength(line) > 65536) throw new Error('Invalid event order or length') + const event = eventSchema.parse(JSON.parse(line)) + if (event.type === 'reading') { + if (readings.length >= 20000) throw new Error('Too many waveform readings') + readings.push(event.reading) + } + else result = event.result + } + catch { + stop(new Error('Waveform worker emitted an invalid event')) + return + } + } + if (Buffer.byteLength(pending) > 65536) stop(new Error('Waveform event exceeds its byte bound')) + }) + child.on('close', (code) => { + pending += decoder.end() + clearTimeout(timeout) + if (killTimer) clearTimeout(killTimer) + options.signal?.removeEventListener('abort', abort) + if (failure) reject(failure) + else if (code !== 0 || !result || (options.expectedBuildHash !== undefined && result.build_hash !== options.expectedBuildHash) || pending.length || result.counts.scored_outputs !== readings.length + || result.raw_records < result.decoded_records || result.decoded_records < result.counts.records + || readings.some(reading => reading.configHash !== result?.config_hashes[reading.side])) { + reject(new Error('Waveform worker failed or returned an incomplete result')) + } + else resolve({ readings, run: result }) + }) + child.stdin.end(serialized) + }) +} diff --git a/src/biometrics/backtest/waveform-contracts.ts b/src/biometrics/backtest/waveform-contracts.ts new file mode 100644 index 000000000..2f1e16cc8 --- /dev/null +++ b/src/biometrics/backtest/waveform-contracts.ts @@ -0,0 +1,57 @@ +import { z } from 'zod' + +const integer = z.number().int().min(0).max(Number.MAX_SAFE_INTEGER) +const hash = z.string().regex(/^[a-f0-9]{64}$/) +export const rawArtifactDescriptorSchema = z.object({ + name: z.string(), sha256: hash, compressed_bytes: integer, expanded_bytes: integer, +}).strict().refine(value => value.name === `${value.sha256}.RAW.gz`, 'Artifact name must match its content hash') +const context = z.object({ + start_ms: integer, end_ms: integer, + active_sides: z.array(z.enum(['left', 'right'])).max(2).nullable().refine(value => value === null + || value.length < 2 || (value[0] === 'left' && value[1] === 'right'), 'Invalid recorded sleeper mode'), + left_occupied: z.boolean().nullable(), right_occupied: z.boolean().nullable(), pump_context_known: z.boolean(), +}).strict() + +/** The same immutable input envelope consumed by the shipped Python driver. */ +export const retainedWaveformInputSchema = z.object({ + version: z.literal('retained-waveform-input-v1'), + retained_start_ms: integer, score_start_ms: integer, score_end_ms: integer, + artifacts: z.array(z.object({ artifact: rawArtifactDescriptorSchema, first_event_ms: integer, last_event_ms: integer }).strict()).min(1).max(64), + contexts: z.array(context).max(20000), +}).strict().superRefine((value, ctx) => { + const invalid = (message: string) => ctx.addIssue({ code: 'custom', message }) + if (!(value.retained_start_ms <= value.score_start_ms && value.score_start_ms < value.score_end_ms)) invalid('Invalid retained replay window') + const hashes = new Set() + let previous = -1, compressed = 0 + for (const source of value.artifacts) { + if (source.first_event_ms > source.last_event_ms || source.last_event_ms < value.retained_start_ms || source.first_event_ms >= value.score_end_ms) invalid('Artifact has no events in the retained window') + if (hashes.has(source.artifact.sha256) || source.first_event_ms <= previous) invalid('Retained artifacts must be distinct with nonoverlapping ordered event bounds') + hashes.add(source.artifact.sha256) + previous = source.last_event_ms + compressed += source.artifact.compressed_bytes + } + if (compressed > 1024 * 1024 * 1024) invalid('Retained replay compressed input budget exceeded') + previous = value.retained_start_ms + for (const interval of value.contexts) { + if (interval.start_ms < previous || interval.start_ms >= interval.end_ms || interval.end_ms > value.score_end_ms) invalid('Context intervals must be ordered and nonoverlapping inside the retained window') + previous = interval.end_ms + } +}) + +export const waveformDatasetInputsSchema = z.array(z.object({ + cohortId: z.string().min(1).max(200), input: retainedWaveformInputSchema, +}).strict()).min(1).max(60).superRefine((value, ctx) => { + if (new Set(value.map(item => item.cohortId)).size !== value.length) ctx.addIssue({ code: 'custom', message: 'Select each waveform cohort once' }) + const artifacts = new Map() + let compressed = 0 + for (const cohort of value) { + for (const source of cohort.input.artifacts) { + const previous = artifacts.get(source.artifact.sha256) + const encoded = JSON.stringify(source) + if (previous && previous !== encoded) ctx.addIssue({ code: 'custom', message: 'Conflicting evidence for shared waveform artifact' }) + if (!previous) compressed += source.artifact.compressed_bytes + artifacts.set(source.artifact.sha256, encoded) + } + } + if (artifacts.size > 1024 || compressed > 1024 * 1024 * 1024) ctx.addIssue({ code: 'custom', message: 'Waveform dataset RAW storage budget exceeded' }) +}) diff --git a/src/biometrics/backtest/waveform-dataset.ts b/src/biometrics/backtest/waveform-dataset.ts new file mode 100644 index 000000000..e0bf924e3 --- /dev/null +++ b/src/biometrics/backtest/waveform-dataset.ts @@ -0,0 +1,45 @@ +import { randomUUID } from 'node:crypto' +import { z } from 'zod' +import { configHash, dataHash } from '../algorithms/canonical' +import { datasetManifestSchema, datasetSnapshotSchema } from './contracts' +import { waveformDatasetInputsSchema } from './waveform-contracts' + +const sourceSchema = z.object({ id: z.string().uuid(), contentHash: z.string().regex(/^[a-f0-9]{64}$/), + manifest: datasetManifestSchema, snapshot: datasetSnapshotSchema }) + +/** Server-internal preparation. Sources and context must come from retained evidence. */ +export function prepareWaveformDataset(source: z.input, rawInputs: unknown, idempotencyKey: string, now = Date.now()) { + const base = sourceSchema.parse(structuredClone(source)) + if (base.snapshot.version !== 'processed-dataset-v1' || base.manifest.version !== 'processed-dataset-manifest-v1') throw new Error('Select a processed source dataset') + if (dataHash({ manifest: base.manifest, snapshot: base.snapshot }) !== base.contentHash) throw new Error('Source dataset integrity mismatch') + z.string().min(1).max(200).parse(idempotencyKey) + z.number().int().min(0).max(Number.MAX_SAFE_INTEGER).parse(now) + const inputs = waveformDatasetInputsSchema.parse(rawInputs) + const cohorts = new Map(base.manifest.splitProtocol.assignments.map(item => [item.cohortId, item])) + if (cohorts.size !== base.manifest.splitProtocol.assignments.length || inputs.length !== cohorts.size) throw new Error('Retain waveform inputs for every whole-night cohort') + for (const item of inputs) { + const cohort = cohorts.get(item.cohortId) + if (!cohort || item.input.score_start_ms !== cohort.startMs || item.input.score_end_ms !== cohort.endMs) throw new Error('Waveform input must match its whole-night cohort') + } + // Follow the immutable split assignment order, independent of selection UI order. + const order = [...cohorts.keys()] + inputs.sort((a, b) => order.indexOf(a.cohortId) - order.indexOf(b.cohortId)) + const snapshot = datasetSnapshotSchema.parse({ ...base.snapshot, version: 'waveform-dataset-v1', waveformInputs: inputs }) + const manifest = datasetManifestSchema.parse({ ...base.manifest, version: 'waveform-dataset-manifest-v1', + parentDataset: { id: base.id, contentHash: base.contentHash }, + capabilities: { ...base.manifest.capabilities, signalReplay: true }, + limitations: ['Waveform files and supplied context intervals are frozen; execution requires configured deployment admission and current resource health.', + 'Event bounds do not establish continuous waveform coverage; gaps in historical context remain unknown.', + 'Replay uses event time and a cold start with retained pre-roll, not serialized historical processor state.', + 'Measured calibration is not reconstructed from current calibration settings.', + 'Historical stage replay must declare its calibration quality assumption when original calibration is unavailable.', + 'Reference deletion does not erase this separately retained dataset.', + 'Historical occupant identity is asserted by the dataset creator, not measured by the pod.'], + }) + const contentHash = dataHash({ manifest, snapshot }) + const prepared = { id: randomUUID(), side: snapshot.side, subjectId: snapshot.subjectId, + idempotencyKey, requestHash: configHash({ version: 'waveform-dataset-request-v1', parentId: base.id, + parentHash: base.contentHash, inputsHash: dataHash(inputs), idempotencyKey }), contentHash, manifest, snapshot, createdAtMs: now } + if (Buffer.byteLength(JSON.stringify(prepared)) > 8 * 1024 * 1024) throw new Error('Waveform dataset exceeds its byte budget') + return prepared +} diff --git a/src/biometrics/backtest/waveform-execution.ts b/src/biometrics/backtest/waveform-execution.ts new file mode 100644 index 000000000..9cf09ea6d --- /dev/null +++ b/src/biometrics/backtest/waveform-execution.ts @@ -0,0 +1,63 @@ +import type { ReplayBudget } from './execution-budget' +import { sql } from 'drizzle-orm' +import { dirname, join } from 'node:path' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import type { z } from 'zod' +import type * as schema from '@/src/db/biometrics-schema' +import { dataHash } from '../algorithms/canonical' +import type { backtestWorkSchema } from './evaluator' +import { waveformEvaluationSchema } from './evaluator' +import { datasetManifestSchema } from './contracts' +import { podRawInspectionLocation, type RawInspectionLocation } from './raw-inspection-child' +import { waveformEvaluatorBuildIdentity } from './waveform-build' +import { runWaveformChild } from './waveform-child' + +export async function verifyWaveformJobBuild(work: z.infer, options: { + workerPath: string + rawLocation?: RawInspectionLocation + signal: AbortSignal +}) { + const build = await waveformEvaluatorBuildIdentity(join(dirname(options.workerPath), 'coordinator.cjs'), options.workerPath, + options.rawLocation ?? podRawInspectionLocation, options.signal) + if (build.hash !== work.job.evaluatorBuildHash || build.python.hash !== work.job.waveform?.pythonBuildHash) throw new Error('Waveform worker build changed since enqueue') +} + +/** Run one cohort/configuration at a time. Raw bytes stay in Python; only + * bounded derived rows reach the isolated Node scorer and retained report. */ +export async function executeWaveformInputs(db: BetterSQLite3Database, work: z.infer, options: { + workerPath: string + rawLocation?: RawInspectionLocation + signal: AbortSignal + timeoutMs?: number + replayBudget?: ReplayBudget +}) { + const { job, dataset } = work + if (job.protocol.mode !== 'signal_replay' || !job.waveform || dataset.snapshot.version !== 'waveform-dataset-v1') throw new Error('Missing retained waveform job inputs') + const databasePath = db.all<{ name: string, file: string }>(sql`PRAGMA database_list`).find(row => row.name === 'main')?.file + if (!databasePath) throw new Error('Waveform execution requires a persistent database') + const location = { ...(options.rawLocation ?? podRawInspectionLocation), retainedDirectory: join(dirname(databasePath), 'biometric-raw') } + await verifyWaveformJobBuild(work, options) + const assignments = datasetManifestSchema.parse(dataset.manifest).splitProtocol.assignments.filter(row => row.split === job.split) + const configurations = [...new Map(job.configs.map(config => [config.resolvedHash, config])).values()] + const outputs: z.infer['outputs'] = [] + let count = 0 + for (const cohort of assignments) { + const input = dataset.snapshot.waveformInputs.find(row => row.cohortId === cohort.cohortId)?.input + if (!input) throw new Error('Missing retained waveform cohort') + for (const config of configurations) { + if (options.signal.aborted) throw new Error('Waveform job cancelled') + const configs = dataset.snapshot.side === 'left' + ? { left: config.resolved, right: job.waveform.otherSide.resolved } + : { left: job.waveform.otherSide.resolved, right: config.resolved } + const output = await runWaveformChild(location, { version: 'retained-waveform-work-v1', input, configs }, { + signal: options.signal, timeoutMs: options.replayBudget?.waveformWallMs ?? options.timeoutMs, + cpuSeconds: options.replayBudget?.waveformCpuSeconds, expectedBuildHash: job.waveform.pythonBuildHash, + }) + count += output.readings.length + if (count > 20000) throw new Error('Waveform job derived reading budget exceeded') + outputs.push({ cohortId: cohort.cohortId, configHash: config.resolvedHash, ...output }) + } + } + await verifyWaveformJobBuild(work, options) + return waveformEvaluationSchema.parse({ hash: dataHash(outputs), outputs }) +} diff --git a/src/biometrics/backtest/waveform-output.ts b/src/biometrics/backtest/waveform-output.ts new file mode 100644 index 000000000..a00eb7f93 --- /dev/null +++ b/src/biometrics/backtest/waveform-output.ts @@ -0,0 +1,33 @@ +import { z } from 'zod' + +const count = z.number().int().min(0).max(Number.MAX_SAFE_INTEGER) +const hash = z.string().regex(/^[a-f0-9]{64}$/) +const side = z.enum(['left', 'right']) +const value = z.number().finite().nullable() +const limitations = z.array(z.string().max(200)).max(40) +export const waveformReadingSchema = z.object({ + side, timestampMs: count, heartRate: value, hrvIndex: value, + hrvMetric: z.literal('window_ibi_variability_v1'), breathingRate: value, + qualityScore: z.number().finite(), flags: z.array(z.string().max(200)).max(40).nullable(), hrRaw: value, + sourceSide: side, configHash: hash, limitations, +}).strict() +/** Optional only for retained reports produced before resource collection. + * These are one Python process's costs, not whole-job or live-ingestion costs. */ +export const waveformProcessResourcesSchema = z.object({ + version: z.literal('waveform-process-resources-v1'), + wallMs: z.number().finite().nonnegative(), cpuMs: z.number().finite().nonnegative(), + peakRssBytes: z.number().int().positive().max(Number.MAX_SAFE_INTEGER).nullable(), + scope: z.literal('python_process_until_result'), memoryScope: z.literal('process_lifetime_high_water_mark'), +}).strict() +export const waveformRunSchema = z.object({ + resources: waveformProcessResourcesSchema.optional(), + version: z.literal('retained-waveform-run-v1'), build_hash: hash, decoded_records: count, raw_records: count, + counts: z.object({ records: count, accepted_frames: count, empty_channel_frames: count, + pump_gated_frames: count, preroll_outputs: count, scored_outputs: count }).strict(), + limitations, config_hashes: z.object({ left: hash, right: hash }).strict(), artifact_hashes: z.array(hash).min(1).max(64), +}).strict() + +export const waveformEvaluationSchema = z.object({ hash: z.string().regex(/^[a-f0-9]{64}$/), outputs: z.array(z.object({ + cohortId: z.string().min(1).max(200), configHash: z.string().regex(/^[a-f0-9]{64}$/), + readings: z.array(waveformReadingSchema).max(20000), run: waveformRunSchema, +}).strict()).max(180) }).strict().refine(value => value.outputs.reduce((sum, output) => sum + output.readings.length, 0) <= 20000, 'Waveform reading budget exceeded') diff --git a/src/biometrics/backtest/waveform-publication.ts b/src/biometrics/backtest/waveform-publication.ts new file mode 100644 index 000000000..82b4a7ebe --- /dev/null +++ b/src/biometrics/backtest/waveform-publication.ts @@ -0,0 +1,62 @@ +import { sql } from 'drizzle-orm' +import { dirname, join } from 'node:path' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import type { z } from 'zod' +import type * as schema from '@/src/db/biometrics-schema' +import { ReferenceInputError } from '../references/time' +import { createWaveformDatasetSchema, prepareSelectedWaveformDataset, selectedWaveformRetry } from './waveform-selection' +import { podRawInspectionLocation, type RawInspectionLocation } from './raw-inspection-child' +import { runRawPublicationChild } from './raw-publication-child' + +/** Copy outside the live DB transaction. Python atomically publishes all refs; + * a lost response or cancellation after commit is recovered by request identity. */ +export async function publishSelectedWaveformDataset(db: BetterSQLite3Database, raw: z.input, options: { + signal: AbortSignal + admit: () => Promise + rawLocation?: RawInspectionLocation +}) { + const input = createWaveformDatasetSchema.parse(raw) + const existing = selectedWaveformRetry(db, input) + if (existing) return { status: 'published' as const, dataset: existing } + const deferred = { status: 'deferred' as const } + const admitted = () => Promise.resolve().then(options.admit).catch(() => false) + if (options.signal.aborted || !await admitted() || options.signal.aborted) return deferred + const databasePath = db.all<{ name: string, file: string }>(sql`PRAGMA database_list`).find(row => row.name === 'main')?.file + if (!databasePath) throw new ReferenceInputError('WORKER_UNAVAILABLE', 'Waveform publication requires a persistent biometrics database') + const work = prepareSelectedWaveformDataset(db, input) + if (!await admitted() || options.signal.aborted) return deferred + const controller = new AbortController() + const abort = () => controller.abort() + options.signal.addEventListener('abort', abort, { once: true }) + if (options.signal.aborted) abort() + let checking: Promise | undefined, denied = false + const timer = setInterval(() => { + if (checking) return + checking = admitted().then((allowed) => { + if (!allowed) { + denied = true + controller.abort() + } + }).finally(() => { checking = undefined }) + }, 5000) + try { + const result = await runRawPublicationChild({ ...(options.rawLocation ?? podRawInspectionLocation), databasePath, + retainedDirectory: join(dirname(databasePath), 'biometric-raw') }, work, { signal: controller.signal }) + const published = selectedWaveformRetry(db, input) + if (!published || published.id !== result.id || published.contentHash !== result.contentHash) throw new Error('Published waveform identity mismatch') + return { status: 'published' as const, dataset: published } + } + catch (error) { + // A fully committed dataset remains published even if IPC was lost or a + // stop raced with commit. Returning it never resurrects a released row. + const published = selectedWaveformRetry(db, input) + if (published) return { status: 'published' as const, dataset: published } + if (denied) return deferred + throw error + } + finally { + clearInterval(timer) + await checking + options.signal.removeEventListener('abort', abort) + } +} diff --git a/src/biometrics/backtest/waveform-resources.ts b/src/biometrics/backtest/waveform-resources.ts new file mode 100644 index 000000000..d227505d5 --- /dev/null +++ b/src/biometrics/backtest/waveform-resources.ts @@ -0,0 +1,28 @@ +import { z } from 'zod' +import { waveformProcessResourcesSchema } from './waveform-output' + +export const waveformResourceSummarySchema = z.object({ + version: z.literal('waveform-resource-summary-v1'), + scope: z.literal('sequential_python_processes_excluding_coordinator_and_scorer'), + expectedProcesses: z.number().int().min(0).max(180), measuredProcesses: z.number().int().min(0).max(180), + totalWallMs: z.number().finite().nonnegative().nullable(), totalCpuMs: z.number().finite().nonnegative().nullable(), + largestProcessPeakRssBytes: z.number().int().positive().max(Number.MAX_SAFE_INTEGER).nullable(), + ingestionHealthMeasured: z.literal(false), +}).strict() + +/** Sequential workers: sum durations and retain the largest process high water + * mark. This deliberately does not claim simultaneous job RSS or live health. */ +export function summarizeWaveformResources(outputs: Array<{ run: { resources?: z.infer } }>) { + const measured = outputs.flatMap(output => output.run.resources ? [waveformProcessResourcesSchema.parse(output.run.resources)] : []) + const complete = outputs.length > 0 && measured.length === outputs.length + const memoryComplete = complete && measured.every(row => row.peakRssBytes !== null) + return { + version: 'waveform-resource-summary-v1' as const, + scope: 'sequential_python_processes_excluding_coordinator_and_scorer' as const, + expectedProcesses: outputs.length, measuredProcesses: measured.length, + totalWallMs: complete ? measured.reduce((sum, row) => sum + row.wallMs, 0) : null, + totalCpuMs: complete ? measured.reduce((sum, row) => sum + row.cpuMs, 0) : null, + largestProcessPeakRssBytes: memoryComplete ? Math.max(...measured.map(row => row.peakRssBytes ?? 0)) : null, + ingestionHealthMeasured: false as const, + } +} diff --git a/src/biometrics/backtest/waveform-selection.ts b/src/biometrics/backtest/waveform-selection.ts new file mode 100644 index 000000000..133d20d7b --- /dev/null +++ b/src/biometrics/backtest/waveform-selection.ts @@ -0,0 +1,131 @@ +import { z } from 'zod' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import { eq } from 'drizzle-orm' +import { configHash } from '../algorithms/canonical' +import * as schema from '@/src/db/biometrics-schema' +import { ReferenceInputError } from '../references/time' +import { DatasetStore } from './datasets' +import { datasetManifestSchema, type DatasetSnapshot } from './contracts' +import { RawSourceCatalog } from './raw-catalog' +import { prepareWaveformDataset } from './waveform-dataset' + +const hash = z.string().regex(/^[a-f0-9]{64}$/) +export const createWaveformDatasetSchema = z.object({ + parentDatasetId: z.string().uuid(), expectedContentHash: hash, + cohorts: z.array(z.object({ cohortId: z.string().min(1).max(200), + sourceIds: z.array(hash).min(1).max(64), + }).strict()).min(1).max(60), + idempotencyKey: z.string().min(1).max(200), +}).strict() + +/** Retry identity depends on the requested immutable IDs, not mutable catalog availability. */ +export function waveformSelectionRequestHash(raw: z.input) { + const input = createWaveformDatasetSchema.parse(raw) + const cohorts = input.cohorts.map(item => ({ ...item, sourceIds: [...item.sourceIds].sort() })) + .sort((a, b) => a.cohortId < b.cohortId ? -1 : a.cohortId > b.cohortId ? 1 : 0) + return configHash({ version: 'selected-waveform-dataset-request-v1', ...input, cohorts }) +} + +export function selectedWaveformRetry(db: BetterSQLite3Database, raw: z.input) { + const input = createWaveformDatasetSchema.parse(raw) + const row = db.select().from(schema.biometricDatasets).where(eq(schema.biometricDatasets.idempotencyKey, input.idempotencyKey)).get() + if (!row) return null + if (row.requestHash !== waveformSelectionRequestHash(input)) throw new ReferenceInputError('REVISION_CONFLICT', 'Dataset idempotency key was already used for a different selection') + return new DatasetStore(db).descriptor(row.id) +} + +/** Only intervals actually recorded in the frozen parent are evidence. Session + * boundaries are not evidence of continuous occupancy, nor of sleeper mode. */ +export function recordedWaveformContexts(snapshot: DatasetSnapshot, startMs: number, endMs: number) { + const events = new Map() + let count = 0 + for (const session of snapshot.sessions) { + for (const [raw, state] of [[session.presentIntervals, 0], [session.notPresentIntervals, 1]] as const) { + if (!Array.isArray(raw)) continue + for (const item of raw) { + if (++count > 20000) throw new ReferenceInputError('RESOURCE_LIMIT', 'Recorded occupancy interval budget exceeded') + // The sleep detector stores epoch seconds (including fractional seconds). + if (!Array.isArray(item) || item.length !== 2 || item.some(value => typeof value !== 'number' || !Number.isFinite(value) || value < 0)) continue + const first = Math.round(item[0] * 1000), last = Math.round(item[1] * 1000) + if (!Number.isSafeInteger(first) || !Number.isSafeInteger(last) || first >= last) continue + const start = Math.max(first, startMs, session.startMs), end = Math.min(last, endMs, session.endMs) + if (start >= end) continue + for (const [time, delta] of [[start, 1], [end, -1]]) { + const event = events.get(time) ?? [0, 0] + event[state] += delta + events.set(time, event) + } + } + } + } + const times = [...events.keys()].sort((a, b) => a - b) + let present = 0, absent = 0 + const contexts: Array<{ + start_ms: number + end_ms: number + active_sides: null + left_occupied: boolean | null + right_occupied: boolean | null + pump_context_known: false + }> = [] + for (let index = 0; index < times.length - 1; index++) { + const [yes, no] = events.get(times[index]) ?? [0, 0] + present += yes + absent += no + // Conflicting evidence and unrecorded gaps both remain unknown. + if ((present > 0) === (absent > 0)) continue + const occupied = present > 0 + const last = contexts.at(-1) + if (last && last.end_ms === times[index] && last[`${snapshot.side}_occupied`] === occupied) last.end_ms = times[index + 1] + else contexts.push({ start_ms: times[index], end_ms: times[index + 1], active_sides: null, + left_occupied: snapshot.side === 'left' ? occupied : null, + right_occupied: snapshot.side === 'right' ? occupied : null, pump_context_known: false }) + } + if (contexts.length > 20000) throw new ReferenceInputError('RESOURCE_LIMIT', 'Recorded occupancy context budget exceeded') + return contexts +} + +/** Resolve IDs to verified catalog evidence and frozen historical context. No + * archive reads or live configuration queries occur while the DB snapshot is held. */ +export function prepareSelectedWaveformDataset(db: BetterSQLite3Database, raw: z.input, now = Date.now()) { + const input = createWaveformDatasetSchema.parse(raw) + if (new Set(input.cohorts.map(item => item.cohortId)).size !== input.cohorts.length) throw new ReferenceInputError('DUPLICATE_SELECTION', 'Select each whole-night cohort once') + return db.transaction(() => { + const parent = new DatasetStore(db).read(input.parentDatasetId) + if (parent.contentHash !== input.expectedContentHash) throw new ReferenceInputError('DATASET_CHANGED', 'Select the expected immutable parent dataset') + const manifest = datasetManifestSchema.parse(parent.manifest) + if (input.cohorts.length !== manifest.splitProtocol.assignments.length) throw new ReferenceInputError('INSUFFICIENT_RAW_DATA', 'Select RAW inputs for every whole-night cohort') + const catalog = new RawSourceCatalog(db) + const selections = new Map['inspection']['artifact'] }>() + const inputs = manifest.splitProtocol.assignments.map((cohort) => { + const selection = input.cohorts.find(item => item.cohortId === cohort.cohortId) + if (!selection) throw new ReferenceInputError('INSUFFICIENT_RAW_DATA', 'Missing whole-night RAW selection') + if (new Set(selection.sourceIds).size !== selection.sourceIds.length) throw new ReferenceInputError('DUPLICATE_SELECTION', 'Select each RAW source once per cohort') + const artifacts = selection.sourceIds.map((id) => { + const source = catalog.read(id) + const { inventory, artifact, source_name } = source.inspection + const cached = source.unavailableAtMs === null + ? undefined + : db.select().from(schema.biometricRawArtifacts) + .where(eq(schema.biometricRawArtifacts.contentHash, artifact.sha256)).get() + const mayReuse = cached?.name === artifact.name && cached.compressedBytes === artifact.compressed_bytes + && cached.expandedBytes === artifact.expanded_bytes + // Metadata only permits attempting reuse; Python still verifies the complete retained file. + if ((source.unavailableAtMs !== null && !mayReuse) || !inventory.ordered_waveform_available + || inventory.first_event_ms === null || inventory.last_event_ms === null) { + throw new ReferenceInputError('INSUFFICIENT_RAW_DATA', 'Selected RAW source is unavailable or cannot provide ordered waveforms') + } + const previous = selections.get(artifact.sha256) + if (!previous || source_name < previous.source_name) selections.set(artifact.sha256, { source_name, artifact }) + return { artifact, first_event_ms: inventory.first_event_ms, last_event_ms: inventory.last_event_ms } + }).sort((a, b) => a.first_event_ms - b.first_event_ms || a.artifact.sha256.localeCompare(b.artifact.sha256)) + const start = Math.max(0, cohort.startMs - 300000) + return { cohortId: cohort.cohortId, input: { version: 'retained-waveform-input-v1' as const, + retained_start_ms: start, score_start_ms: cohort.startMs, score_end_ms: cohort.endMs, + artifacts, contexts: recordedWaveformContexts(parent.snapshot, start, cohort.endMs) } } + }) + const prepared = prepareWaveformDataset({ ...parent, manifest }, inputs, input.idempotencyKey, now) + prepared.requestHash = waveformSelectionRequestHash(input) + return { prepared, selections: [...selections.values()].sort((a, b) => a.artifact.sha256.localeCompare(b.artifact.sha256)) } + }) +} diff --git a/src/biometrics/backtest/worker-entry.ts b/src/biometrics/backtest/worker-entry.ts new file mode 100644 index 000000000..930a4979f --- /dev/null +++ b/src/biometrics/backtest/worker-entry.ts @@ -0,0 +1,30 @@ +import { evaluateBacktest } from './evaluator' + +// Shipped evaluator only. This process is not an untrusted-code sandbox. +const MAX_INPUT = 10 * 1024 * 1024 +let received = 0 +const chunks: Buffer[] = [] +process.stdin.on('data', (chunk: Buffer) => { + received += chunk.length + if (received > MAX_INPUT) { + process.stderr.write('Replay input exceeds the worker byte limit\n') + process.exit(2) + } + chunks.push(chunk) +}) +process.stdin.on('end', () => { + try { + const input: unknown = JSON.parse(Buffer.concat(chunks).toString('utf8')) + // Parsing at the boundary is performed again by the shared contract. + const started = performance.now() + const report = evaluateBacktest(input as Parameters[0], (completedNights, totalNights) => { + process.stdout.write(`${JSON.stringify({ type: 'progress', completedNights, totalNights })}\n`) + }) + process.stdout.write(`${JSON.stringify({ type: 'result', report: { ...report, operational: { runtimeMs: performance.now() - started, peakRssKiB: process.resourceUsage().maxRSS, heapLimitMiB: 128 } } })}\n`) + } + catch { + // Input exports and annotations are health data, not diagnostic log text. + process.stderr.write('Replay failed input validation or evaluation\n') + process.exitCode = 1 + } +}) diff --git a/src/biometrics/packages/README.md b/src/biometrics/packages/README.md new file mode 100644 index 000000000..e387e9148 --- /dev/null +++ b/src/biometrics/packages/README.md @@ -0,0 +1,71 @@ +# Algorithm package proposals + +`manifest.ts` validates inert proposals for the algorithm-development workflow. +It does not import, execute, install, or approve their code. No API currently +accepts these proposals. The [off-pod experiment workflow](../../../scripts/algorithm-experiments/README.md) +provides Docker execution, deterministic/noise checks, managed whole-night +comparisons, resource assessments and combined external qualification evidence. +That evidence cannot install code. Promotion requires representative-fixture +review, matching target-pod and prospective evidence, and a reviewed release +into the shipped registry. + +The versioned manifest declares engine identity, frame schema identifiers, +metric definitions and units, supported hardware/sample rates, causal or +retrospective behavior, deterministic initialization and reset, JSON state +serialization, resource requests, an entrypoint, and a dependency lock. Every +file is listed with its exact byte length and SHA-256 digest. Package identity +is SHA-256 over the complete manifest using `finite-binary64-json-v1` canonical +JSON. Artifact array order is part of that identity. + +`verifyAlgorithmPackage` takes a map of bytes, avoiding filesystem resolution +or archive extraction. It rejects unexpected files, missing files, mismatched +sizes/hashes, traversal paths, duplicate identities, and oversized proposals. +The 32 MiB intake ceiling is a storage bound, not a qualified runtime budget. +A future transport must enforce size limits before buffering. Intake verifies +only the dependency-lock artifact identity. The external +executor additionally requires an exact Python environment lock and independently +probes its reviewed runtime image; see the executor README for that format. + +Execution must use an external isolation boundary without pod credentials, +network access by default, writable input, or access to live sinks. Resource +requests must be checked against server-owned limits and enforced outside the +candidate process. Deterministic replay, artifact/noise fixtures, resource +measurements, held-out comparisons, and compatibility checks remain required. +An experiment result never grants installation rights: promotion goes through +code review and a release into the shipped registry. + +## Streaming frame contract + +`protocol.ts` defines the host-side exchange. A run names the exact package and +input hashes, hardware, sample rate, side, seed, and finite JSON configuration. +Its canonical hash binds all those choices. Create a fresh validator and reset +candidate state for each independent run; never carry state between nights or +qualification repetitions unless a future protocol explicitly declares it. + +Frames carry increasing sequence numbers from zero, explicit sample-start +timestamps, one or both channels of at most 4096 finite samples, and explicit +nullable occupancy/pump context. Missing context is unknown, not false. +Channels must agree in length. Sample duration determines the next permissible +frame time; gaps are permitted, overlaps are rejected. The executor must call +`validateFrame` before exposing each frame to the candidate. + +A response echoes the run hash, canonical input-frame hash, and sequence. It +contains bounded numeric/null metric readings and JSON state. The host checks +metric definitions, side, ranges, strictly increasing timestamps per metric, +observed time bounds, declared retrospective lookahead, state bytes, and the +cumulative canonical response-byte budget. Validation failure does not advance +the run. No response can authorize database or hardware writes. + +The external executor must bound transport bytes before JSON decoding (including +whitespace), enforce process resources independently, and expose only the current +frame and previously returned state. The validator cannot prove that code obeys +causality, initialization, reset, or determinism; those need the executor and +repeated qualification runs. This protocol is not itself an isolation boundary. + +A metric can optionally declare a structured `reference` contract for external +comparison: `metric` is one of `heart_rate`, `breathing_rate`, +`window_ibi_variability_v1`, `hrv_rmssd`, or `hrv_sdnn`; `windowMs` is required and +may be null only for non-HRV metrics. The declaration requires canonical units +and participates in package identity. Metrics without this declaration remain +valid experimental outputs but cannot enter reference comparison based on their +name or prose definition alone. diff --git a/src/biometrics/packages/manifest.ts b/src/biometrics/packages/manifest.ts new file mode 100644 index 000000000..50c173b8e --- /dev/null +++ b/src/biometrics/packages/manifest.ts @@ -0,0 +1,75 @@ +import { createHash } from 'node:crypto' +import { z } from 'zod' +import { configHash } from '../algorithms/canonical' + +const hash = z.string().regex(/^[a-f0-9]{64}$/) +const identifier = z.string().regex(/^[a-z][a-z0-9_.-]{0,99}$/) +const positiveInteger = z.number().int().positive().max(Number.MAX_SAFE_INTEGER) +const relativePath = z.string().min(1).max(240).refine(path => + path.split('/').every(part => /^[A-Za-z0-9_][A-Za-z0-9_.-]*$/.test(part)), 'Expected a relative package file path') +const artifactSchema = z.object({ path: relativePath, sha256: hash, bytes: positiveInteger }).strict() +const referenceContractSchema = z.object({ + metric: z.enum(['heart_rate', 'breathing_rate', 'window_ibi_variability_v1', 'hrv_rmssd', 'hrv_sdnn']), + windowMs: z.number().int().positive().max(86400000).nullable(), +}).strict().refine(value => !['window_ibi_variability_v1', 'hrv_rmssd', 'hrv_sdnn'].includes(value.metric) || value.windowMs !== null, 'HRV comparison requires an explicit window') +const metricSchema = z.object({ + id: identifier, definition: z.string().min(1).max(2000), unit: z.string().min(1).max(80), + minimum: z.number().finite(), maximum: z.number().finite(), + reference: referenceContractSchema.optional(), +}).strict().refine(metric => metric.minimum < metric.maximum, 'Metric bounds must be ordered') + .refine(metric => !metric.reference || metric.unit === (metric.reference.metric === 'heart_rate' ? 'bpm' : metric.reference.metric === 'breathing_rate' ? 'breaths/min' : 'ms'), 'Reference metric requires canonical units') + +/** Inert proposal metadata. Passing this schema grants no execution or installation rights. */ +export const algorithmPackageManifestSchema = z.object({ + schema: z.literal('biometric-algorithm-package-v1'), + canonicalization: z.literal('finite-binary64-json-v1'), + engine: z.object({ id: identifier, version: z.string().regex(/^\d+\.\d+\.\d+$/) }).strict(), + runtime: z.literal('python-frame-v1'), entrypoint: relativePath, + inputSchema: z.literal('biometric-signal-frame-v1'), outputSchema: z.literal('biometric-metric-frame-v1'), + metrics: z.array(metricSchema).min(1).max(16), + hardware: z.array(z.object({ id: identifier, sampleRatesHz: z.array(positiveInteger).min(1).max(16) }).strict()).min(1).max(16), + temporal: z.discriminatedUnion('mode', [ + z.object({ mode: z.literal('causal'), lookaheadMs: z.literal(0) }).strict(), + z.object({ mode: z.literal('retrospective'), lookaheadMs: positiveInteger }).strict(), + ]), + state: z.object({ + schemaId: identifier, initialization: z.literal('explicit_seed_and_config'), + reset: z.literal('reset_before_each_independent_run'), serialization: z.literal('json'), maximumBytes: positiveInteger, + }).strict(), + resources: z.object({ + cpuMs: positiveInteger, wallMs: positiveInteger, memoryBytes: positiveInteger, + processes: positiveInteger, outputBytes: positiveInteger, + }).strict(), + dependencyLock: relativePath, + artifacts: z.array(artifactSchema).min(2).max(128), +}).strict().superRefine((value, context) => { + const unique = (values: (string | number)[], label: string) => { + if (new Set(values).size !== values.length) context.addIssue({ code: 'custom', message: `Duplicate ${label}` }) + } + unique(value.metrics.map(metric => metric.id), 'metric') + unique(value.hardware.map(hardware => hardware.id), 'hardware') + for (const hardware of value.hardware) unique(hardware.sampleRatesHz, 'sample rate') + unique(value.artifacts.map(artifact => artifact.path), 'artifact path') + for (const path of [value.entrypoint, value.dependencyLock]) { + if (!value.artifacts.some(artifact => artifact.path === path)) context.addIssue({ code: 'custom', message: `Unhashed package file: ${path}` }) + } + if (value.entrypoint === value.dependencyLock) context.addIssue({ code: 'custom', message: 'Entrypoint and dependency lock must be separate files' }) +}) +export type AlgorithmPackageManifest = z.infer + +/** Bounded byte inputs, deliberately with no filesystem paths, imports, or subprocesses. + * Callers must bound transport size before buffering. These limits are proposal intake + * limits, not evidence that an algorithm fits the pod's runtime budget. */ +export const maximumPackageBytes = 32 * 1024 * 1024 +export function verifyAlgorithmPackage(manifestInput: unknown, files: ReadonlyMap) { + const manifest = algorithmPackageManifestSchema.parse(manifestInput) + if (files.size !== manifest.artifacts.length) throw new Error('Package file set differs from manifest') + const totalBytes = manifest.artifacts.reduce((sum, artifact) => sum + artifact.bytes, 0) + if (!Number.isSafeInteger(totalBytes) || totalBytes > maximumPackageBytes) throw new Error('Package exceeds intake byte limit') + for (const artifact of manifest.artifacts) { + const bytes = files.get(artifact.path) + if (!bytes || bytes.byteLength !== artifact.bytes) throw new Error(`Package file size mismatch: ${artifact.path}`) + if (createHash('sha256').update(bytes).digest('hex') !== artifact.sha256) throw new Error(`Package file hash mismatch: ${artifact.path}`) + } + return { manifest, packageHash: configHash(manifest), totalBytes, status: 'inert_proposal' as const } +} diff --git a/src/biometrics/packages/protocol.ts b/src/biometrics/packages/protocol.ts new file mode 100644 index 000000000..13ac10e6b --- /dev/null +++ b/src/biometrics/packages/protocol.ts @@ -0,0 +1,92 @@ +import { z } from 'zod' +import { canonicalJson, configHash, dataHash } from '../algorithms/canonical' +import { algorithmPackageManifestSchema } from './manifest' + +const hash = z.string().regex(/^[a-f0-9]{64}$/) +const time = z.number().finite().nonnegative().max(Number.MAX_SAFE_INTEGER) +const integer = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) +const side = z.enum(['left', 'right']) +const samples = z.array(z.number().finite()).min(1).max(4096) +export const packageRunRequestSchema = z.object({ + schema: z.literal('biometric-package-run-v1'), packageHash: hash, inputHash: hash, + hardwareId: z.string().min(1).max(100), sampleRateHz: z.number().int().positive(), side, + seed: integer, config: z.record(z.string(), z.unknown()), +}).strict() +export const signalFrameSchema = z.object({ + schema: z.literal('biometric-signal-frame-v1'), sequence: integer, timestampMs: time, + samples: z.object({ left: samples.nullable(), right: samples.nullable() }).strict(), + context: z.object({ occupied: z.boolean().nullable(), pumpActive: z.boolean().nullable() }).strict(), +}).strict().refine(frame => frame.samples.left !== null || frame.samples.right !== null, 'Frame must contain samples') + .refine(frame => !frame.samples.left || !frame.samples.right || frame.samples.left.length === frame.samples.right.length, 'Channel lengths differ') +export const metricFrameSchema = z.object({ + schema: z.literal('biometric-metric-frame-v1'), runHash: hash, frameHash: hash, sequence: integer, + readings: z.array(z.object({ + timestampMs: time, side, metric: z.string().min(1).max(100), value: z.number().finite().nullable(), + quality: z.number().min(0).max(1), flags: z.array(z.string().min(1).max(100)).max(32), + }).strict()).max(256), + state: z.unknown(), +}).strict() + +/** Host-side validation of a streaming exchange. This class never runs a package. + * One validator belongs to one independent run and starts at sequence zero. + * The executor must send only the current frame, after resetting package state. */ +export class PackageFrameValidator { + readonly manifest + readonly request + readonly runHash: string + private nextSequence = 0 + private outputBytes = 0 + private nextInputTime = 0 + private firstInputTime: number | null = null + private lastReadingTimes = new Map() + + constructor(manifestInput: unknown, requestInput: unknown) { + this.manifest = algorithmPackageManifestSchema.parse(manifestInput) + this.request = packageRunRequestSchema.parse(requestInput) + if (this.request.packageHash !== configHash(this.manifest)) throw new Error('Package identity mismatch') + const hardware = this.manifest.hardware.find(item => item.id === this.request.hardwareId) + if (!hardware?.sampleRatesHz.includes(this.request.sampleRateHz)) throw new Error('Unsupported hardware or sample rate') + // Canonical hashing also rejects non-JSON config values and bounds complexity. + this.runHash = configHash(this.request) + } + + validateFrame(frameInput: unknown) { + const frame = signalFrameSchema.parse(frameInput) + if (frame.sequence !== this.nextSequence || frame.timestampMs < this.nextInputTime) throw new Error('Out-of-order or overlapping frame') + const channel = frame.samples[this.request.side] + if (!channel) throw new Error('Requested side has no samples') + const endMs = frame.timestampMs + channel.length * 1000 / this.request.sampleRateHz + if (!Number.isFinite(endMs) || endMs > Number.MAX_SAFE_INTEGER) throw new Error('Frame end exceeds timestamp range') + return { frame, endMs } + } + + validateStep(frameInput: unknown, resultInput: unknown) { + const { frame, endMs } = this.validateFrame(frameInput) + const result = metricFrameSchema.parse(resultInput) + if (result.runHash !== this.runHash || result.frameHash !== dataHash(frame) || result.sequence !== frame.sequence) throw new Error('Output identity mismatch') + // A transport must additionally cap stdout before buffering/parsing it. + const outputBytes = this.outputBytes + Buffer.byteLength(canonicalJson(result)) + if (outputBytes > this.manifest.resources.outputBytes) throw new Error('Output byte limit exceeded') + if (Buffer.byteLength(canonicalJson(result.state)) > this.manifest.state.maximumBytes) throw new Error('State byte limit exceeded') + const firstInputTime = this.firstInputTime ?? frame.timestampMs + const lastTimes = new Map(this.lastReadingTimes) + for (const reading of result.readings) { + const metric = this.manifest.metrics.find(item => item.id === reading.metric) + if (!metric || reading.side !== this.request.side) throw new Error('Undeclared metric or side') + if (reading.value !== null && (reading.value < metric.minimum || reading.value > metric.maximum)) throw new Error('Metric outside declared range') + // A retrospective reading may lag the latest available sample by its + // declared lookahead. No output may refer to unavailable future samples. + const latestSampleMs = endMs - 1000 / this.request.sampleRateHz + if (reading.timestampMs < firstInputTime || reading.timestampMs + this.manifest.temporal.lookaheadMs > latestSampleMs) throw new Error('Reading outside observed time range') + if (reading.timestampMs <= (lastTimes.get(reading.metric) ?? -1)) throw new Error('Duplicate or out-of-order reading') + lastTimes.set(reading.metric, reading.timestampMs) + } + // A rejected response leaves the sequence and watermarks unchanged. + this.outputBytes = outputBytes + this.nextSequence++ + this.nextInputTime = endMs + this.firstInputTime = firstInputTime + this.lastReadingTimes = lastTimes + return { frame, result, endMs } + } +} diff --git a/src/biometrics/packages/tests/fixture.ts b/src/biometrics/packages/tests/fixture.ts new file mode 100644 index 000000000..41affcbee --- /dev/null +++ b/src/biometrics/packages/tests/fixture.ts @@ -0,0 +1,17 @@ +import { createHash } from 'node:crypto' + +export function packageFixture() { + const files = new Map([['engine.py', Buffer.from('raise RuntimeError("must never execute during intake")')], ['uv.lock', Buffer.from('version = 1')]]) + const manifest = { + schema: 'biometric-algorithm-package-v1', canonicalization: 'finite-binary64-json-v1', engine: { id: 'test.hr', version: '1.0.0' }, + runtime: 'python-frame-v1', entrypoint: 'engine.py', dependencyLock: 'uv.lock', + inputSchema: 'biometric-signal-frame-v1', outputSchema: 'biometric-metric-frame-v1', + metrics: [{ id: 'heart_rate', definition: 'Heart beats per minute', unit: 'bpm', minimum: 20, maximum: 240 }], + hardware: [{ id: 'pod4', sampleRatesHz: [500] }], temporal: { mode: 'causal', lookaheadMs: 0 }, + state: { schemaId: 'test.hr.state.v1', initialization: 'explicit_seed_and_config', reset: 'reset_before_each_independent_run', serialization: 'json', maximumBytes: 10000 }, + resources: { cpuMs: 1000, wallMs: 2000, memoryBytes: 100000000, processes: 1, outputBytes: 10000 }, + artifacts: [...files].map(([path, bytes]) => ({ path, bytes: bytes.byteLength, sha256: createHash('sha256').update(bytes).digest('hex') })), + } + return { files, manifest } +} + diff --git a/src/biometrics/packages/tests/manifest.test.ts b/src/biometrics/packages/tests/manifest.test.ts new file mode 100644 index 000000000..2b39ec749 --- /dev/null +++ b/src/biometrics/packages/tests/manifest.test.ts @@ -0,0 +1,66 @@ +import { packageFixture as fixture } from './fixture' +import { describe, expect, it } from 'vitest' +import { algorithmPackageManifestSchema, maximumPackageBytes, verifyAlgorithmPackage } from '../manifest' + +describe('inert algorithm package intake', () => { + it('verifies executable bytes without executing them and binds all declarations to the package identity', () => { + const { files, manifest } = fixture() + const original = verifyAlgorithmPackage(manifest, files) + expect(original.status).toBe('inert_proposal') + expect(verifyAlgorithmPackage(JSON.parse(JSON.stringify(manifest)), files).packageHash).toBe(original.packageHash) + manifest.resources.cpuMs++ + expect(verifyAlgorithmPackage(manifest, files).packageHash).not.toBe(original.packageHash) + }) + it('rejects substituted code and locks even when the file size is unchanged', () => { + for (const path of ['engine.py', 'uv.lock']) { + const { files, manifest } = fixture() + const original = files.get(path) + if (!original) throw new Error('Missing fixture file') + files.set(path, Buffer.alloc(original.length, 65)) + expect(() => verifyAlgorithmPackage(manifest, files)).toThrow('hash mismatch') + } + }) + it('rejects missing, extra, truncated and oversized files', () => { + const { files, manifest } = fixture() + files.set('extra.py', Buffer.from('pass')) + expect(() => verifyAlgorithmPackage(manifest, files)).toThrow('file set') + files.delete('extra.py') + files.set('engine.py', Buffer.from('pass')) + expect(() => verifyAlgorithmPackage(manifest, files)).toThrow('size mismatch') + files.delete('engine.py') + expect(() => verifyAlgorithmPackage(manifest, files)).toThrow('file set') + manifest.artifacts[0].bytes = maximumPackageBytes + expect(() => verifyAlgorithmPackage(manifest, fixture().files)).toThrow('byte limit') + }) + it.each(['../engine.py', '/engine.py', 'a/../../engine.py', 'a\\engine.py', 'a//engine.py', './engine.py', 'a/engine.py\u0000'])('rejects unsafe artifact path %s', (path) => { + const { manifest } = fixture() + manifest.artifacts[0].path = path + manifest.entrypoint = path + expect(algorithmPackageManifestSchema.safeParse(manifest).success).toBe(false) + }) + it('requires canonical units and explicit HRV windows for comparable metrics', () => { + const { manifest } = fixture() + const metric = manifest.metrics[0] + const parse = (unit: string, reference: unknown) => algorithmPackageManifestSchema.safeParse({ ...manifest, metrics: [{ ...metric, unit, reference }] }).success + expect(parse('bpm', { metric: 'heart_rate', windowMs: null })).toBe(true) + expect(parse('ms', { metric: 'hrv_rmssd', windowMs: 300000 })).toBe(true) + expect(parse('breaths/min', { metric: 'breathing_rate', windowMs: null })).toBe(true) + expect(parse('bpm', { metric: 'breathing_rate', windowMs: null })).toBe(false) + expect(parse('bpm', { metric: 'hrv_rmssd', windowMs: 300000 })).toBe(false) + expect(parse('ms', { metric: 'hrv_rmssd', windowMs: null })).toBe(false) + expect(parse('ms', { metric: 'hrv_unknown', windowMs: 300000 })).toBe(false) + }) + it('rejects ambiguous identity, unhashed entrypoints, missing locks and unknown privileges', () => { + const { manifest } = fixture() + for (const altered of [ + { ...manifest, metrics: [...manifest.metrics, manifest.metrics[0]] }, + { ...manifest, artifacts: [...manifest.artifacts, manifest.artifacts[0]] }, + { ...manifest, hardware: [...manifest.hardware, manifest.hardware[0]] }, + { ...manifest, hardware: [{ id: 'pod4', sampleRatesHz: [500, 500] }] }, + { ...manifest, entrypoint: 'other.py' }, { ...manifest, dependencyLock: 'absent.lock' }, + { ...manifest, dependencyLock: manifest.entrypoint }, { ...manifest, network: true }, + { ...manifest, resources: { ...manifest.resources, processes: 0 } }, + { ...manifest, temporal: { mode: 'causal', lookaheadMs: 1000 } }, + ]) expect(algorithmPackageManifestSchema.safeParse(altered).success).toBe(false) + }) +}) diff --git a/src/biometrics/packages/tests/protocol.test.ts b/src/biometrics/packages/tests/protocol.test.ts new file mode 100644 index 000000000..85870d078 --- /dev/null +++ b/src/biometrics/packages/tests/protocol.test.ts @@ -0,0 +1,83 @@ +import { describe, expect, it } from 'vitest' +import { canonicalJson, configHash, dataHash } from '../../algorithms/canonical' +import { PackageFrameValidator } from '../protocol' +import { packageFixture } from './fixture' + +function fixture() { + const { manifest } = packageFixture() + const request = { schema: 'biometric-package-run-v1', packageHash: configHash(manifest), inputHash: 'a'.repeat(64), hardwareId: 'pod4', sampleRateHz: 500, side: 'left', seed: 123, config: {} } + const frame = { schema: 'biometric-signal-frame-v1', sequence: 0, timestampMs: 1000, samples: { left: [1, 2, 3], right: null }, context: { occupied: null, pumpActive: null } } + const validator = new PackageFrameValidator(manifest, request) + const result = { schema: 'biometric-metric-frame-v1', runHash: validator.runHash, frameHash: dataHash(frame), sequence: 0, + readings: [{ timestampMs: 1004, side: 'left', metric: 'heart_rate', value: 60, quality: 0.9, flags: [] }], state: { count: 3 } } + return { manifest, request, frame, result, validator } +} + +describe('algorithm frame exchange', () => { + it('accepts deterministic identities and ordered frames with explicit unknown context', () => { + const { frame, result, validator } = fixture() + expect(validator.validateStep(frame, result).endMs).toBe(1006) + const next = { ...frame, sequence: 1, timestampMs: 1006 } + expect(validator.validateStep(next, { ...result, frameHash: dataHash(next), sequence: 1, readings: [] }).endMs).toBe(1012) + }) + it('binds the run to package, input, configuration and deterministic seed', () => { + const { manifest, request, validator } = fixture() + for (const changed of [{ ...request, seed: 124 }, { ...request, config: { threshold: 10 } }, { ...request, inputHash: 'b'.repeat(64) }]) { + expect(new PackageFrameValidator(manifest, changed).runHash).not.toBe(validator.runHash) + } + for (const changed of [{ ...request, packageHash: 'b'.repeat(64) }, { ...request, hardwareId: 'other' }, { ...request, sampleRateHz: 100 }]) { + expect(() => new PackageFrameValidator(manifest, changed)).toThrow() + } + }) + it('rejects output substitution and leaves the run retryable', () => { + const { frame, result, validator } = fixture() + for (const changed of [{ ...result, runHash: 'b'.repeat(64) }, { ...result, frameHash: 'b'.repeat(64) }, { ...result, sequence: 1 }]) { + expect(() => validator.validateStep(frame, changed)).toThrow('identity') + } + expect(validator.validateStep(frame, result).endMs).toBe(1006) + }) + it('rejects future, undeclared, out-of-range and duplicated readings', () => { + const { frame, result } = fixture() + const reading = result.readings[0] + for (const readings of [[{ ...reading, timestampMs: 1006 }], [{ ...reading, timestampMs: 999 }], [{ ...reading, side: 'right' }], + [{ ...reading, metric: 'hrv' }], [{ ...reading, value: 300 }], [reading, reading]]) { + expect(() => fixture().validator.validateStep(frame, { ...result, readings })).toThrow() + } + }) + it('rejects overlapping frames, missing channels and inconsistent channel lengths', () => { + const { frame, result, validator } = fixture() + for (const changed of [{ ...frame, sequence: 1 }, { ...frame, samples: { left: null, right: [1] } }, { ...frame, samples: { left: [1], right: [1, 2] } }]) { + expect(() => validator.validateStep(changed, result)).toThrow() + } + expect(() => validator.validateFrame({ ...frame, timestampMs: Number.MAX_SAFE_INTEGER })).toThrow('timestamp range') + validator.validateStep(frame, result) + expect(() => validator.validateStep({ ...frame, sequence: 1, timestampMs: 1005 }, result)).toThrow('overlapping') + const next = { ...frame, sequence: 1, timestampMs: 1006 } + expect(() => validator.validateStep(next, { ...result, sequence: 1, frameHash: dataHash(next) })).toThrow('Duplicate') + }) + it('enforces declared lookahead without accepting future readings', () => { + const { manifest, request, frame, result } = fixture() + const retrospective = { ...manifest, temporal: { mode: 'retrospective', lookaheadMs: 4 } } + const validator = new PackageFrameValidator(retrospective, { ...request, packageHash: configHash(retrospective) }) + const output = { ...result, runHash: validator.runHash } + expect(() => validator.validateStep(frame, output)).toThrow('time range') + expect(validator.validateStep(frame, { ...output, readings: [{ ...output.readings[0], timestampMs: 1000 }] })).toBeDefined() + }) + it('enforces the cumulative output budget across individually small responses', () => { + const { manifest, request, frame, result } = fixture() + manifest.resources.outputBytes = Buffer.byteLength(canonicalJson(result)) + 10 + const validator = new PackageFrameValidator(manifest, { ...request, packageHash: configHash(manifest) }) + validator.validateStep(frame, { ...result, runHash: validator.runHash }) + const next = { ...frame, sequence: 1, timestampMs: 1006 } + expect(() => validator.validateStep(next, { ...result, runHash: validator.runHash, frameHash: dataHash(next), sequence: 1, readings: [] })).toThrow('Output byte limit') + }) + it('bounds state and response sizes and rejects non-JSON state', () => { + for (const kind of ['state', 'output', 'non-json']) { + const { manifest, request, frame, result } = fixture() + if (kind === 'state') manifest.state.maximumBytes = 2 + if (kind === 'output') manifest.resources.outputBytes = 2 + const validator = new PackageFrameValidator(manifest, { ...request, packageHash: configHash(manifest) }) + expect(() => validator.validateStep(frame, { ...result, runHash: validator.runHash, state: kind === 'non-json' ? new Date() : result.state })).toThrow() + } + }) +}) diff --git a/src/biometrics/profiles/activation.ts b/src/biometrics/profiles/activation.ts new file mode 100644 index 000000000..fbb2152cc --- /dev/null +++ b/src/biometrics/profiles/activation.ts @@ -0,0 +1,130 @@ +import { randomUUID } from 'node:crypto' +import { and, eq } from 'drizzle-orm' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import { z } from 'zod' +import type * as schema from '@/src/db/schema' +import type * as bioSchema from '@/src/db/biometrics-schema' +import { biometricActivationEvents as events, biometricConfigOutbox as outbox, biometricProfileBindings as bindings, sideSettings } from '@/src/db/schema' +import { defaultConfig, manifestHash } from '../algorithms/registry' +import { configHash } from '../algorithms/canonical' +import { ReferenceInputError } from '../references/time' +import { stageClassifierSchema } from '@/src/lib/sleep-stage-rules' +import { ProfileStore } from './store' +import { prepareVerifiedEvaluation, type PromotionBuild } from './evaluation-store' + +export const activationIntentSchema = z.object({ + side: z.enum(['left', 'right']), action: z.enum(['activate', 'reset', 'rollback']), + revisionId: z.string().uuid().nullable(), expectedGeneration: z.number().int().nonnegative(), + expectedResolvedHash: z.string().regex(/^[a-f0-9]{64}$/), boundary: z.enum(['immediate', 'next_session']), + requester: z.string().min(1).max(200), idempotencyKey: z.string().min(1).max(200), + allowUnvalidated: z.literal(true), +}).strict() + +export const validatedActivationIntentSchema = activationIntentSchema.omit({ allowUnvalidated: true }).extend({ + action: z.literal('activate'), revisionId: z.string().uuid(), evaluationId: z.string().uuid(), + expectedReportHash: z.string().regex(/^[a-f0-9]{64}$/), +}).strict() + +export const validatedActivationReviewSchema = validatedActivationIntentSchema.omit({ action: true, boundary: true, requester: true, idempotencyKey: true }) + +/** A point-in-time review only; application repeats every verification. */ +export async function reviewValidatedActivation(db: BetterSQLite3Database, bioDb: BetterSQLite3Database, + raw: z.input, resolveBuild: () => Promise) { + const input = validatedActivationReviewSchema.parse(raw) + const verified = prepareVerifiedEvaluation(db, bioDb, input, await resolveBuild()) + verified.recheck() + return { eligible: true as const } +} + +function retryActivation(db: BetterSQLite3Database, idempotencyKey: string, requestHash: string) { + const existing = db.select().from(events).where(eq(events.idempotencyKey, idempotencyKey)).get() + if (existing && existing.requestHash !== requestHash) throw new ReferenceInputError('REVISION_CONFLICT', 'Activation idempotency key changed') + return existing +} + +export function requestManualActivation(db: BetterSQLite3Database, raw: z.input, now = Date.now()) { + const input = activationIntentSchema.parse(raw) + return commitActivationIntent(db, input, configHash(input), now) +} + +/** Configuration is the only database written here. The short biometrics write + * reservation prevents exposure/release/runtime changes across the config + * commit; it is not a transaction committing writes to two databases. */ +export function requestValidatedActivation(db: BetterSQLite3Database, bioDb: BetterSQLite3Database, + raw: z.input, build: PromotionBuild, now = Date.now()) { + const input = validatedActivationIntentSchema.parse(raw) + const requestHash = configHash(input) + const existing = retryActivation(db, input.idempotencyKey, requestHash) + if (existing) return existing + const verified = prepareVerifiedEvaluation(db, bioDb, input, build) + const intent = activationIntentSchema.strip().parse({ ...input, allowUnvalidated: true }) + return bioDb.transaction(() => commitActivationIntent(db, intent, requestHash, now, + { ...verified, evaluationId: input.evaluationId }), { behavior: 'immediate' }) +} + +/** Retry before artifact/build access. Recheck again after the async probe so + * concurrent identical requests still create only one activation event. */ +export async function requestValidatedActivationWithBuild(db: BetterSQLite3Database, bioDb: BetterSQLite3Database, + raw: z.input, resolveBuild: () => Promise, now = Date.now()) { + const input = validatedActivationIntentSchema.parse(raw) + const existing = retryActivation(db, input.idempotencyKey, configHash(input)) + if (existing) return existing + const build = await resolveBuild() + return requestValidatedActivation(db, bioDb, input, build, now) +} + +function commitActivationIntent(db: BetterSQLite3Database, input: z.infer, requestHash: string, now: number, + verified?: ReturnType & { evaluationId: string }) { + return db.transaction((tx) => { + const existing = retryActivation(db, input.idempotencyKey, requestHash) + if (existing) return existing + const profiles = new ProfileStore(db) + const storedBinding = tx.select().from(bindings).where(eq(bindings.side, input.side)).get() + const binding = profiles.binding(input.side) + let invalidLegacy: unknown + if (!storedBinding) { + const legacy = tx.select({ classifier: sideSettings.stageClassifier }).from(sideSettings).where(eq(sideSettings.side, input.side)).get() + if (legacy?.classifier != null) { + // Valid customizations must still migrate losslessly. Malformed legacy + // data cannot become a revision; an explicit reset preserves it as audit + // evidence and leaves the original legacy row untouched. + if (input.action !== 'reset' || stageClassifierSchema.safeParse(legacy.classifier).success) { + throw new ReferenceInputError('LEGACY_MIGRATION_REQUIRED', 'Preserve the existing stage profile before activation') + } + invalidLegacy = legacy.classifier + } + } + if (binding.generation !== input.expectedGeneration) throw new ReferenceInputError('STALE_BINDING', 'Binding changed; inspect the side before applying this revision') + if (input.action === 'reset' && input.revisionId !== null) throw new ReferenceInputError('INVALID_SELECTION', 'Reset must select system defaults') + if (input.action === 'activate' && input.revisionId === null) throw new ReferenceInputError('INVALID_SELECTION', 'Activation must select an exact revision; use reset for defaults') + if (input.action === 'rollback' && (binding.generation === 0 || input.revisionId !== binding.previousRevisionId)) throw new ReferenceInputError('INVALID_SELECTION', 'Rollback must select the previous binding') + const selected = input.revisionId === null + ? { resolved: defaultConfig, hash: configHash(defaultConfig) } + : profiles.validateRevision(input.revisionId, input.side) + if (selected.hash !== input.expectedResolvedHash) throw new ReferenceInputError('REVISION_CONFLICT', 'Resolved configuration changed; inspect its exact hash before applying') + const previous = profiles.effective(input.side) + verified?.recheck() + const generation = binding.generation + 1 + if (!storedBinding) { + tx.insert(bindings).values({ side: input.side, generation, desiredRevisionId: input.revisionId, + previousRevisionId: binding.desiredRevisionId, updatedAtMs: now }).run() + } + else { + const changed = tx.update(bindings).set({ generation, desiredRevisionId: input.revisionId, + previousRevisionId: binding.desiredRevisionId, updatedAtMs: now }) + .where(and(eq(bindings.side, input.side), eq(bindings.generation, input.expectedGeneration))).run() + if (changed.changes !== 1) throw new ReferenceInputError('STALE_BINDING', 'Binding changed during activation') + } + const event = tx.insert(events).values({ id: randomUUID(), side: input.side, generation, action: input.action, + previousRevisionId: binding.desiredRevisionId, nextRevisionId: input.revisionId, previousHash: previous.hash, nextHash: selected.hash, + evaluationId: verified?.evaluationId ?? null, + evidence: verified?.evidence ?? { status: 'manual_unvalidated', manifestHash, resolvedHash: selected.hash, + ...(invalidLegacy === undefined ? {} : { recovery: 'invalid_legacy_reset', preservedLegacyClassifier: invalidLegacy }) }, requester: input.requester, + state: 'pending_delivery', idempotencyKey: input.idempotencyKey, requestHash, createdAtMs: now, updatedAtMs: now }).returning().get() + tx.insert(outbox).values({ id: randomUUID(), side: input.side, generation, + payload: { schemaVersion: 1, side: input.side, generation, revisionId: input.revisionId, manifestHash, + resolved: selected.resolved, resolvedHash: selected.hash, boundary: input.boundary }, + resolvedHash: selected.hash, state: 'pending', nextAttemptAtMs: now, createdAtMs: now }).run() + return event + }, { behavior: 'immediate' }) +} diff --git a/src/biometrics/profiles/contracts.ts b/src/biometrics/profiles/contracts.ts new file mode 100644 index 000000000..cb04673e0 --- /dev/null +++ b/src/biometrics/profiles/contracts.ts @@ -0,0 +1,40 @@ +import { z } from 'zod' +import { stageViewSchema } from './stage-view' +import { runtimeHealthSchema } from './health' +import { stageClassifierSchema } from '@/src/lib/sleep-stage-rules' +import { biometricManifest, validateConfig, type BiometricConfig } from '../algorithms/registry' + +// The canonical manifest remains the source of parameter bounds. The typed +// adapter supplies UI inference while the shared validator checks interactions. +export const biometricConfigSchema = (z.fromJSONSchema(biometricManifest.parameters as unknown as z.core.JSONSchema.JSONSchema) as z.ZodType) + .superRefine((value, context) => { + try { + validateConfig(value) + } + catch { + context.addIssue({ code: 'custom', message: 'Invalid biometric parameter relationship' }) + } + }) +export const overrideSchema = z.object({ path: z.string(), value: z.unknown() }) +export const revisionSchema = z.object({ + id: z.string().uuid(), familyId: z.string().uuid(), revision: z.number().int(), side: z.enum(['left', 'right']), + baseManifestHash: z.string(), baseManifest: z.unknown(), overrides: z.array(overrideSchema), + resolved: z.unknown(), resolvedHash: z.string(), dependencyHashes: z.record(z.string(), z.string()), + parentRevisionId: z.string().uuid().nullable(), origin: z.enum(['manual', 'migration', 'agent', 'system']), + requester: z.string(), idempotencyKey: z.string(), requestHash: z.string(), createdAtMs: z.number(), +}) +export const bindingSchema = z.object({ side: z.enum(['left', 'right']), generation: z.number().int(), + desiredRevisionId: z.string().uuid().nullable(), previousRevisionId: z.string().uuid().nullable(), updatedAtMs: z.number() }) +export const profileOverviewSchema = z.object({ + stageView: stageViewSchema, + desired: z.object({ binding: bindingSchema, resolved: biometricConfigSchema, hash: z.string(), isDefault: z.boolean(), warning: z.string().nullable() }), + legacyStageView: z.object({ classifier: stageClassifierSchema, personalized: z.boolean(), migrationPending: z.boolean() }), + runtimeHealth: runtimeHealthSchema, + runtimeStatus: z.enum(['unreported', 'receipts_available']), + receipts: z.array(z.object({ component: z.string(), generation: z.number(), appliedHash: z.string(), effectiveSampleTimeMs: z.number().nullable(), + state: z.enum(['pending_boundary', 'applied', 'failed']), error: z.string().nullable(), heartbeatAtMs: z.number() })), + sensorCalibration: z.array(z.object({ id: z.number(), sensorType: z.string(), status: z.string(), qualityScore: z.number().nullable(), + createdAtMs: z.number(), expiresAtMs: z.number().nullable(), freshness: z.enum(['expired', 'current', 'unknown']), + sourceWindowStartMs: z.number().nullable(), sourceWindowEndMs: z.number().nullable() })), + activationAvailable: z.boolean(), limitations: z.array(z.string()), +}) diff --git a/src/biometrics/profiles/diff.ts b/src/biometrics/profiles/diff.ts new file mode 100644 index 000000000..e7437da6c --- /dev/null +++ b/src/biometrics/profiles/diff.ts @@ -0,0 +1,26 @@ +import { configHash } from '../algorithms/canonical' +import { defaultConfig, type BiometricConfig, type Override } from '../algorithms/registry' +import type { ProfileStore } from './store' + +export function diffProfiles(store: ProfileStore, side: 'left' | 'right', revisionId: string | null, againstRevisionId: string | null) { + const load = (id: string | null) => { + if (id === null) return { resolved: defaultConfig, hash: configHash(defaultConfig), overrides: [] as Override[] } + const validated = store.validateRevision(id, side) + return { resolved: validated.resolved, hash: validated.hash, overrides: validated.revision.overrides as Override[] } + } + const before = load(againstRevisionId), after = load(revisionId) + const changes: Array<{ path: string, before: unknown, after: unknown }> = [] + const walk = (left: unknown, right: unknown, path: string) => { + if (left !== undefined && right !== undefined && configHash(left) === configHash(right)) return + if (left !== null && right !== null && typeof left === 'object' && typeof right === 'object' && !Array.isArray(left) && !Array.isArray(right)) { + for (const key of new Set([...Object.keys(left), ...Object.keys(right)])) walk((left as Record)[key], (right as Record)[key], path ? `${path}.${key}` : key) + } + else changes.push({ path, before: left, after: right }) + } + walk(before.resolved as BiometricConfig, after.resolved as BiometricConfig, '') + const oldIntent = new Map(before.overrides.map(row => [row.path, row.value])) + const newIntent = new Map(after.overrides.map(row => [row.path, row.value])) + const intentChangedPaths = [...new Set([...oldIntent.keys(), ...newIntent.keys()])].filter(path => + !oldIntent.has(path) || !newIntent.has(path) || configHash(oldIntent.get(path)) !== configHash(newIntent.get(path))) + return { revisionId, againstRevisionId, beforeHash: before.hash, afterHash: after.hash, changes, intentChangedPaths } +} diff --git a/src/biometrics/profiles/evaluation-assessment.ts b/src/biometrics/profiles/evaluation-assessment.ts new file mode 100644 index 000000000..2de6e5c07 --- /dev/null +++ b/src/biometrics/profiles/evaluation-assessment.ts @@ -0,0 +1,103 @@ +import { verifyReplayContext, type ReplayContextQualification } from './replay-context' +import { numericObjectivePolicies } from '../backtest/contracts' +import { z } from 'zod' +import { configHash } from '../algorithms/canonical' +import { validateConfig } from '../algorithms/registry' +import { finalizeObjectiveReport, finalizeStageObjectiveReport } from '../backtest/objective-finalization' +import type { ObjectivePolicy } from '../backtest/objective-policy' +import type { StageObjectivePolicy } from '../backtest/stage-objective' +import { ReferenceInputError } from '../references/time' +import { verifyEvaluationIdentity } from './evaluation-identity' + +const assessmentSchema = z.object({ + baseline: z.enum(['default', 'current']), candidate: z.literal('candidate'), baselineHash: z.string(), candidateHash: z.string(), + status: z.enum(['improved', 'no_material_change', 'regressed', 'insufficient_evidence', 'incompatible']), + designated: z.boolean(), missingEvidence: z.array(z.string()), failedThresholds: z.array(z.string()), + evidence: z.object({ split: z.literal('holdout'), holdoutUse: z.literal(1), nights: z.array(z.object({ cohortId: z.string() })) }), +}) + +/** Internal qualified-build dependency, never populated from experiment JSON. */ +export interface DesignatedEvaluationPolicies { + /** Full frozen protocol, including alignment/calibration/uncertainty settings. */ + protocolHash: string | null + waveformContext: ReplayContextQualification | null + numeric: ObjectivePolicy | readonly ObjectivePolicy[] | null + stages: StageObjectivePolicy | null +} + +/** Reassess retained evidence; claimed report statuses are never authority. + * The caller still must commit activation under a current binding generation + * and recheck holdout exposure immediately before that commit. */ +export function assessEvaluation( + rawJob: Parameters[0], + result: Parameters[1], + expected: Parameters[2], + designations: DesignatedEvaluationPolicies, +) { + const identity = verifyEvaluationIdentity(rawJob, result, expected) + const reject = (message: string): never => { + throw new ReferenceInputError('EVALUATION_NOT_VALIDATED', message) + } + const { job } = identity + const configuration = (label: 'candidate' | 'current' | 'default') => job.configs.find(row => row.label === label) ?? reject('Missing evaluated configuration') + const candidate = configuration('candidate') + const current = configuration('current') + const after = validateConfig(candidate.resolved), before = validateConfig(current.resolved) + const domains = (Object.keys(after) as Array).filter(domain => configHash(after[domain]) !== configHash(before[domain])) + if (!domains.length) reject('Candidate does not change the current configuration') + const numericMetrics = { heart_rate: 'heart_rate', breathing_rate: 'breathing_rate', hrv_index: 'window_ibi_variability_v1' } as const + const requiredMetrics = domains.filter((domain): domain is keyof typeof numericMetrics => domain !== 'sleep_stages').map(domain => numericMetrics[domain]) + const policies = numericObjectivePolicies(job.protocol) + if (requiredMetrics.some(metric => !policies.some(policy => policy.metric === metric))) reject('Every changed numeric component requires its own objective evidence') + if (!job.protocol.stageObjectivePolicy) reject('Stage agreement and downstream effects require stage objective evidence') + if (requiredMetrics.length && (job.protocol.mode !== 'signal_replay' || job.protocol.waveformStages !== true)) reject('Numeric changes require waveform replay with downstream stages') + const raw = z.record(z.string(), z.unknown()).parse(result.report) + const numeric = finalizeObjectiveReport(raw, policies, raw.jobOperations, designations.numeric) + const finalized = finalizeStageObjectiveReport(numeric, job.protocol.stageObjectivePolicy, raw.jobOperations, designations.stages) + const outcomes: Array<{ status: string, designated: boolean, missingEvidence: string[], failedThresholds: string[], improvementRequired: boolean }> = [] + const check = (rawAssessments: unknown, improvementRequired: boolean) => { + const assessments = z.array(assessmentSchema).length(2).parse(rawAssessments) + for (const assessment of assessments) { + const baseline = configuration(assessment.baseline) + if (assessment.baselineHash !== baseline.resolvedHash || assessment.candidateHash !== candidate.resolvedHash) reject('Objective evidence belongs to another configuration') + const cohorts = assessment.evidence.nights.map(row => row.cohortId) + if (new Set(cohorts).size !== cohorts.length || cohorts.length !== identity.cohortIds.length + || cohorts.some(id => !identity.cohortIds.includes(id))) reject('Objective evidence omits frozen holdout nights') + outcomes.push({ ...assessment, improvementRequired }) + } + return assessments.map(row => ({ baseline: row.baseline, baselineHash: row.baselineHash, candidateHash: row.candidateHash, status: row.status })) + } + const numericRows = z.array(z.object({ policy: z.object({ metric: z.string() }) }).passthrough()).parse(finalized.objectiveAssessments ?? []) + const numericAssessments = policies.flatMap(policy => check(numericRows.filter(row => row.policy.metric === policy.metric), true) + .map(row => ({ ...row, metric: policy.metric }))) + const stageAssessments = check(finalized.stageObjectiveAssessments, domains.includes('sleep_stages')) + if (designations.protocolHash !== job.protocolHash) reject('Evaluation protocol is outside the qualified context') + const replayContext = expected.waveform + ? verifyReplayContext(raw.waveform, { ...expected.waveform, side: expected.side, + cohortIds: identity.cohortIds, configHashes: job.configs.map(config => config.resolvedHash) }, designations.waveformContext) + : null + const status = outcomes.some(row => row.status === 'incompatible') + ? 'incompatible' as const + : outcomes.some(row => !row.designated || row.missingEvidence.length || row.status === 'insufficient_evidence') + ? 'insufficient_evidence' as const + : outcomes.some(row => row.failedThresholds.length || row.status === 'regressed') + ? 'regressed' as const + : outcomes.every(row => row.status === 'improved' || (!row.improvementRequired && row.status === 'no_material_change')) + ? 'improved' as const + : 'no_material_change' as const + return { status, audit: { version: 'verified-evaluation-v1' as const, interpretation: 'agreement_with_named_reference_not_clinical_validation' as const, + reportHash: result.contentHash, revisionId: expected.revisionId, resolvedHash: expected.resolvedHash, + currentResolvedHash: expected.currentResolvedHash, datasetId: job.datasetId, datasetHash: job.datasetHash, + evaluatorBuildHash: job.evaluatorBuildHash, manifestHash: identity.report.manifestHash, protocolHash: job.protocolHash, + changedDomains: domains, numericAssessments, stageAssessments, replayContext, + policies: { numeric: job.protocol.objectivePolicies ?? job.protocol.objectivePolicy ?? null, stages: job.protocol.stageObjectivePolicy }, + ...(expected.waveform ? { waveform: expected.waveform } : {}), + } } +} + +/** Activation requires the independently reassessed whole candidate to improve. */ +export function verifyEvaluationAssessments(...args: Parameters) { + const result = assessEvaluation(...args) + if (result.status !== 'improved') throw new ReferenceInputError('EVALUATION_NOT_VALIDATED', 'Objective evidence does not support this activation') + return result.audit +} diff --git a/src/biometrics/profiles/evaluation-identity.ts b/src/biometrics/profiles/evaluation-identity.ts new file mode 100644 index 000000000..b8a8fe715 --- /dev/null +++ b/src/biometrics/profiles/evaluation-identity.ts @@ -0,0 +1,73 @@ +import { z } from 'zod' +import { configHash, dataHash } from '../algorithms/canonical' +import { defaultConfig, manifestHash, validateConfig } from '../algorithms/registry' +import { backtestJobSnapshotSchema, experimentProtocolSchema } from '../backtest/contracts' +import { ReferenceInputError } from '../references/time' + +const hash = z.string().regex(/^[a-f0-9]{64}$/) +const reportIdentitySchema = z.object({ + version: z.enum(['waveform-backtest-report-v1', 'processed-backtest-report-v1']), + side: z.enum(['left', 'right']), datasetId: z.string().uuid(), datasetHash: hash, + protocol: experimentProtocolSchema, protocolHash: hash, evaluatorBuildHash: hash, manifestHash: hash, + split: z.literal('holdout'), holdoutUse: z.literal(1), + support: z.enum(['identical_retained_waveform_inputs', 'identical_retained_epoch_intervals']), + nights: z.array(z.object({ cohortId: z.string() })).min(1).max(60), + waveform: z.object({ pythonBuildHash: hash, otherSideConfigHash: hash }).optional(), +}) + +/** These expectations are loaded by the service from the profile binding, + * immutable dataset and installed build. They must never come from a client. */ +export interface EvaluationIdentityExpectation { + side: 'left' | 'right' + revisionId: string + resolvedHash: string + currentResolvedHash: string + evaluatorBuildHash: string + dataset: { id: string, contentHash: string, side: 'left' | 'right', available: boolean, holdoutCohortIds: string[] } + waveform?: { pythonBuildHash: string, otherSideConfigHash: string } +} + +/** Necessary identity checks only: this does not designate a policy, award + * improvement, or authorize activation. Assessments must be verified next. */ +export function verifyEvaluationIdentity(rawJob: unknown, result: { + report: unknown + contentHash: string + available: boolean + currentHoldoutUseCount: number +}, expected: EvaluationIdentityExpectation) { + const reject = (message: string): never => { + throw new ReferenceInputError('EVALUATION_MISMATCH', message) + } + if (!result.available || !expected.dataset.available) reject('Retained evaluation artifacts are unavailable') + if (dataHash(result.report) !== result.contentHash) reject('Evaluation content hash does not match') + const job = backtestJobSnapshotSchema.parse(rawJob) + const report = reportIdentitySchema.parse(result.report) + if (job.split !== 'holdout' || job.holdoutUse !== 1 || result.currentHoldoutUseCount !== 1 || job.evidence === 'reused_holdout_is_validation') reject('Independent holdout has been reused or was not evaluated') + if (report.side !== expected.side || expected.dataset.side !== expected.side + || job.datasetId !== expected.dataset.id || report.datasetId !== job.datasetId + || job.datasetHash !== expected.dataset.contentHash || report.datasetHash !== job.datasetHash) reject('Evaluation dataset or side changed') + if (job.evaluatorBuildHash !== expected.evaluatorBuildHash || report.evaluatorBuildHash !== job.evaluatorBuildHash + || report.manifestHash !== manifestHash) reject('Evaluation build is incompatible') + if (configHash(job.protocol) !== job.protocolHash || report.protocolHash !== job.protocolHash + || configHash(report.protocol) !== job.protocolHash) reject('Evaluation protocol identity changed') + if (job.configs.length !== 3 || new Set(job.configs.map(row => row.label)).size !== 3) reject('Evaluation must include default, current and candidate') + for (const config of job.configs) { + if (config.manifestHash !== manifestHash || configHash(validateConfig(config.resolved)) !== config.resolvedHash) reject('Evaluated configuration identity changed') + if (config.label === 'default' && (config.revisionId !== null || config.resolvedHash !== configHash(defaultConfig))) reject('Evaluated default does not match the shipped default') + if (config.label === 'current' && config.resolvedHash !== expected.currentResolvedHash) reject('Current baseline changed since evaluation') + if (config.label === 'candidate' && (config.revisionId !== expected.revisionId || config.resolvedHash !== expected.resolvedHash)) reject('Evaluation belongs to another candidate revision') + } + const cohorts = report.nights.map(row => row.cohortId) + if (new Set(cohorts).size !== cohorts.length || new Set(expected.dataset.holdoutCohortIds).size !== expected.dataset.holdoutCohortIds.length + || cohorts.length !== expected.dataset.holdoutCohortIds.length || cohorts.some(id => !expected.dataset.holdoutCohortIds.includes(id))) reject('Evaluation does not cover the exact frozen holdout nights') + if (job.protocol.mode === 'signal_replay') { + if (!expected.waveform || !job.waveform || !report.waveform + || job.waveform.pythonBuildHash !== expected.waveform.pythonBuildHash || report.waveform.pythonBuildHash !== job.waveform.pythonBuildHash + || job.waveform.otherSide.resolvedHash !== expected.waveform.otherSideConfigHash || report.waveform.otherSideConfigHash !== job.waveform.otherSide.resolvedHash + || job.waveform.otherSide.manifestHash !== manifestHash || configHash(validateConfig(job.waveform.otherSide.resolved)) !== job.waveform.otherSide.resolvedHash + || report.version !== 'waveform-backtest-report-v1' || report.support !== 'identical_retained_waveform_inputs') reject('Waveform dependencies changed since evaluation') + } + else if (job.protocol.mode !== 'stage_replay' || job.waveform || report.waveform + || report.version !== 'processed-backtest-report-v1' || report.support !== 'identical_retained_epoch_intervals') reject('Evaluation mode cannot validate the selected replay') + return { job, report, contentHash: result.contentHash, cohortIds: cohorts } +} diff --git a/src/biometrics/profiles/evaluation-store.ts b/src/biometrics/profiles/evaluation-store.ts new file mode 100644 index 000000000..9b8ba9a0e --- /dev/null +++ b/src/biometrics/profiles/evaluation-store.ts @@ -0,0 +1,86 @@ +import { eq } from 'drizzle-orm' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import type * as configSchema from '@/src/db/schema' +import * as bioSchema from '@/src/db/biometrics-schema' +import { configHash } from '../algorithms/canonical' +import { BacktestJobStore } from '../backtest/jobs' +import { DatasetStore } from '../backtest/datasets' +import { backtestJobSnapshotSchema, datasetManifestSchema } from '../backtest/contracts' +import { ReferenceInputError } from '../references/time' +import { verifyEvaluationAssessments, type DesignatedEvaluationPolicies } from './evaluation-assessment' +import { ProfileStore } from './store' +import { resolveReplayBaseline } from './replay-baseline' + +type ConfigDB = BetterSQLite3Database +type BioDB = BetterSQLite3Database +export interface EvaluationSelection { + side: 'left' | 'right' + revisionId: string + expectedResolvedHash: string + expectedGeneration: number + evaluationId: string + expectedReportHash: string +} +/** Supplied by the installed build, never an RPC request. */ +export interface PromotionBuild { + evaluatorBuildHash: string + pythonBuildHash?: string + designations: DesignatedEvaluationPolicies +} + +function baseline(configDb: ConfigDB, bioDb: BioDB, side: 'left' | 'right') { + const desired = new ProfileStore(configDb).effective(side) + const actual = resolveReplayBaseline(configDb, bioDb, side) + if (desired.warning || (actual.provenance.warning && desired.binding.generation > 0) + || desired.hash !== actual.hash || (desired.binding.generation > 0 && actual.provenance.generation !== desired.binding.generation)) { + throw new ReferenceInputError('BASELINE_PENDING', 'Resolve pending or incompatible runtime configuration before validated activation') + } + return actual +} + +/** Full retained artifact parsing/hashing runs before the short commit locks. + * The returned guard must run while both databases are held by the caller. */ +export function prepareVerifiedEvaluation(configDb: ConfigDB, bioDb: BioDB, input: EvaluationSelection, build: PromotionBuild) { + const profiles = new ProfileStore(configDb), jobs = new BacktestJobStore(bioDb) + const revision = profiles.validateRevision(input.revisionId, input.side) + if (revision.hash !== input.expectedResolvedHash) throw new ReferenceInputError('REVISION_CONFLICT', 'Candidate hash changed') + if (profiles.binding(input.side).generation !== input.expectedGeneration) throw new ReferenceInputError('STALE_BINDING', 'Binding changed before verification') + const storedJob = jobs.get(input.evaluationId) + const job = backtestJobSnapshotSchema.parse(storedJob.input) + const experiment = jobs.getExperiment(storedJob.experimentId) + if (job.experimentId !== experiment.id || experiment.datasetId !== job.datasetId || experiment.side !== input.side + || experiment.protocolHash !== job.protocolHash || configHash(experiment.protocol) !== job.protocolHash) throw new ReferenceInputError('EVALUATION_MISMATCH', 'Stored experiment and job disagree') + const result = jobs.result(input.evaluationId) + if (!result || result.contentHash !== input.expectedReportHash) throw new ReferenceInputError('EVALUATION_MISMATCH', 'Completed evaluation hash does not match') + const dataset = new DatasetStore(bioDb).read(job.datasetId) + const manifest = datasetManifestSchema.parse(dataset.manifest) + const current = baseline(configDb, bioDb, input.side) + const otherSide = input.side === 'left' ? 'right' : 'left' + const other = job.protocol.mode === 'signal_replay' ? baseline(configDb, bioDb, otherSide) : null + const evidence = verifyEvaluationAssessments(job, result, { + side: input.side, revisionId: input.revisionId, resolvedHash: revision.hash, currentResolvedHash: current.hash, + evaluatorBuildHash: build.evaluatorBuildHash, + dataset: { id: dataset.id, contentHash: dataset.contentHash, side: dataset.side, available: dataset.available, + holdoutCohortIds: manifest.splitProtocol.assignments.filter(row => row.split === 'holdout').map(row => row.cohortId) }, + ...(other && build.pythonBuildHash ? { waveform: { pythonBuildHash: build.pythonBuildHash, otherSideConfigHash: other.hash } } : {}), + }, build.designations) + return { + evidence: { status: 'validated_reference_agreement' as const, evaluationId: input.evaluationId, ...evidence }, + recheck() { + const latestJob = bioDb.select({ state: bioSchema.biometricBacktestJobs.state, fencingToken: bioSchema.biometricBacktestJobs.fencingToken }) + .from(bioSchema.biometricBacktestJobs).where(eq(bioSchema.biometricBacktestJobs.id, input.evaluationId)).get() + const latestResult = bioDb.select({ fencingToken: bioSchema.biometricBacktestResults.fencingToken, + contentHash: bioSchema.biometricBacktestResults.contentHash, releasedAtMs: bioSchema.biometricBacktestResults.releasedAtMs }) + .from(bioSchema.biometricBacktestResults).where(eq(bioSchema.biometricBacktestResults.jobId, input.evaluationId)).get() + const latestDataset = bioDb.select({ contentHash: bioSchema.biometricDatasets.contentHash, releasedAtMs: bioSchema.biometricDatasets.releasedAtMs }) + .from(bioSchema.biometricDatasets).where(eq(bioSchema.biometricDatasets.id, dataset.id)).get() + if (!latestJob || latestJob.state !== 'succeeded' || latestJob.fencingToken !== storedJob.fencingToken || !latestResult + || latestResult.fencingToken !== storedJob.fencingToken || latestResult.contentHash !== result.contentHash || latestResult.releasedAtMs !== null + || !latestDataset || latestDataset.contentHash !== dataset.contentHash || latestDataset.releasedAtMs !== null + || jobs.holdoutUseCount(dataset.id) !== 1) throw new ReferenceInputError('EVALUATION_MISMATCH', 'Evaluation was released, replaced or reused before activation') + const actual = baseline(configDb, bioDb, input.side) + if (actual.hash !== current.hash || actual.provenance.generation !== current.provenance.generation + || (other && baseline(configDb, bioDb, otherSide).hash !== other.hash)) throw new ReferenceInputError('EVALUATION_MISMATCH', 'Runtime baseline changed before activation') + }, + } +} diff --git a/src/biometrics/profiles/health.ts b/src/biometrics/profiles/health.ts new file mode 100644 index 000000000..52e1ed476 --- /dev/null +++ b/src/biometrics/profiles/health.ts @@ -0,0 +1,42 @@ +import { z } from 'zod' + +// Numerical consumers heartbeat every 30 seconds after accepted frames. +// A stage view is retrospective and consumed on demand, so its receipt age +// cannot establish whether the continuous processor is healthy. +export const RUNTIME_HEARTBEAT_STALE_MS = 120000 +export const runtimeHealthSchema = z.object({ + state: z.enum(['unreported', 'current', 'stale', 'failed', 'mixed_generations', 'clock_skew']), + staleAfterMs: z.number(), + components: z.array(z.object({ component: z.string(), generation: z.number().nullable(), + state: z.enum(['unreported', 'current', 'stale', 'failed', 'pending_boundary', 'clock_skew']), ageMs: z.number().nullable() })), +}) +interface Receipt { component: string, generation: number, appliedHash: string, state: string, heartbeatAtMs: number, effectiveSampleTimeMs: number | null } +export function runtimeHealth(receipts: Receipt[], now = Date.now()): z.infer { + const names = ['heart_rate', 'breathing_rate', 'hrv_index'] + const selected = names.map(component => receipts.filter(row => row.component === component) + .sort((a, b) => b.generation - a.generation || b.heartbeatAtMs - a.heartbeatAtMs)[0]) + const components = names.map((component, index) => { + const receipt = selected[index] + if (!receipt) return { component, generation: null, state: 'unreported' as const, ageMs: null } + const ageMs = now - receipt.heartbeatAtMs + const state = receipt.state === 'failed' + ? 'failed' as const + : ageMs < 0 + ? 'clock_skew' as const + : ageMs > RUNTIME_HEARTBEAT_STALE_MS + ? 'stale' as const + : receipt.state !== 'applied' || receipt.effectiveSampleTimeMs === null ? 'pending_boundary' as const : 'current' as const + return { component, generation: receipt.generation, state, ageMs } + }) + const identities = new Set(selected.filter(row => row !== undefined).map(row => `${row.generation}:${row.appliedHash}`)) + const state = components.some(row => row.state === 'failed') + ? 'failed' + : components.some(row => row.state === 'clock_skew') + ? 'clock_skew' + : components.some(row => row.state === 'unreported' || row.state === 'pending_boundary') + ? 'unreported' + : components.some(row => row.state === 'stale') + ? 'stale' + : identities.size > 1 ? 'mixed_generations' : 'current' + return { state, staleAfterMs: RUNTIME_HEARTBEAT_STALE_MS, components } +} diff --git a/src/biometrics/profiles/history.ts b/src/biometrics/profiles/history.ts new file mode 100644 index 000000000..f816a1146 --- /dev/null +++ b/src/biometrics/profiles/history.ts @@ -0,0 +1,23 @@ +import { and, desc, eq, lt } from 'drizzle-orm' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import { z } from 'zod' +import * as schema from '@/src/db/schema' + +export const activationEventSchema = z.object({ + id: z.string().uuid(), side: z.enum(['left', 'right']), generation: z.number().int(), + action: z.enum(['activate', 'reset', 'rollback', 'migration', 'fallback']), + previousRevisionId: z.string().nullable(), nextRevisionId: z.string().nullable(), + previousHash: z.string(), nextHash: z.string(), evaluationId: z.string().nullable(), evidence: z.unknown(), + requester: z.string(), state: z.enum(['saved', 'pending_delivery', 'pending_boundary', 'applied', 'failed']), + reason: z.string().nullable(), createdAtMs: z.number(), updatedAtMs: z.number(), +}) +export const activationHistorySchema = z.object({ items: z.array(activationEventSchema), nextGeneration: z.number().int().nullable() }) + +export function listActivationHistory(db: BetterSQLite3Database, side: 'left' | 'right', beforeGeneration?: number) { + const events = schema.biometricActivationEvents + const rows = db.select().from(events).where(and(eq(events.side, side), + beforeGeneration === undefined ? undefined : lt(events.generation, beforeGeneration))) + .orderBy(desc(events.generation)).limit(51).all() + const items = rows.slice(0, 50).map(row => activationEventSchema.parse(row)) + return { items, nextGeneration: rows.length > 50 ? items[items.length - 1].generation : null } +} diff --git a/src/biometrics/profiles/legacy-stage.ts b/src/biometrics/profiles/legacy-stage.ts new file mode 100644 index 000000000..956e98b71 --- /dev/null +++ b/src/biometrics/profiles/legacy-stage.ts @@ -0,0 +1,47 @@ +import { randomUUID } from 'node:crypto' +import { eq } from 'drizzle-orm' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import * as schema from '@/src/db/schema' +import { stageClassifierSchema, type StageClassifier } from '@/src/lib/sleep-stage-rules' +import { defaultConfig, legacyStageOverrides, type Override } from '../algorithms/registry' +import { configHash } from '../algorithms/canonical' +import { ReferenceInputError } from '../references/time' +import { requestManualActivation } from './activation' +import { ProfileStore } from './store' + +/** Compatibility writes retain other metric overrides and use the same atomic + * intent/outbox protocol. The legacy column is a migration baseline, not a + * second writable authority. Null clears only sleep-stage personalization. */ +export function writeLegacyStageClassifier(db: BetterSQLite3Database, side: 'left' | 'right', classifier: StageClassifier | null, now = Date.now()) { + const parsed = classifier === null ? null : stageClassifierSchema.parse(classifier) + return db.transaction((tx) => { + const settings = tx.select({ side: schema.sideSettings.side, classifier: schema.sideSettings.stageClassifier }).from(schema.sideSettings).where(eq(schema.sideSettings.side, side)).get() + if (!settings) throw new ReferenceInputError('NOT_FOUND', `Side settings for ${side} not found`) + const profiles = new ProfileStore(db, () => now) + // An invalid legacy value already resolves to defaults. It cannot become a + // pinned revision, so a new save first records the reset recovery that + // preserves it as audit evidence, then activates the replacement. + const malformed = settings.classifier !== null && !stageClassifierSchema.safeParse(settings.classifier).success + if (!malformed) profiles.migrateLegacy(side) + const binding = profiles.binding(side) + const current = binding.desiredRevisionId ? profiles.validateRevision(binding.desiredRevisionId, side) : null + const overrides: Override[] = current + ? (current.revision.overrides as Override[]).filter(override => !override.path.startsWith('sleep_stages.')) + : [] + if (parsed) overrides.push(...legacyStageOverrides(parsed)) + const key = `legacy-stage-write:${randomUUID()}` + const revision = overrides.length + ? profiles.createRevision({ side, ...(current ? { parentRevisionId: current.revision.id } : {}), + overrides, origin: 'manual', requester: 'legacy-stage-settings', idempotencyKey: key }) + : null + const recovered = revision && malformed && binding.generation === 0 + ? requestManualActivation(db, { side, action: 'reset', revisionId: null, expectedGeneration: 0, expectedResolvedHash: configHash(defaultConfig), + boundary: 'immediate', requester: 'legacy-stage-settings', idempotencyKey: `${key}:recover`, allowUnvalidated: true }, now) + : null + const event = requestManualActivation(db, { side, action: revision ? 'activate' : 'reset', revisionId: revision?.id ?? null, + expectedGeneration: recovered?.generation ?? binding.generation, expectedResolvedHash: revision?.resolvedHash ?? configHash(defaultConfig), + boundary: 'immediate', requester: 'legacy-stage-settings', idempotencyKey: key, allowUnvalidated: true }, now) + return { classifier: parsed ?? defaultConfig.sleep_stages, isDefault: parsed === null, + activation: { generation: event.generation, revisionId: revision?.id ?? null, state: event.state } } + }, { behavior: 'immediate' }) +} diff --git a/src/biometrics/profiles/overview.ts b/src/biometrics/profiles/overview.ts new file mode 100644 index 000000000..8a228b27b --- /dev/null +++ b/src/biometrics/profiles/overview.ts @@ -0,0 +1,34 @@ +import { desc, eq } from 'drizzle-orm' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import * as configSchema from '@/src/db/schema' +import * as bioSchema from '@/src/db/biometrics-schema' +import { resolveStageClassifier } from '@/src/lib/sleep-stage-rules' +import { ProfileStore } from './store' +import { resolveStageView } from './stage-view' +import { runtimeHealth } from './health' + +/** Read-only inspection keeps desired intent separate from runtime evidence. */ +export function profileOverview(configDb: BetterSQLite3Database, bioDb: BetterSQLite3Database, side: 'left' | 'right', now = Date.now()) { + const profiles = new ProfileStore(configDb) + const desired = profiles.effective(side) + const legacy = configDb.select({ classifier: configSchema.sideSettings.stageClassifier }).from(configSchema.sideSettings).where(eq(configSchema.sideSettings.side, side)).get() + const receipts = bioDb.select().from(bioSchema.biometricRuntimeReceipts).where(eq(bioSchema.biometricRuntimeReceipts.side, side)) + .orderBy(desc(bioSchema.biometricRuntimeReceipts.generation), desc(bioSchema.biometricRuntimeReceipts.heartbeatAtMs)).limit(20).all() + const latest = receipts.filter((row, index) => receipts.findIndex(other => other.component === row.component) === index) + const sensorCalibration = bioDb.select().from(bioSchema.calibrationProfiles).where(eq(bioSchema.calibrationProfiles.side, side)).all() + .map(row => ({ id: row.id, sensorType: row.sensorType, status: row.status, qualityScore: row.qualityScore, + createdAtMs: row.createdAt.getTime(), expiresAtMs: row.expiresAt?.getTime() ?? null, + freshness: row.status !== 'completed' || row.expiresAt === null ? 'unknown' as const : row.expiresAt.getTime() <= now ? 'expired' as const : 'current' as const, + sourceWindowStartMs: row.sourceWindowStart === null ? null : row.sourceWindowStart * 1000, + sourceWindowEndMs: row.sourceWindowEnd === null ? null : row.sourceWindowEnd * 1000, + })) + return { desired, stageView: resolveStageView(configDb, bioDb, side), + legacyStageView: { classifier: resolveStageClassifier(legacy?.classifier), personalized: legacy?.classifier != null, + migrationPending: legacy?.classifier != null && desired.binding.generation === 0 }, + runtimeHealth: runtimeHealth(latest, now), + runtimeStatus: latest.length ? 'receipts_available' as const : 'unreported' as const, receipts: latest, sensorCalibration, + activationAvailable: true, + limitations: ['Desired configuration is not proof of runtime adoption.', 'Measured sensor calibration is independent of algorithm personalization.', + 'Manual changes are unvalidated; applied status requires matching component receipts.'], + } +} diff --git a/src/biometrics/profiles/promotion-build.ts b/src/biometrics/profiles/promotion-build.ts new file mode 100644 index 000000000..dbf882562 --- /dev/null +++ b/src/biometrics/profiles/promotion-build.ts @@ -0,0 +1,17 @@ +import { installedPromotionPolicies } from './qualification-registry' +import { packagedWorkerPath } from '../backtest/location' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import type * as bioSchema from '@/src/db/biometrics-schema' +import { BacktestJobStore } from '../backtest/jobs' +import { backtestJobSnapshotSchema } from '../backtest/contracts' +import { resolveInstalledReplayBuild } from '../backtest/submission' +import type { PromotionBuild } from './evaluation-store' + +/** This registry is server owned. Frozen client thresholds cannot designate a + * policy; qualified production policies must be installed explicitly. */ +export async function resolvePromotionBuild(bioDb: BetterSQLite3Database, evaluationId: string): Promise { + const job = new BacktestJobStore(bioDb).get(evaluationId) + const snapshot = backtestJobSnapshotSchema.parse(job.input) + const identity = await resolveInstalledReplayBuild(snapshot.protocol.mode) + return { ...identity, designations: await installedPromotionPolicies(packagedWorkerPath(), { ...identity, protocolHash: snapshot.protocolHash }) } +} diff --git a/src/biometrics/profiles/qualification-registry.ts b/src/biometrics/profiles/qualification-registry.ts new file mode 100644 index 000000000..0d05fbc61 --- /dev/null +++ b/src/biometrics/profiles/qualification-registry.ts @@ -0,0 +1,59 @@ +import { open } from 'node:fs/promises' +import { dirname, join } from 'node:path' +import { z } from 'zod' +import { configHash } from '../algorithms/canonical' +import { manifestHash } from '../algorithms/registry' +import { experimentProtocolSchema, numericObjectivePolicies } from '../backtest/contracts' +import { replayContextQualificationSchema } from './replay-context' +import type { DesignatedEvaluationPolicies } from './evaluation-assessment' + +const hash = z.string().regex(/^[a-f0-9]{64}$/) +export const qualificationRegistrySchema = z.object({ + version: z.literal('biometric-promotion-qualifications-v1'), + entries: z.array(z.object({ + qualificationId: z.string().min(1).max(200), manifestHash: hash, evaluatorBuildHash: hash, + pythonBuildHash: hash.optional(), protocol: experimentProtocolSchema, + fixtureEvidenceHash: hash, podEvidenceHash: hash, + waveformContext: replayContextQualificationSchema.nullable(), + }).strict()).max(32), +}).strict() +export const unqualifiedPolicies = (): DesignatedEvaluationPolicies => ({ protocolHash: null, waveformContext: null, numeric: null, stages: null }) + +/** Release-installed authority only. No RPC can write this file. Evidence hashes + * identify the reviewed qualification artifacts; they are not client assertions. */ +export async function installedPromotionPolicies(workerPath: string, identity: { + evaluatorBuildHash: string + pythonBuildHash?: string + protocolHash: string +}): Promise { + try { + const file = await open(join(dirname(workerPath), 'promotion-qualifications.json'), 'r') + let raw: Buffer + try { + if (!(await file.stat()).isFile()) return unqualifiedPolicies() + const buffer = Buffer.alloc(256 * 1024 + 1) + let size = 0 + while (size < buffer.length) { + const read = await file.read(buffer, size, buffer.length - size, size) + if (!read.bytesRead) break + size += read.bytesRead + } + if (size === buffer.length) return unqualifiedPolicies() + raw = buffer.subarray(0, size) + } + finally { await file.close() } + const registry = qualificationRegistrySchema.parse(JSON.parse(raw.toString('utf8'))) + const matches = registry.entries.filter(entry => entry.manifestHash === manifestHash + && entry.evaluatorBuildHash === identity.evaluatorBuildHash && entry.pythonBuildHash === identity.pythonBuildHash + && configHash(entry.protocol) === identity.protocolHash) + if (matches.length !== 1) return unqualifiedPolicies() + const entry = matches[0] + const numeric = numericObjectivePolicies(entry.protocol) + const stages = entry.protocol.stageObjectivePolicy + if (!stages || [stages, ...numeric].some(policy => policy.qualificationId !== entry.qualificationId) + || (entry.protocol.mode === 'signal_replay' && (!entry.pythonBuildHash || !entry.waveformContext)) + || (entry.protocol.mode !== 'signal_replay' && (entry.pythonBuildHash || entry.waveformContext))) return unqualifiedPolicies() + return { protocolHash: identity.protocolHash, numeric, stages, waveformContext: entry.waveformContext } + } + catch { return unqualifiedPolicies() } +} diff --git a/src/biometrics/profiles/recovery.ts b/src/biometrics/profiles/recovery.ts new file mode 100644 index 000000000..849493ccb --- /dev/null +++ b/src/biometrics/profiles/recovery.ts @@ -0,0 +1,72 @@ +import { randomUUID } from 'node:crypto' +import { and, eq } from 'drizzle-orm' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import * as schema from '@/src/db/schema' +import { configHash } from '../algorithms/canonical' +import { manifestHash, validateConfig } from '../algorithms/registry' +import { ProfileStore } from './store' +import { runtimePayloadSchema } from './runtime' + +/** Publish a new immutable generation when the shipped engine changes or a + * projection is unavailable. The desired pinned revision and rollback target + * survive fallback. This records intent, never processor adoption. */ +export function recoverRuntimeIntent(db: BetterSQLite3Database, side: 'left' | 'right', now = Date.now()) { + return db.transaction((tx) => { + const binding = tx.select().from(schema.biometricProfileBindings).where(eq(schema.biometricProfileBindings.side, side)).get() + if (!binding) return null // An unpersonalized side follows the daemon's shipped default. + const desired = new ProfileStore(db).effective(side) + const actualRevisionId = desired.warning ? null : binding.desiredRevisionId + const previous = tx.select().from(schema.biometricConfigOutbox).where(and(eq(schema.biometricConfigOutbox.side, side), + eq(schema.biometricConfigOutbox.generation, binding.generation))).get() + const decoded = runtimePayloadSchema.safeParse(previous?.payload) + let matching = false + if (previous && decoded.success) { + const payload = decoded.data + try { + matching = previous.state !== 'failed' && payload.side === side && payload.generation === binding.generation + && payload.resolvedHash === previous.resolvedHash && payload.manifestHash === manifestHash + && payload.revisionId === actualRevisionId && payload.resolvedHash === desired.hash + && configHash(validateConfig(payload.resolved)) === desired.hash + } + catch { /* A corrupt immutable projection requires a new audited generation. */ } + } + if (matching) return null + + const defaultUpdate = binding.desiredRevisionId === null && decoded.success && decoded.data.manifestHash !== manifestHash + const restoringProfile = binding.desiredRevisionId !== null && decoded.success && decoded.data.revisionId === null + const status = desired.warning + ? 'profile_incompatible' + : defaultUpdate ? 'system_default_update' : restoringProfile ? 'profile_restored' : 'runtime_repair' + const reason = status === 'profile_incompatible' + ? 'Pinned profile is unavailable in this build; using shipped defaults' + : status === 'system_default_update' + ? 'Refresh shipped system defaults' + : status === 'profile_restored' + ? 'Pinned profile is compatible again; restore at the next session' + : 'Repair missing or incompatible runtime intent' + const boundary = status === 'profile_restored' + ? 'next_session' + : status === 'runtime_repair' + ? decoded.success ? decoded.data.boundary : binding.desiredRevisionId ? 'next_session' : 'immediate' + : 'immediate' + const generation = binding.generation + 1 + const key = `runtime-recovery:${side}:${generation}:${manifestHash}` + const evidence = { status, requestedRevisionId: binding.desiredRevisionId, actualRevisionId, + manifestHash, resolvedHash: desired.hash, warning: desired.warning } + tx.update(schema.biometricProfileBindings).set({ generation, updatedAtMs: now }) + .where(and(eq(schema.biometricProfileBindings.side, side), eq(schema.biometricProfileBindings.generation, binding.generation))).run() + // Prevent an undelivered obsolete payload from racing the recovery payload. + if (previous?.state === 'pending') tx.update(schema.biometricConfigOutbox).set({ state: 'failed', lastError: 'Superseded by runtime recovery' }) + .where(eq(schema.biometricConfigOutbox.id, previous.id)).run() + const event = tx.insert(schema.biometricActivationEvents).values({ id: randomUUID(), side, generation, action: 'fallback', + previousRevisionId: binding.desiredRevisionId, nextRevisionId: actualRevisionId, + previousHash: previous?.resolvedHash ?? desired.hash, nextHash: desired.hash, evidence, + requester: 'runtime-coordinator', state: 'pending_delivery', reason, + idempotencyKey: key, requestHash: configHash(evidence), createdAtMs: now, updatedAtMs: now }).returning().get() + tx.insert(schema.biometricConfigOutbox).values({ id: randomUUID(), side, generation, + resolvedHash: desired.hash, payload: { schemaVersion: 1, side, generation, revisionId: actualRevisionId, + manifestHash, resolved: desired.resolved, resolvedHash: desired.hash, boundary }, + state: 'pending', nextAttemptAtMs: now, createdAtMs: now }).run() + return event + }, { behavior: 'immediate' }) +} diff --git a/src/biometrics/profiles/replay-baseline.ts b/src/biometrics/profiles/replay-baseline.ts new file mode 100644 index 000000000..bcd1ae2e3 --- /dev/null +++ b/src/biometrics/profiles/replay-baseline.ts @@ -0,0 +1,50 @@ +import { and, desc, eq, ne } from 'drizzle-orm' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import type * as configSchema from '@/src/db/schema' +import * as bioSchema from '@/src/db/biometrics-schema' +import { configHash } from '../algorithms/canonical' +import { defaultConfig, manifestHash, validateConfig } from '../algorithms/registry' +import { ReferenceInputError } from '../references/time' +import { runtimePayloadSchema } from './runtime' +import { resolveStageView } from './stage-view' + +/** Freeze a coherent numerical baseline, never desired-but-unadopted intent. + * Call inside the caller's read snapshot when capturing both logical sides. */ +export function resolveReplayBaseline(configDb: BetterSQLite3Database, bioDb: BetterSQLite3Database, side: 'left' | 'right') { + const receipts = bioSchema.biometricRuntimeReceipts + const components = ['heart_rate', 'breathing_rate', 'hrv_index'] + const latest = components.map(component => bioDb.select().from(receipts) + .where(and(eq(receipts.side, side), eq(receipts.component, component), ne(receipts.state, 'pending_boundary'))).orderBy(desc(receipts.generation)).limit(1).get()) + if (latest.some(row => row?.state === 'failed')) throw new ReferenceInputError('MISSING_CONTEXT', 'Numerical adoption failed; inspect runtime health before waveform evaluation') + // Pending boundary receipts do not replace the last acknowledged generation. + const applied = components.map(component => bioDb.select().from(receipts) + .where(and(eq(receipts.side, side), eq(receipts.component, component), eq(receipts.state, 'applied'))) + .orderBy(desc(receipts.generation)).limit(1).get()) + if (applied.every(row => !row)) { + const view = resolveStageView(configDb, bioDb, side) + if (view.provenance.warning) throw new ReferenceInputError('MISSING_CONTEXT', 'Current fallback configuration is uncertain; inspect profile compatibility') + const resolved = validateConfig({ ...structuredClone(defaultConfig), sleep_stages: view.classifier }) + return { resolved, hash: configHash(resolved), revisionId: null, + provenance: { source: 'unacknowledged_defaults' as const, generation: null, effectiveSampleTimeMs: null, + warning: 'No numerical configuration has been acknowledged; this baseline uses shipped numerical defaults and the current stage view.' } } + } + const first = applied[0] + if (!first || first.effectiveSampleTimeMs === null || applied.some(row => !row || row.effectiveSampleTimeMs === null + || row.generation !== first.generation || row.appliedHash !== first.appliedHash)) { + throw new ReferenceInputError('MISSING_CONTEXT', 'Numerical components have not acknowledged one consistent configuration') + } + const projection = bioDb.select().from(bioSchema.biometricRuntimeConfigs).where(and( + eq(bioSchema.biometricRuntimeConfigs.side, side), eq(bioSchema.biometricRuntimeConfigs.generation, first.generation))).get() + try { + if (!projection) throw new Error('Missing projection') + const payload = runtimePayloadSchema.parse(projection.payload) + const resolved = validateConfig(payload.resolved) + if (payload.side !== side || payload.generation !== first.generation || payload.boundary !== projection.boundary + || payload.manifestHash !== manifestHash || payload.resolvedHash !== first.appliedHash + || projection.resolvedHash !== first.appliedHash || configHash(resolved) !== first.appliedHash) throw new Error('Inconsistent projection') + return { resolved, hash: first.appliedHash, revisionId: payload.revisionId, + provenance: { source: 'acknowledged_runtime' as const, generation: first.generation, + effectiveSampleTimeMs: Math.max(...applied.map(row => row?.effectiveSampleTimeMs ?? 0)), warning: null } } + } + catch { throw new ReferenceInputError('MISSING_CONTEXT', 'Acknowledged numerical configuration is unavailable or incompatible') } +} diff --git a/src/biometrics/profiles/replay-context.ts b/src/biometrics/profiles/replay-context.ts new file mode 100644 index 000000000..a42ad2f8c --- /dev/null +++ b/src/biometrics/profiles/replay-context.ts @@ -0,0 +1,48 @@ +import { z } from 'zod' +import { waveformEvaluationSchema } from '../backtest/waveform-output' +import { dataHash } from '../algorithms/canonical' +import { ReferenceInputError } from '../references/time' + +const limitation = z.enum(['event_time_replaces_arrival_time', 'cold_start_without_serialized_state', + 'less_than_five_minutes_preroll', 'missing_sleeper_mode', 'missing_occupancy_context', 'missing_pump_context']) +export const replayContextQualificationSchema = z.object({ allowedLimitations: z.array(limitation).max(6).refine(codes => new Set(codes).size === codes.length, 'Duplicate qualified limitation') }).strict() +export type ReplayContextQualification = z.infer + +/** Build-pinned driver diagnostics from every frozen cohort/configuration. A + * threshold policy cannot implicitly approve replay approximations. */ +export function verifyReplayContext(raw: unknown, expected: { + cohortIds: string[] + configHashes: string[] + side: 'left' | 'right' + pythonBuildHash: string + otherSideConfigHash: string +}, qualification: ReplayContextQualification | null) { + const reject = (message: string): never => { + throw new ReferenceInputError('EVALUATION_NOT_VALIDATED', message) + } + if (!qualification) return reject('Waveform context has not been qualified') + const allowed = replayContextQualificationSchema.parse(qualification).allowedLimitations + const envelope = z.object({ hash: z.unknown(), outputs: z.unknown() }).parse(raw) + const evaluation = waveformEvaluationSchema.parse(envelope) + if (dataHash(evaluation.outputs) !== evaluation.hash) reject('Waveform diagnostic content hash changed') + const required = new Set(expected.cohortIds.flatMap(cohort => [...new Set(expected.configHashes)].map(hash => `${cohort}:${hash}`))) + const observed = new Set() + const otherSide = expected.side === 'left' ? 'right' : 'left' + for (const output of evaluation.outputs) { + if (!required.delete(`${output.cohortId}:${output.configHash}`)) reject('Unexpected or duplicated waveform context output') + if (output.run.build_hash !== expected.pythonBuildHash || output.run.config_hashes[expected.side] !== output.configHash + || output.run.config_hashes[otherSide] !== expected.otherSideConfigHash + || output.run.counts.scored_outputs !== output.readings.length) reject('Waveform context identity changed') + for (const reading of output.readings) { + if (reading.configHash !== output.run.config_hashes[reading.side]) reject('Waveform reading identity changed') + } + const codes = [...output.run.limitations, ...output.readings.flatMap(reading => reading.limitations)] + for (const code of codes) { + if (!allowed.some(item => item === code)) reject(`Unqualified replay limitation: ${code}`) + observed.add(code) + } + } + if (required.size) reject('Missing waveform context output') + return { version: 'qualified-replay-context-v1' as const, observedLimitations: [...observed].sort(), + allowedLimitations: [...allowed].sort(), outputs: evaluation.outputs.length } +} diff --git a/src/biometrics/profiles/runtime-host.ts b/src/biometrics/profiles/runtime-host.ts new file mode 100644 index 000000000..a533e5c27 --- /dev/null +++ b/src/biometrics/profiles/runtime-host.ts @@ -0,0 +1,51 @@ +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import type * as configSchema from '@/src/db/schema' +import type * as bioSchema from '@/src/db/biometrics-schema' +import { ProfileStore } from './store' +import { recoverRuntimeIntent } from './recovery' +import { deliverRuntimeOutbox, reconcileActivation, repairCurrentRuntimeProjections } from './runtime' + +type ConfigDB = BetterSQLite3Database +type BioDB = BetterSQLite3Database + +/** Bounded startup/recovery pass. One failure must not suppress the other side + * or a later retry. No hardware commands, numerical work or fabricated receipts. */ +export function tickBiometricRuntime(configDb: ConfigDB, bioDb: BioDB, now = Date.now()) { + const errors: string[] = [] + const attempt = (name: string, operation: () => unknown) => { + try { + operation() + } + catch { + errors.push(name) + } + } + const profiles = new ProfileStore(configDb, () => now) + for (const side of ['left', 'right'] as const) attempt(`migration:${side}`, () => profiles.migrateLegacy(side)) + for (const side of ['left', 'right'] as const) attempt(`intent-recovery:${side}`, () => recoverRuntimeIntent(configDb, side, now)) + attempt('projection-repair', () => repairCurrentRuntimeProjections(configDb, bioDb, now)) + attempt('outbox-delivery', () => deliverRuntimeOutbox(configDb, bioDb, now)) + for (const side of ['left', 'right'] as const) attempt(`reconcile:${side}`, () => reconcileActivation(configDb, bioDb, side, now)) + return { errors } +} + +interface Host { timer: ReturnType, tick: () => void } +const shared = globalThis as typeof globalThis & { __biometricRuntimeHost?: Host } +export function startBiometricRuntime(configDb: ConfigDB, bioDb: BioDB) { + if (shared.__biometricRuntimeHost) return + let previousError = '' + const tick = () => { + const result = tickBiometricRuntime(configDb, bioDb) + const signature = result.errors.join(', ') + if (signature && signature !== previousError) console.warn(`[biometrics] Runtime coordination will retry: ${signature}`) + previousError = signature + } + const timer = setInterval(tick, 5000) + timer.unref() + shared.__biometricRuntimeHost = { timer, tick } + tick() +} +export function stopBiometricRuntime() { + if (shared.__biometricRuntimeHost) clearInterval(shared.__biometricRuntimeHost.timer) + delete shared.__biometricRuntimeHost +} diff --git a/src/biometrics/profiles/runtime.ts b/src/biometrics/profiles/runtime.ts new file mode 100644 index 000000000..4ee20e269 --- /dev/null +++ b/src/biometrics/profiles/runtime.ts @@ -0,0 +1,111 @@ +import { and, asc, eq, lte } from 'drizzle-orm' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import { z } from 'zod' +import * as configSchema from '@/src/db/schema' +import * as bioSchema from '@/src/db/biometrics-schema' +import { configHash } from '../algorithms/canonical' +import { manifestHash, validateConfig } from '../algorithms/registry' +import { ReferenceInputError } from '../references/time' + +export const runtimePayloadSchema = z.object({ schemaVersion: z.literal(1), side: z.enum(['left', 'right']), generation: z.number().int().positive(), + revisionId: z.string().uuid().nullable(), manifestHash: z.string(), resolved: z.unknown(), resolvedHash: z.string(), + boundary: z.enum(['immediate', 'next_session']) }).strict() +export const RUNTIME_COMPONENTS = ['heart_rate', 'breathing_rate', 'hrv_index', 'sleep_stages'] as const + +/** Idempotent projection only; never writes canonical vitals or runtime receipts. */ +export function projectRuntimeConfig(db: BetterSQLite3Database, raw: unknown, now = Date.now()) { + const payload = runtimePayloadSchema.parse(raw) + const resolved = validateConfig(payload.resolved) + if (payload.manifestHash !== manifestHash || configHash(resolved) !== payload.resolvedHash) throw new ReferenceInputError('PROFILE_INCOMPATIBLE', 'Runtime payload does not match its validated engine and configuration') + return db.transaction((tx) => { + const existing = tx.select().from(bioSchema.biometricRuntimeConfigs).where(and(eq(bioSchema.biometricRuntimeConfigs.side, payload.side), eq(bioSchema.biometricRuntimeConfigs.generation, payload.generation))).get() + if (existing) { + if (existing.resolvedHash !== payload.resolvedHash || configHash(existing.payload) !== configHash(payload)) throw new ReferenceInputError('REVISION_CONFLICT', 'Runtime generation already contains a different immutable payload') + return { side: payload.side, generation: payload.generation, alreadyProjected: true } + } + tx.insert(bioSchema.biometricRuntimeConfigs).values({ side: payload.side, generation: payload.generation, + resolvedHash: payload.resolvedHash, payload, boundary: payload.boundary, createdAtMs: now }).run() + return { side: payload.side, generation: payload.generation, alreadyProjected: false } + }, { behavior: 'immediate' }) +} + +/** Two short independent DB transactions; never assumes cross-database atomicity. */ +export function deliverRuntimeOutbox(configDb: BetterSQLite3Database, bioDb: BetterSQLite3Database, now = Date.now()) { + const outbox = configSchema.biometricConfigOutbox + const pending = configDb.select().from(outbox).where(and(eq(outbox.state, 'pending'), lte(outbox.nextAttemptAtMs, now))) + .orderBy(asc(outbox.createdAtMs), asc(outbox.generation)).limit(20).all() + let delivered = 0, failed = 0 + for (const row of pending) { + try { + const payload = runtimePayloadSchema.parse(row.payload) + if (payload.side !== row.side || payload.generation !== row.generation || payload.resolvedHash !== row.resolvedHash) throw new ReferenceInputError('PROFILE_INCOMPATIBLE', 'Outbox envelope and payload disagree') + projectRuntimeConfig(bioDb, payload, now) + configDb.update(outbox).set({ state: 'delivered', deliveredAtMs: now, attempts: row.attempts + 1, lastError: null }) + .where(eq(outbox.id, row.id)).run() + delivered++ + } + catch (error) { + const permanent = error instanceof z.ZodError || error instanceof ReferenceInputError + || (error instanceof Error && /^(Invalid |Unknown |Expected finite|JSON resource)/.test(error.message)) + configDb.transaction((tx) => { + tx.update(outbox).set({ state: permanent ? 'failed' : 'pending', attempts: row.attempts + 1, + nextAttemptAtMs: now + Math.min(300000, 1000 * 2 ** Math.min(row.attempts, 8)), + lastError: permanent ? 'Incompatible or conflicting runtime payload' : 'Runtime database delivery failed; retry scheduled', + }).where(eq(outbox.id, row.id)).run() + if (permanent) tx.update(configSchema.biometricActivationEvents).set({ state: 'failed', reason: 'Runtime payload rejected during delivery', updatedAtMs: now }) + .where(and(eq(configSchema.biometricActivationEvents.side, row.side), eq(configSchema.biometricActivationEvents.generation, row.generation))).run() + }, { behavior: 'immediate' }) + failed++ + } + } + return { delivered, failed } +} + +/** Recreate authoritative projections after a biometric-history reset. */ +export function repairCurrentRuntimeProjections(configDb: BetterSQLite3Database, bioDb: BetterSQLite3Database, now = Date.now()) { + let restored = 0 + for (const binding of configDb.select().from(configSchema.biometricProfileBindings).all()) { + const row = configDb.select().from(configSchema.biometricConfigOutbox).where(and(eq(configSchema.biometricConfigOutbox.side, binding.side), + eq(configSchema.biometricConfigOutbox.generation, binding.generation), eq(configSchema.biometricConfigOutbox.state, 'delivered'))).get() + if (row) { + const payload = runtimePayloadSchema.parse(row.payload) + if (payload.side !== row.side || payload.generation !== row.generation || payload.resolvedHash !== row.resolvedHash) throw new ReferenceInputError('PROFILE_INCOMPATIBLE', 'Stored runtime envelope and payload disagree') + try { + if (!projectRuntimeConfig(bioDb, payload, now).alreadyProjected) restored++ + } + catch (error) { + if (!(error instanceof ReferenceInputError) || error.code !== 'REVISION_CONFLICT') throw error + // Preserve the conflicting immutable row for diagnosis. Recovery will + // publish a fresh audited generation on its next pass. + configDb.update(configSchema.biometricConfigOutbox).set({ state: 'failed', lastError: 'Runtime projection conflicts with authoritative intent' }) + .where(eq(configSchema.biometricConfigOutbox.id, row.id)).run() + } + } + } + return restored +} + +/** Only matching receipts for the desired generation can establish adoption. */ +export function reconcileActivation(configDb: BetterSQLite3Database, bioDb: BetterSQLite3Database, side: 'left' | 'right', now = Date.now()) { + const binding = configDb.select().from(configSchema.biometricProfileBindings).where(eq(configSchema.biometricProfileBindings.side, side)).get() + if (!binding) return null + const event = configDb.select().from(configSchema.biometricActivationEvents).where(and(eq(configSchema.biometricActivationEvents.side, side), eq(configSchema.biometricActivationEvents.generation, binding.generation))).get() + if (!event) return null + const delivery = configDb.select().from(configSchema.biometricConfigOutbox).where(and(eq(configSchema.biometricConfigOutbox.side, side), eq(configSchema.biometricConfigOutbox.generation, binding.generation))).get() + const receipts = bioDb.select().from(bioSchema.biometricRuntimeReceipts).where(and(eq(bioSchema.biometricRuntimeReceipts.side, side), eq(bioSchema.biometricRuntimeReceipts.generation, binding.generation))).all() + .filter(row => row.appliedHash === event.nextHash && RUNTIME_COMPONENTS.includes(row.component as typeof RUNTIME_COMPONENTS[number])) + const rejected = receipts.find(row => row.state === 'failed') + const allApplied = RUNTIME_COMPONENTS.every(component => receipts.some(row => row.component === component && row.state === 'applied' && row.effectiveSampleTimeMs !== null)) + const state = delivery?.state === 'failed' || rejected + ? 'failed' + : delivery?.state !== 'delivered' ? 'pending_delivery' : allApplied ? 'applied' : 'pending_boundary' + // Re-check desired generation inside the write transaction. A newer request + // cannot be acknowledged by receipts read for an older request. + return configDb.transaction((tx) => { + const current = tx.select().from(configSchema.biometricProfileBindings).where(eq(configSchema.biometricProfileBindings.side, side)).get() + if (current?.generation !== binding.generation) return null + tx.update(configSchema.biometricActivationEvents).set({ state, reason: rejected ? 'Runtime component rejected configuration' : delivery?.state === 'failed' || event.action === 'fallback' ? event.reason : null, + updatedAtMs: state === event.state ? event.updatedAtMs : now }).where(eq(configSchema.biometricActivationEvents.id, event.id)).run() + return { side, generation: binding.generation, state, resolvedHash: event.nextHash, receipts } + }, { behavior: 'immediate' }) +} diff --git a/src/biometrics/profiles/stage-view.ts b/src/biometrics/profiles/stage-view.ts new file mode 100644 index 000000000..2131d3603 --- /dev/null +++ b/src/biometrics/profiles/stage-view.ts @@ -0,0 +1,100 @@ +import { and, desc, eq, isNotNull } from 'drizzle-orm' +import { alias } from 'drizzle-orm/sqlite-core' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import { z } from 'zod' +import * as configSchema from '@/src/db/schema' +import * as bioSchema from '@/src/db/biometrics-schema' +import { resolveStageClassifier, stageClassifierSchema } from '@/src/lib/sleep-stage-rules' +import { configHash } from '../algorithms/canonical' +import { defaultConfig, manifestHash, validateConfig } from '../algorithms/registry' +import { runtimePayloadSchema } from './runtime' +import { ProfileStore } from './store' + +export const stageViewSchema = z.object({ + classifier: stageClassifierSchema, + isDefault: z.boolean(), + provenance: z.object({ + interpretation: z.literal('current_classifier_view'), source: z.enum(['runtime', 'legacy', 'default', 'demo']), + generation: z.number().int().nullable(), revisionId: z.string().nullable(), + bundleHash: z.string(), stageHash: z.string(), manifestHash: z.string().nullable(), + effectiveSampleTimeMs: z.number().nullable(), warning: z.string().nullable(), + }), +}) +export type StageView = z.infer +type ConfigDB = BetterSQLite3Database +type BioDB = BetterSQLite3Database +type Side = 'left' | 'right' + +/** The stage consumer follows the boundary actually reached by all numerical + * components. Desired intent alone must never rescore history early. */ +export function resolveStageView(configDb: ConfigDB, bioDb: BioDB, side: Side): StageView { + const projection = bioSchema.biometricRuntimeConfigs + const heart = alias(bioSchema.biometricRuntimeReceipts, 'stage_heart') + const breathing = alias(bioSchema.biometricRuntimeReceipts, 'stage_breathing') + const hrv = alias(bioSchema.biometricRuntimeReceipts, 'stage_hrv') + const match = (receipt: typeof heart | typeof breathing | typeof hrv, component: string) => and( + eq(receipt.side, projection.side), eq(receipt.generation, projection.generation), + eq(receipt.appliedHash, projection.resolvedHash), eq(receipt.component, component), + eq(receipt.state, 'applied'), isNotNull(receipt.effectiveSampleTimeMs), + ) + const eligible = bioDb.select({ projection, + heartTime: heart.effectiveSampleTimeMs, breathingTime: breathing.effectiveSampleTimeMs, hrvTime: hrv.effectiveSampleTimeMs, + }).from(projection) + .innerJoin(heart, match(heart, 'heart_rate')) + .innerJoin(breathing, match(breathing, 'breathing_rate')) + .innerJoin(hrv, match(hrv, 'hrv_index')) + .where(eq(projection.side, side)).orderBy(desc(projection.generation)).limit(1).get() + const desiredWarning = new ProfileStore(configDb).effective(side).warning + let warning: string | null = desiredWarning + if (eligible) { + try { + if (eligible.heartTime === null || eligible.breathingTime === null || eligible.hrvTime === null) throw new Error('Missing runtime boundary') + const row = eligible.projection + const payload = runtimePayloadSchema.parse(row.payload) + const resolved = validateConfig(payload.resolved) + if (payload.side !== side || payload.generation !== row.generation || payload.boundary !== row.boundary + || payload.manifestHash !== manifestHash || payload.resolvedHash !== row.resolvedHash || configHash(resolved) !== row.resolvedHash) { + throw new Error('Incompatible runtime projection') + } + return { classifier: resolved.sleep_stages, isDefault: configHash(resolved.sleep_stages) === configHash(defaultConfig.sleep_stages), + provenance: { interpretation: 'current_classifier_view', source: 'runtime', generation: row.generation, + revisionId: payload.revisionId, bundleHash: row.resolvedHash, stageHash: configHash(resolved.sleep_stages), manifestHash, + effectiveSampleTimeMs: Math.max(eligible.heartTime, eligible.breathingTime, eligible.hrvTime), warning: payload.revisionId === null ? desiredWarning : null } } + } + catch { + warning = 'Applied profile is incompatible with this build; stage view uses system defaults' + } + } + // Preserve legacy behavior until its first migration/adoption. Incompatible + // engines must visibly fall back to shipped defaults, never be relabeled. + const legacy = warning + ? null + : configDb.select({ classifier: configSchema.sideSettings.stageClassifier }) + .from(configSchema.sideSettings).where(eq(configSchema.sideSettings.side, side)).get()?.classifier + const classifier = resolveStageClassifier(legacy) + const personalized = legacy != null && stageClassifierSchema.safeParse(legacy).success + if (legacy != null && !personalized) warning = 'Stored legacy stage rules are invalid; using system defaults' + return { classifier, isDefault: !personalized, + provenance: { interpretation: 'current_classifier_view', source: personalized ? 'legacy' : 'default', generation: null, + revisionId: null, bundleHash: configHash({ ...defaultConfig, sleep_stages: classifier }), stageHash: configHash(classifier), + manifestHash: personalized ? null : manifestHash, effectiveSampleTimeMs: null, warning } } +} + +/** Call at the stage-consumer read boundary. This acknowledges rules delivered + * for classification, not canonical historical stage observations. */ +export function consumeStageView(configDb: ConfigDB, bioDb: BioDB, side: Side, now = Date.now()): StageView { + return bioDb.transaction((tx) => { + const view = resolveStageView(configDb, tx, side) + const provenance = view.provenance + if (provenance.source === 'runtime' && provenance.generation !== null) { + const receipt = bioSchema.biometricRuntimeReceipts + tx.insert(receipt).values({ side, component: 'sleep_stages', generation: provenance.generation, + appliedHash: provenance.bundleHash, effectiveSampleTimeMs: provenance.effectiveSampleTimeMs, + state: 'applied', error: null, heartbeatAtMs: now, + }).onConflictDoUpdate({ target: [receipt.side, receipt.component, receipt.generation], + set: { appliedHash: provenance.bundleHash, effectiveSampleTimeMs: provenance.effectiveSampleTimeMs, + state: 'applied', error: null, heartbeatAtMs: now } }).run() + } + return view + }, { behavior: 'immediate' }) +} diff --git a/src/biometrics/profiles/status.ts b/src/biometrics/profiles/status.ts new file mode 100644 index 000000000..1ba91e229 --- /dev/null +++ b/src/biometrics/profiles/status.ts @@ -0,0 +1,46 @@ +import { and, eq } from 'drizzle-orm' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import { z } from 'zod' +import * as configSchema from '@/src/db/schema' +import * as bioSchema from '@/src/db/biometrics-schema' +import { ReferenceInputError } from '../references/time' +import { activationEventSchema } from './history' +import { runtimeHealth, runtimeHealthSchema } from './health' +import { RUNTIME_COMPONENTS } from './runtime' + +export const activationStatusSchema = z.object({ + desiredGeneration: z.number().int(), isCurrent: z.boolean(), event: activationEventSchema.nullable(), + deliveryState: z.enum(['pending', 'delivered', 'failed']).nullable(), + observedState: z.enum(['unreported', 'pending_delivery', 'pending_boundary', 'applied', 'failed']), + receiptHealth: runtimeHealthSchema, + receipts: z.array(z.object({ component: z.string(), generation: z.number().int(), appliedHash: z.string(), + effectiveSampleTimeMs: z.number().nullable(), state: z.enum(['pending_boundary', 'applied', 'failed']), + error: z.string().nullable(), heartbeatAtMs: z.number() })), +}) + +/** Observation only: polling never acknowledges a processor or writes intent. */ +export function activationStatus(configDb: BetterSQLite3Database, bioDb: BetterSQLite3Database, + side: 'left' | 'right', id?: string, now = Date.now()): z.infer { + const binding = configDb.select().from(configSchema.biometricProfileBindings).where(eq(configSchema.biometricProfileBindings.side, side)).get() + const desiredGeneration = binding?.generation ?? 0 + const events = configSchema.biometricActivationEvents + const event = configDb.select().from(events).where(and(eq(events.side, side), id ? eq(events.id, id) : eq(events.generation, desiredGeneration))).get() + if (id && !event) throw new ReferenceInputError('NOT_FOUND', 'Activation event not found for this side') + const delivery = event + ? configDb.select().from(configSchema.biometricConfigOutbox).where(and( + eq(configSchema.biometricConfigOutbox.side, side), eq(configSchema.biometricConfigOutbox.generation, event.generation))).get() + : undefined + const receipts = event + ? bioDb.select().from(bioSchema.biometricRuntimeReceipts).where(and( + eq(bioSchema.biometricRuntimeReceipts.side, side), eq(bioSchema.biometricRuntimeReceipts.generation, event.generation))).all() + .filter(row => row.appliedHash === event.nextHash && RUNTIME_COMPONENTS.includes(row.component as typeof RUNTIME_COMPONENTS[number])) + : [] + const allApplied = RUNTIME_COMPONENTS.every(component => receipts.some(row => row.component === component && row.state === 'applied' && row.effectiveSampleTimeMs !== null)) + const observedState = !event + ? 'unreported' + : delivery?.state === 'failed' || receipts.some(row => row.state === 'failed') + ? 'failed' + : delivery?.state !== 'delivered' ? 'pending_delivery' : allApplied ? 'applied' : 'pending_boundary' + return { desiredGeneration, isCurrent: event?.generation === desiredGeneration, event: event ? activationEventSchema.parse(event) : null, + deliveryState: delivery?.state ?? null, observedState, receiptHealth: runtimeHealth(receipts, now), receipts } +} diff --git a/src/biometrics/profiles/store.ts b/src/biometrics/profiles/store.ts new file mode 100644 index 000000000..6db2e5049 --- /dev/null +++ b/src/biometrics/profiles/store.ts @@ -0,0 +1,151 @@ +import { randomUUID } from 'node:crypto' +import { and, desc, eq, lt, or } from 'drizzle-orm' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import { z } from 'zod' +import * as schema from '@/src/db/schema' +import { biometricActivationEvents as events, biometricConfigOutbox as outbox, biometricProfileBindings as bindings, biometricProfileRevisions as revisions } from '@/src/db/schema' +import { stageClassifierSchema } from '@/src/lib/sleep-stage-rules' +import { biometricManifest, defaultConfig, legacyStageOverrides, manifestHash, resolveConfig, validateConfig, type Override } from '../algorithms/registry' +import { configHash } from '../algorithms/canonical' + +const sideSchema = z.enum(['left', 'right']) +const keySchema = z.string().min(1).max(200) +export const revisionInput = z.object({ + side: sideSchema, + parentRevisionId: z.string().uuid().optional(), + baseManifestHash: z.string().length(64).default(manifestHash), + overrides: z.array(z.object({ path: z.string().min(1).max(200), value: z.unknown() }).strict()).max(64), + origin: z.enum(['manual', 'migration', 'agent', 'system']), + requester: z.string().min(1).max(200), + idempotencyKey: keySchema, +}).strict() + +type Side = z.infer +type ConfigDB = BetterSQLite3Database +type Revision = typeof revisions.$inferSelect + +/** + * Configuration-only repository. No biometrics DB, numerical work or hardware + * access is allowed here. Synchronous SQLite transactions protect revisions, + * compare-and-swap bindings, audit events and outbox publication together. + */ +export class ProfileStore { + constructor(private readonly db: ConfigDB, private readonly now = Date.now) {} + + createRevision(raw: z.input): Revision { + const input = revisionInput.parse(raw) + // tRPC/SuperJSON can preserve an explicitly undefined optional field. + // Missing parent means a new family and has one canonical request identity. + if (input.parentRevisionId === undefined) delete input.parentRevisionId + const requestHash = configHash(input) + return this.db.transaction((tx) => { + const existing = tx.select().from(revisions).where(eq(revisions.idempotencyKey, input.idempotencyKey)).get() + if (existing) { + if (existing.requestHash !== requestHash) throw new Error('IDEMPOTENCY_CONFLICT: revision request changed') + return existing + } + const parent = input.parentRevisionId + ? tx.select().from(revisions).where(eq(revisions.id, input.parentRevisionId)).get() + : undefined + if (input.parentRevisionId && !parent) throw new Error('NOT_FOUND: parent revision') + if (parent && parent.side !== input.side) throw new Error('PROFILE_INCOMPATIBLE: parent belongs to the other side') + const familyId = parent?.familyId ?? randomUUID() + const latest = tx.select().from(revisions).where(eq(revisions.familyId, familyId)).orderBy(desc(revisions.revision)).get() + const resolved = resolveConfig(input.overrides, input.baseManifestHash) + const row: typeof revisions.$inferInsert = { + id: randomUUID(), familyId, revision: (latest?.revision ?? 0) + 1, + side: input.side, parentRevisionId: parent?.id ?? null, + baseManifestHash: manifestHash, baseManifest: biometricManifest, + overrides: resolved.overrides, resolved: resolved.resolved, resolvedHash: resolved.hash, + dependencyHashes: biometricManifest.dependencyHashes, + origin: input.origin, requester: input.requester, + idempotencyKey: input.idempotencyKey, requestHash, createdAtMs: this.now(), + } + return tx.insert(revisions).values(row).returning().get() + }, { behavior: 'immediate' }) + } + + getRevision(id: string): Revision { + const revision = this.db.select().from(revisions).where(eq(revisions.id, id)).get() + if (!revision) throw new Error('NOT_FOUND: profile revision') + return revision + } + + listRevisions(side: Side, cursor?: { createdAtMs: number, id: string }, familyId?: string) { + const rows = this.db.select().from(revisions).where(and( + eq(revisions.side, side), + familyId ? eq(revisions.familyId, familyId) : undefined, + cursor + ? or(lt(revisions.createdAtMs, cursor.createdAtMs), + and(eq(revisions.createdAtMs, cursor.createdAtMs), lt(revisions.id, cursor.id))) + : undefined, + )).orderBy(desc(revisions.createdAtMs), desc(revisions.id)).limit(101).all() + const items = rows.slice(0, 100) + const last = items.at(-1) + return { items, nextCursor: rows.length > 100 && last ? { createdAtMs: last.createdAtMs, id: last.id } : null } + } + + validateRevision(id: string, side: Side) { + const revision = this.getRevision(id) + if (revision.side !== side) throw new Error('PROFILE_INCOMPATIBLE: revision belongs to the other side') + if (revision.baseManifestHash !== manifestHash || configHash(revision.baseManifest) !== manifestHash) { + throw new Error('PROFILE_INCOMPATIBLE: pinned engine is not available in this build') + } + const resolved = validateConfig(revision.resolved) + const rebuilt = resolveConfig(revision.overrides as Override[], revision.baseManifestHash) + if (configHash(resolved) !== revision.resolvedHash || rebuilt.hash !== revision.resolvedHash + || configHash(revision.dependencyHashes) !== configHash(biometricManifest.dependencyHashes)) { + throw new Error('PROFILE_INCOMPATIBLE: immutable snapshot integrity check failed') + } + return { revision, resolved, hash: revision.resolvedHash } + } + + binding(side: Side) { + return this.db.select().from(bindings).where(eq(bindings.side, side)).get() + ?? { side, generation: 0, desiredRevisionId: null, previousRevisionId: null, updatedAtMs: 0 } + } + + /** Desired state only. Actual runtime state must come from component receipts. */ + effective(side: Side) { + const binding = this.binding(side) + if (!binding.desiredRevisionId) return { binding, resolved: defaultConfig, hash: configHash(defaultConfig), isDefault: true, warning: null } + try { + const selected = this.validateRevision(binding.desiredRevisionId, side) + return { binding, resolved: selected.resolved, hash: selected.hash, isDefault: false, warning: null } + } + catch (error) { + return { binding, resolved: defaultConfig, hash: configHash(defaultConfig), isDefault: true, + warning: error instanceof Error ? error.message : 'PROFILE_INCOMPATIBLE' } + } + } + + /** + * Legacy migration is idempotent and never overwrites an existing binding. + * Copies all rule leaves, not only differences, so future defaults cannot + * silently change the meaning of an old manual customization. + */ + migrateLegacy(side: Side) { + return this.db.transaction((tx) => { + const existing = tx.select().from(bindings).where(eq(bindings.side, side)).get() + if (existing) return existing + const stored = tx.select({ classifier: schema.sideSettings.stageClassifier }).from(schema.sideSettings) + .where(eq(schema.sideSettings.side, side)).get()?.classifier + if (stored == null) return null // no binding means follow defaults + const classifier = stageClassifierSchema.parse(stored) + const revision = this.createRevision({ side, overrides: legacyStageOverrides(classifier), origin: 'migration', + requester: 'legacy-stage-settings', idempotencyKey: `legacy-stage:${side}:${configHash(classifier)}` }) + const now = this.now() + tx.insert(bindings).values({ side, desiredRevisionId: revision.id, generation: 1, updatedAtMs: now }).run() + tx.insert(events).values({ id: randomUUID(), side, generation: 1, action: 'migration', + previousHash: configHash(defaultConfig), nextHash: revision.resolvedHash, + nextRevisionId: revision.id, evidence: { status: 'legacy_unvalidated' }, requester: 'legacy-stage-settings', + state: 'pending_delivery', idempotencyKey: `legacy-stage:${side}`, requestHash: configHash(classifier), + createdAtMs: now, updatedAtMs: now }).run() + tx.insert(outbox).values({ id: randomUUID(), side, generation: 1, + payload: { schemaVersion: 1, side, generation: 1, revisionId: revision.id, manifestHash, + resolved: revision.resolved, resolvedHash: revision.resolvedHash, boundary: 'immediate' }, + resolvedHash: revision.resolvedHash, state: 'pending', nextAttemptAtMs: now, createdAtMs: now }).run() + return this.binding(side) + }, { behavior: 'immediate' }) + } +} diff --git a/src/biometrics/profiles/tests/activation.test.ts b/src/biometrics/profiles/tests/activation.test.ts new file mode 100644 index 000000000..ec3f76790 --- /dev/null +++ b/src/biometrics/profiles/tests/activation.test.ts @@ -0,0 +1,110 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import * as schema from '@/src/db/schema' +import { ProfileStore } from '../store' +import { requestManualActivation } from '../activation' +import { configHash } from '../../algorithms/canonical' +import { defaultConfig } from '../../algorithms/registry' +let sqlite: Database.Database +let db: ReturnType> +let profiles: ProfileStore +beforeEach(() => { + sqlite = new Database(':memory:') + db = drizzle(sqlite, { schema }) + migrate(db, { migrationsFolder: 'src/db/migrations' }) + profiles = new ProfileStore(db) +}) +afterEach(() => sqlite.close()) +const candidate = () => profiles.createRevision({ side: 'left', overrides: [{ path: 'heart_rate.trackingToleranceBpm', value: 20 }], origin: 'manual', requester: 'fixture', idempotencyKey: 'candidate' }) +const request = () => { + const revision = candidate() + return { side: 'left' as const, action: 'activate' as const, revisionId: revision.id, expectedGeneration: 0, + expectedResolvedHash: revision.resolvedHash, boundary: 'next_session' as const, requester: 'fixture', idempotencyKey: 'activation', allowUnvalidated: true as const } +} +describe('atomic activation intent', () => { + it('writes one binding, audit event and immutable outbox payload, without claiming adoption', () => { + const input = request() + const event = requestManualActivation(db, input) + expect(event).toMatchObject({ generation: 1, state: 'pending_delivery', nextHash: input.expectedResolvedHash }) + expect(requestManualActivation(db, input)).toEqual(event) + expect(profiles.binding('left')).toMatchObject({ generation: 1, desiredRevisionId: input.revisionId, previousRevisionId: null }) + expect(profiles.binding('right').generation).toBe(0) + expect(db.select().from(schema.biometricConfigOutbox).all()).toHaveLength(1) + expect(() => requestManualActivation(db, { ...input, idempotencyKey: 'stale' })).toThrow('Binding changed') + }) + it('distinguishes rollback from reset and verifies exact target hashes', () => { + const input = request() + requestManualActivation(db, input) + expect(() => requestManualActivation(db, { ...input, action: 'rollback', expectedGeneration: 1, idempotencyKey: 'wrong' })).toThrow('previous binding') + const rollback = requestManualActivation(db, { ...input, action: 'rollback', revisionId: null, expectedGeneration: 1, + expectedResolvedHash: configHash(defaultConfig), idempotencyKey: 'rollback' }) + expect(rollback).toMatchObject({ action: 'rollback', generation: 2, nextRevisionId: null }) + const reset = requestManualActivation(db, { ...input, action: 'reset', revisionId: null, expectedGeneration: 2, + expectedResolvedHash: configHash(defaultConfig), idempotencyKey: 'reset' }) + expect(reset).toMatchObject({ action: 'reset', generation: 3 }) + }) + it('rolls back all intent writes when outbox insertion fails', () => { + const input = request() + sqlite.exec('CREATE TRIGGER reject_outbox BEFORE INSERT ON biometric_config_outbox BEGIN SELECT RAISE(ABORT, \'fixture failure\'); END;') + expect(() => requestManualActivation(db, input)).toThrow('fixture failure') + expect(profiles.binding('left').generation).toBe(0) + expect(db.select().from(schema.biometricActivationEvents).all()).toEqual([]) + }) + it('requires legacy personalization to be preserved before a new binding can replace it', () => { + db.insert(schema.sideSettings).values({ side: 'left', name: 'Left', stageClassifier: defaultConfig.sleep_stages }).run() + expect(() => requestManualActivation(db, request())).toThrow('Preserve the existing stage profile') + expect(profiles.binding('left').generation).toBe(0) + expect(() => requestManualActivation(db, { ...request(), action: 'reset', revisionId: null, expectedResolvedHash: configHash(defaultConfig) })).toThrow('Preserve the existing stage profile') + profiles.migrateLegacy('left') + const input = request() + expect(requestManualActivation(db, { ...input, expectedGeneration: 1 }).generation).toBe(2) + }) + + it('rejects incorrect hashes and changed idempotency payloads', () => { + const input = request() + expect(() => requestManualActivation(db, { ...input, expectedResolvedHash: '0'.repeat(64) })).toThrow('Resolved configuration changed') + requestManualActivation(db, input) + expect(() => requestManualActivation(db, { ...input, boundary: 'immediate' })).toThrow('idempotency key changed') + }) +}) + +it('allows explicit reset of malformed legacy rules while preserving the original as audit evidence', () => { + const malformed = { engine: 'unknown-engine', ruleset: { cannotMigrate: true } } + db.insert(schema.sideSettings).values({ side: 'left', name: 'Left', stageClassifier: malformed as unknown as typeof defaultConfig.sleep_stages }).run() + expect(() => profiles.migrateLegacy('left')).toThrow() + const input = { ...request(), action: 'reset' as const, revisionId: null, expectedResolvedHash: configHash(defaultConfig), idempotencyKey: 'recover-invalid-legacy' } + expect(() => requestManualActivation(db, { ...input, expectedResolvedHash: '0'.repeat(64) })).toThrow('Resolved configuration changed') + expect(profiles.binding('left').generation).toBe(0) + const event = requestManualActivation(db, input) + expect(event).toMatchObject({ action: 'reset', generation: 1, state: 'pending_delivery', nextRevisionId: null, + evidence: { status: 'manual_unvalidated', recovery: 'invalid_legacy_reset', preservedLegacyClassifier: malformed } }) + expect(db.select().from(schema.sideSettings).get()?.stageClassifier).toEqual(malformed) + expect(requestManualActivation(db, input).id).toBe(event.id) + expect(profiles.migrateLegacy('left')).toMatchObject({ generation: 1, desiredRevisionId: null }) +}) + +it('does not use malformed legacy recovery to bypass preservation or atomic publication', () => { + db.insert(schema.sideSettings).values({ side: 'left', name: 'Left', stageClassifier: { broken: true } as unknown as typeof defaultConfig.sleep_stages }).run() + expect(() => requestManualActivation(db, request())).toThrow('Preserve the existing stage profile') + sqlite.exec('CREATE TRIGGER fail_reset_outbox BEFORE INSERT ON biometric_config_outbox BEGIN SELECT RAISE(ABORT, \'outbox failure\'); END;') + expect(() => requestManualActivation(db, { ...request(), action: 'reset', revisionId: null, expectedResolvedHash: configHash(defaultConfig) })).toThrow('outbox failure') + expect(profiles.binding('left').generation).toBe(0) + expect(db.select().from(schema.biometricActivationEvents).all()).toEqual([]) + expect(db.select().from(schema.sideSettings).get()?.stageClassifier).toEqual({ broken: true }) +}) + +it('resets an unavailable pinned engine without deleting or relabeling its previous revision', () => { + const input = request() + requestManualActivation(db, input) + sqlite.prepare('UPDATE biometric_profile_revisions SET base_manifest_hash = ? WHERE id = ?').run('old-engine', input.revisionId) + expect(profiles.effective('left').warning).toContain('PROFILE_INCOMPATIBLE') + const event = requestManualActivation(db, { ...input, action: 'reset', revisionId: null, expectedGeneration: 1, expectedResolvedHash: configHash(defaultConfig), idempotencyKey: 'reset-unavailable' }) + expect(event).toMatchObject({ generation: 2, nextRevisionId: null, previousRevisionId: input.revisionId }) + expect(profiles.binding('left').previousRevisionId).toBe(input.revisionId) + expect(profiles.getRevision(input.revisionId).baseManifestHash).toBe('old-engine') + expect(() => requestManualActivation(db, { ...input, action: 'rollback', expectedGeneration: 2, idempotencyKey: 'rollback-unavailable' })).toThrow('PROFILE_INCOMPATIBLE') + expect(profiles.binding('left').generation).toBe(2) +}) diff --git a/src/biometrics/profiles/tests/contracts.test.ts b/src/biometrics/profiles/tests/contracts.test.ts new file mode 100644 index 000000000..44da7828c --- /dev/null +++ b/src/biometrics/profiles/tests/contracts.test.ts @@ -0,0 +1,29 @@ +// @vitest-environment node +import { expect, it } from 'vitest' +import { defaultConfig } from '../../algorithms/registry' +import { bindingSchema, biometricConfigSchema, overrideSchema, profileOverviewSchema, revisionSchema } from '../contracts' + +const uuid = '11111111-1111-4111-8111-111111111111' + +it('derives the typed configuration schema from the manifest and still applies relationship checks', () => { + expect(biometricConfigSchema.parse(defaultConfig)).toEqual(defaultConfig) + const inverted = structuredClone(defaultConfig) + inverted.sleep_stages.ruleset.outliers.heartRate = { min: 100, max: 50 } + const result = biometricConfigSchema.safeParse(inverted) + expect(result.success).toBe(false) + expect(result.error?.issues.map(issue => issue.message)).toContain('Invalid biometric parameter relationship') + expect(biometricConfigSchema.safeParse({ ...defaultConfig, heart_rate: { ...defaultConfig.heart_rate, trackingToleranceBpm: 1000 } }).success).toBe(false) +}) + +it('validates revision, binding and override records', () => { + expect(overrideSchema.parse({ path: 'heart_rate.trackingToleranceBpm', value: 20 })).toEqual({ path: 'heart_rate.trackingToleranceBpm', value: 20 }) + expect(() => overrideSchema.parse({ value: 20 })).toThrow() + const revision = { id: uuid, familyId: uuid, revision: 1, side: 'left', baseManifestHash: 'm', baseManifest: {}, overrides: [], resolved: defaultConfig, + resolvedHash: 'r', dependencyHashes: { manifest: 'm' }, parentRevisionId: null, origin: 'manual', requester: 'test', idempotencyKey: 'k', requestHash: 'h', createdAtMs: 1 } + expect(revisionSchema.parse(revision)).toEqual(revision) + expect(() => revisionSchema.parse({ ...revision, origin: 'unknown' })).toThrow() + expect(bindingSchema.parse({ side: 'right', generation: 0, desiredRevisionId: null, previousRevisionId: uuid, updatedAtMs: 1 })).toMatchObject({ generation: 0 }) + expect(() => bindingSchema.parse({ side: 'right', generation: 0.5, desiredRevisionId: null, previousRevisionId: null, updatedAtMs: 1 })).toThrow() + expect(Object.keys(profileOverviewSchema.shape)).toEqual(['stageView', 'desired', 'legacyStageView', 'runtimeHealth', 'runtimeStatus', 'receipts', 'sensorCalibration', 'activationAvailable', 'limitations']) + expect(profileOverviewSchema.safeParse({}).success).toBe(false) +}) diff --git a/src/biometrics/profiles/tests/diff.test.ts b/src/biometrics/profiles/tests/diff.test.ts new file mode 100644 index 000000000..0b3bd9b47 --- /dev/null +++ b/src/biometrics/profiles/tests/diff.test.ts @@ -0,0 +1,27 @@ +// @vitest-environment node +import { expect, it } from 'vitest' +import { configHash } from '../../algorithms/canonical' +import { defaultConfig } from '../../algorithms/registry' +import { diffProfiles } from '../diff' +import type { ProfileStore } from '../store' + +function storeWith(resolved: unknown) { + return { validateRevision: () => ({ resolved, hash: configHash(resolved), revision: { overrides: [] } }) } as unknown as ProfileStore +} + +it('records keys present on only one side of the resolved configuration', () => { + const added = { ...defaultConfig, extra: { nested: 1 }, heart_rate: { ...defaultConfig.heart_rate, introduced: true } } + const toAdded = diffProfiles(storeWith(added), 'left', 'revision', null) + expect(toAdded.changes).toEqual(expect.arrayContaining([ + { path: 'extra', before: undefined, after: { nested: 1 } }, + { path: 'heart_rate.introduced', before: undefined, after: true }, + ])) + expect(toAdded.changes).toHaveLength(2) + const fromAdded = diffProfiles(storeWith(added), 'left', null, 'revision') + expect(fromAdded.changes).toEqual(expect.arrayContaining([ + { path: 'extra', before: { nested: 1 }, after: undefined }, + { path: 'heart_rate.introduced', before: true, after: undefined }, + ])) + expect(fromAdded.changes).toHaveLength(2) + expect(diffProfiles(storeWith(defaultConfig), 'left', 'revision', null).changes).toEqual([]) +}) diff --git a/src/biometrics/profiles/tests/evaluation-assessment.test.ts b/src/biometrics/profiles/tests/evaluation-assessment.test.ts new file mode 100644 index 000000000..22d8170b2 --- /dev/null +++ b/src/biometrics/profiles/tests/evaluation-assessment.test.ts @@ -0,0 +1,174 @@ +import { replayContextFixture } from './fixtures/replay-context' +// @vitest-environment node +import { expect, it } from 'vitest' +import { verifyEvaluationAssessments } from '../evaluation-assessment' +import { evaluationFixture as fixture } from './fixtures/evaluation' +import { backtestFixture } from '../../backtest/tests/fixtures/backtest' +import { assessNumericObjective, type ObjectivePolicy, type ObjectiveEvidence } from '../../backtest/objective-policy' +import { configHash, dataHash } from '../../algorithms/canonical' +import { defaultConfig, manifestHash } from '../../algorithms/registry' + +it('reassesses exact retained evidence under a trusted policy and returns a bounded audit snapshot', () => { + const work = fixture() + expect(work.report.stageObjectiveAssessments[0].status).toBe('insufficient_evidence') + const verified = verifyEvaluationAssessments(work.job, work.result(), work.expected, work.designations) + expect(verified).toMatchObject({ version: 'verified-evaluation-v1', reportHash: work.result().contentHash, + changedDomains: ['sleep_stages'], numericAssessments: [], stageAssessments: [{ baseline: 'default', status: 'improved' }, { baseline: 'current', status: 'improved' }] }) + expect(verified).not.toHaveProperty('nights') + expect(verified.policies).toEqual({ numeric: null, stages: work.designations.stages }) +}) + +it('cannot promote a client-selected policy or trust claimed successful assessments', () => { + const work = fixture() + expect(() => verifyEvaluationAssessments(work.job, work.result(), work.expected, { protocolHash: null, waveformContext: null, numeric: null, stages: null })).toThrow() + work.report.stageObjectiveAssessments[0].status = 'improved' + work.report.stageObjectiveAssessments[0].evidence.nights[0].candidateOutputCoverage = 0.5 + expect(() => verifyEvaluationAssessments(work.job, work.result(), work.expected, work.designations)).toThrow() +}) + +it('requires both baseline comparisons and exact cohort membership even when scores pass', () => { + for (const change of [ + (work: ReturnType) => { work.report.stageObjectiveAssessments.pop() }, + (work: ReturnType) => { work.report.stageObjectiveAssessments[0].baselineHash = '0'.repeat(64) }, + (work: ReturnType) => { work.report.stageObjectiveAssessments[0].candidateHash = '0'.repeat(64) }, + (work: ReturnType) => { work.report.stageObjectiveAssessments[0].evidence.nights[0].cohortId = 'other-night' }, + ]) { + const work = fixture() + change(work) + expect(() => verifyEvaluationAssessments(work.job, work.result(), work.expected, work.designations)).toThrow() + } + const incomplete = fixture() + incomplete.report.nights.push({ cohortId: 'd' }) + incomplete.expected.dataset.holdoutCohortIds.push('d') + expect(() => verifyEvaluationAssessments(incomplete.job, incomplete.result(), incomplete.expected, incomplete.designations)).toThrow('omits frozen') +}) + +it('requires an actual configuration change and a frozen stage policy', () => { + const work = fixture() + work.job.configs[2].resolved = defaultConfig + work.job.configs[2].resolvedHash = configHash(defaultConfig) + work.expected.resolvedHash = configHash(defaultConfig) + expect(() => verifyEvaluationAssessments(work.job, work.result(), work.expected, work.designations)).toThrow('does not change') + const missing = fixture() + const protocol = backtestFixture().job.protocol + const job = { ...missing.job, protocol, protocolHash: configHash(protocol) } + const report = { ...missing.report, protocol, protocolHash: job.protocolHash } + expect(() => verifyEvaluationAssessments(job, { ...missing.result(), report, contentHash: dataHash(report) }, missing.expected, missing.designations)).toThrow('require stage objective') +}) + +it('rejects missing supervisor measurements and no material primary improvement', () => { + const work = fixture() + work.report.jobOperations.memory.allSamplesAvailable = false + expect(() => verifyEvaluationAssessments(work.job, work.result(), work.expected, work.designations)).toThrow() + work.report.jobOperations.memory.allSamplesAvailable = true + work.report.stageObjectiveAssessments[1].evidence.intervalLower = 0 + expect(() => verifyEvaluationAssessments(work.job, work.result(), work.expected, work.designations)).toThrow('does not support') +}) + +it('does not let stage agreement validate a simultaneous numeric parameter change', () => { + const work = fixture() + const candidate = work.job.configs[2] + candidate.resolved.heart_rate.trackingToleranceBpm = 20 + candidate.resolvedHash = configHash(candidate.resolved) + work.expected.resolvedHash = candidate.resolvedHash + expect(() => verifyEvaluationAssessments(work.job, work.result(), work.expected, work.designations)).toThrow('numeric component') +}) + +it('requires numeric improvement while allowing unchanged downstream stages, and rejects an unassessed second metric', () => { + const work = fixture() + const policy: ObjectivePolicy = { version: 'numeric-objective-v1', qualificationId: work.designations.stages.qualificationId, + metric: 'heart_rate', minimumNights: 3, minimumPairedReferencesPerNight: 1, maximumLostObservationFraction: 0.1, + minimumMaeImprovement: 2, maximumNightMaeRegression: 1, maximumRegressedNightFraction: 0.34, + maximumSubgroupMaeRegression: 1, requiredSubgroups: [], operational: work.designations.stages.operational } + const candidate = structuredClone(defaultConfig) + candidate.heart_rate.trackingToleranceBpm = 20 + work.job.configs[2].resolved = candidate + work.job.configs[2].resolvedHash = configHash(candidate) + const protocol = { ...work.job.protocol, mode: 'signal_replay' as const, waveformStages: true, objectivePolicy: policy } + const pythonBuildHash = 'a'.repeat(64), otherSideConfigHash = configHash(defaultConfig) + const job = { ...work.job, protocol, protocolHash: configHash(protocol), waveform: { pythonBuildHash, + otherSide: { resolved: defaultConfig, resolvedHash: otherSideConfigHash, manifestHash } } } + const evidence: ObjectiveEvidence = { metricCompatible: true, split: 'holdout', holdoutUse: 1, intervalUpper: -2, subgroups: [], operational: null, + nights: ['a', 'b', 'c'].map(cohortId => ({ cohortId, pairedReferences: 10, maeDifference: -2, baselineObservationCount: 10, lostObservations: 0 })) } + const report = { ...work.report, protocol, protocolHash: job.protocolHash, version: 'waveform-backtest-report-v1', + support: 'identical_retained_waveform_inputs', waveform: { pythonBuildHash, otherSideConfigHash, + ...replayContextFixture(['a', 'b', 'c'], job.configs.map(row => row.resolvedHash), pythonBuildHash, otherSideConfigHash) }, + objectiveAssessments: ['default', 'current'].map(baseline => ({ baseline, candidate: 'candidate', baselineHash: configHash(defaultConfig), + candidateHash: configHash(candidate), ...assessNumericObjective(policy, structuredClone(evidence), null) })) } + for (const row of report.stageObjectiveAssessments) { + row.candidateHash = configHash(candidate) + row.evidence.intervalLower = 0 + for (const night of row.evidence.nights) night.agreementDifference = 0 + } + const result = () => ({ report, contentHash: dataHash(report), available: true, currentHoldoutUseCount: 1 }) + const expected = { ...work.expected, resolvedHash: configHash(candidate), waveform: { pythonBuildHash, otherSideConfigHash } } + const designations = { ...work.designations, protocolHash: job.protocolHash, waveformContext: { allowedLimitations: ['event_time_replaces_arrival_time' as const] }, numeric: policy } + expect(verifyEvaluationAssessments(job, result(), expected, designations)).toMatchObject({ changedDomains: ['heart_rate'], + numericAssessments: [{ status: 'improved' }, { status: 'improved' }], stageAssessments: [{ status: 'no_material_change' }, { status: 'no_material_change' }] }) + expect(verifyEvaluationAssessments(job, result(), expected, designations)).toMatchObject({ policies: { numeric: policy }, waveform: expected.waveform }) + report.objectiveAssessments[1].evidence.intervalUpper = 0 + expect(() => verifyEvaluationAssessments(job, result(), expected, designations)).toThrow('does not support') + candidate.breathing_rate.intervalLowerRatio = 0.6 + job.configs[2].resolvedHash = configHash(candidate) + expected.resolvedHash = configHash(candidate) + expect(() => verifyEvaluationAssessments(job, result(), expected, designations)).toThrow('numeric component') + + const breathingPolicy: ObjectivePolicy = { ...policy, metric: 'breathing_rate' } + const rest = { ...work.job.protocol, mode: protocol.mode, waveformStages: true } + const multiProtocol = { ...rest, objectivePolicies: [policy, breathingPolicy] } + const multiJob = { ...job, protocol: multiProtocol, protocolHash: configHash(multiProtocol) } + const multiReport = { ...report, protocol: multiProtocol, protocolHash: multiJob.protocolHash, + waveform: { pythonBuildHash, otherSideConfigHash, ...replayContextFixture(['a', 'b', 'c'], multiJob.configs.map(row => row.resolvedHash), pythonBuildHash, otherSideConfigHash) }, + stageObjectiveAssessments: report.stageObjectiveAssessments.map(row => ({ ...row, candidateHash: configHash(candidate) })), + objectiveAssessments: multiProtocol.objectivePolicies.flatMap(item => ['default', 'current'].map(baseline => ({ + baseline, candidate: 'candidate', baselineHash: configHash(defaultConfig), candidateHash: configHash(candidate), + ...assessNumericObjective(item, structuredClone(evidence), null), + }))) } + const multiResult = () => ({ ...result(), report: multiReport, contentHash: dataHash(multiReport) }) + const qualified = { ...designations, protocolHash: multiJob.protocolHash, numeric: multiProtocol.objectivePolicies } + expect(verifyEvaluationAssessments(multiJob, multiResult(), expected, qualified)).toMatchObject({ + changedDomains: ['heart_rate', 'breathing_rate'], numericAssessments: [ + { metric: 'heart_rate', status: 'improved' }, { metric: 'heart_rate', status: 'improved' }, + { metric: 'breathing_rate', status: 'improved' }, { metric: 'breathing_rate', status: 'improved' }, + ], + }) + expect(() => verifyEvaluationAssessments(multiJob, multiResult(), expected, designations)).toThrow() + multiReport.objectiveAssessments[3].evidence.nights.pop() + expect(() => verifyEvaluationAssessments(multiJob, multiResult(), expected, qualified)).toThrow() +}) + +it('does not transfer threshold qualification to changed alignment, calibration or uncertainty settings', () => { + for (const patch of [ + { clockOffsetMs: 1000 }, + { pointToleranceMs: 1000 }, + { assumedStageCalibrationQuality: 0.75 }, + { nightUncertainty: { version: 'paired-night-percentile-v1' as const, seed: 'changed-seed', resamples: 200, minimumNights: 3 } }, + ]) { + const work = fixture() + const protocol = { ...work.job.protocol, ...patch } + const job = { ...work.job, protocol, protocolHash: configHash(protocol) } + const report = { ...work.report, protocol, protocolHash: job.protocolHash } + const result = { ...work.result(), report, contentHash: dataHash(report) } + expect(() => verifyEvaluationAssessments(job, result, work.expected, work.designations)).toThrow('qualified context') + expect(verifyEvaluationAssessments(job, result, work.expected, { ...work.designations, protocolHash: job.protocolHash })) + .toMatchObject({ protocolHash: job.protocolHash }) + } + const work = fixture() + expect(() => verifyEvaluationAssessments(work.job, work.result(), work.expected, { ...work.designations, protocolHash: null })) + .toThrow('qualified context') +}) + +it('derives aggregate status from reassessed evidence without accepting a claimed report status', async () => { + const { assessEvaluation } = await import('../evaluation-assessment') + const work = fixture() + expect(assessEvaluation(work.job, work.result(), work.expected, work.designations).status).toBe('improved') + work.report.stageObjectiveAssessments[0].status = 'improved' + work.report.stageObjectiveAssessments[0].evidence.intervalLower = 0 + for (const night of work.report.stageObjectiveAssessments[0].evidence.nights) night.agreementDifference = 0 + expect(assessEvaluation(work.job, work.result(), work.expected, work.designations).status).toBe('no_material_change') + work.report.stageObjectiveAssessments[0].evidence.nights[0].candidateOutputCoverage = 0.1 + expect(assessEvaluation(work.job, work.result(), work.expected, work.designations).status).toBe('regressed') + work.report.jobOperations.memory.allSamplesAvailable = false + expect(assessEvaluation(work.job, work.result(), work.expected, work.designations).status).toBe('insufficient_evidence') + expect(() => verifyEvaluationAssessments(work.job, work.result(), work.expected, work.designations)).toThrow('does not support') +}) diff --git a/src/biometrics/profiles/tests/evaluation-identity.test.ts b/src/biometrics/profiles/tests/evaluation-identity.test.ts new file mode 100644 index 000000000..f3c458c85 --- /dev/null +++ b/src/biometrics/profiles/tests/evaluation-identity.test.ts @@ -0,0 +1,135 @@ +// @vitest-environment node +import { expect, it } from 'vitest' +import { verifyEvaluationIdentity, type EvaluationIdentityExpectation } from '../evaluation-identity' +import { backtestFixture } from '../../backtest/tests/fixtures/backtest' +import { configHash, dataHash } from '../../algorithms/canonical' +import { defaultConfig, manifestHash } from '../../algorithms/registry' + +function fixture() { + const source = backtestFixture() + const candidate = { ...defaultConfig, heart_rate: { ...defaultConfig.heart_rate, trackingToleranceBpm: 20 } } + const revisionId = '33333333-3333-4333-8333-333333333333' + const job = { ...source.job, split: 'holdout' as const, holdoutUse: 1, configs: [...source.job.configs, + { label: 'candidate' as const, revisionId, resolved: candidate, resolvedHash: configHash(candidate), manifestHash }] } + const report = { version: 'processed-backtest-report-v1', side: 'left', datasetId: job.datasetId, datasetHash: job.datasetHash, + protocol: job.protocol, protocolHash: job.protocolHash, evaluatorBuildHash: job.evaluatorBuildHash, manifestHash, + split: 'holdout', holdoutUse: 1, support: 'identical_retained_epoch_intervals', nights: [{ cohortId: 'night' }] } + const expected: EvaluationIdentityExpectation = { side: 'left', revisionId, resolvedHash: configHash(candidate), + currentResolvedHash: configHash(defaultConfig), evaluatorBuildHash: job.evaluatorBuildHash, + dataset: { id: job.datasetId, contentHash: job.datasetHash, side: 'left', available: true, holdoutCohortIds: ['night'] } } + const result = { report, contentHash: dataHash(report), available: true, currentHoldoutUseCount: 1 } + return { job, report, expected, result } +} + +it('binds retained report bytes to all three configurations without awarding activation', () => { + const work = fixture() + expect(verifyEvaluationIdentity(work.job, work.result, work.expected)).toMatchObject({ contentHash: work.result.contentHash, cohortIds: ['night'] }) + expect(verifyEvaluationIdentity(work.job, work.result, work.expected)).not.toHaveProperty('status') +}) + +it('rejects unavailable or changed artifacts and reused holdout evidence', () => { + for (const patch of [{ available: false }, { contentHash: '0'.repeat(64) }, { currentHoldoutUseCount: 2 }]) { + const work = fixture() + expect(() => verifyEvaluationIdentity(work.job, { ...work.result, ...patch }, work.expected)).toThrow() + } + const work = fixture() + expect(() => verifyEvaluationIdentity(work.job, work.result, { ...work.expected, dataset: { ...work.expected.dataset, available: false } })).toThrow('unavailable') + expect(() => verifyEvaluationIdentity({ ...work.job, split: 'validation' }, work.result, work.expected)).toThrow('holdout') + expect(() => verifyEvaluationIdentity({ ...work.job, holdoutUse: 2 }, work.result, work.expected)).toThrow('holdout') +}) + +it('rejects another revision, baseline, build, side, or frozen dataset even with matching report bytes', () => { + const work = fixture() + for (const patch of [{ revisionId: '44444444-4444-4444-8444-444444444444' }, { resolvedHash: '0'.repeat(64) }, + { currentResolvedHash: '0'.repeat(64) }, { evaluatorBuildHash: '0'.repeat(64) }, { side: 'right' as const }, + { dataset: { ...work.expected.dataset, contentHash: '0'.repeat(64) } }]) { + expect(() => verifyEvaluationIdentity(work.job, work.result, { ...work.expected, ...patch })).toThrow() + } +}) + +it('requires all exact frozen cohorts, a replay protocol, and matching protocol contents', () => { + const work = fixture() + for (const nights of [[], [{ cohortId: 'night' }, { cohortId: 'night' }], [{ cohortId: 'other' }]]) { + const report = { ...work.report, nights } + expect(() => verifyEvaluationIdentity(work.job, { ...work.result, report, contentHash: dataHash(report) }, work.expected)).toThrow() + } + const protocol = { ...work.job.protocol, clockOffsetMs: 1000 } + const report = { ...work.report, protocol } + expect(() => verifyEvaluationIdentity(work.job, { ...work.result, report, contentHash: dataHash(report) }, work.expected)).toThrow('protocol') + const comparison = { ...work.job.protocol, mode: 'comparison' as const } + const comparisonReport = { ...work.report, protocol: comparison, protocolHash: configHash(comparison) } + expect(() => verifyEvaluationIdentity({ ...work.job, protocol: comparison, protocolHash: configHash(comparison) }, + { ...work.result, report: comparisonReport, contentHash: dataHash(comparisonReport) }, work.expected)).toThrow('mode') +}) + +it('checks independently altered job and report identities, not only their content hashes', () => { + const work = fixture() + const otherId = '44444444-4444-4444-8444-444444444444' + for (const patch of [{ side: 'right' }, { datasetId: otherId }, { datasetHash: '0'.repeat(64) }, + { evaluatorBuildHash: '0'.repeat(64) }, { manifestHash: '0'.repeat(64) }, { protocolHash: '0'.repeat(64) }, + { support: 'identical_retained_waveform_inputs' }, { version: 'waveform-backtest-report-v1' }]) { + const report = { ...work.report, ...patch } + expect(() => verifyEvaluationIdentity(work.job, { ...work.result, report, contentHash: dataHash(report) }, work.expected)).toThrow() + } + for (const patch of [{ datasetId: otherId }, { datasetHash: '0'.repeat(64) }, { protocolHash: '0'.repeat(64) }, + { evidence: 'reused_holdout_is_validation' }]) { + expect(() => verifyEvaluationIdentity({ ...work.job, ...patch }, work.result, work.expected)).toThrow() + } + expect(() => verifyEvaluationIdentity(work.job, work.result, { ...work.expected, dataset: { ...work.expected.dataset, side: 'right' } })).toThrow('side changed') + for (const holdoutCohortIds of [['night', 'night'], ['night', 'missing']]) { + expect(() => verifyEvaluationIdentity(work.job, work.result, { ...work.expected, dataset: { ...work.expected.dataset, holdoutCohortIds } })).toThrow('exact frozen') + } +}) + +it('does not let mutually consistent replacements bypass the service expectations', () => { + const work = fixture() + const otherId = '44444444-4444-4444-8444-444444444444' + const report = { ...work.report, datasetId: otherId } + expect(() => verifyEvaluationIdentity({ ...work.job, datasetId: otherId }, + { ...work.result, report, contentHash: dataHash(report) }, work.expected)).toThrow('dataset') + expect(() => verifyEvaluationIdentity({ ...work.job, protocol: { ...work.job.protocol, clockOffsetMs: 1 } }, work.result, work.expected)).toThrow('protocol') + const configs = work.job.configs.map(row => row.label === 'current' ? { ...row, resolvedHash: '0'.repeat(64) } : row) + expect(() => verifyEvaluationIdentity({ ...work.job, configs }, work.result, { ...work.expected, currentResolvedHash: '0'.repeat(64) })).toThrow('configuration identity') + const expected = { ...work.expected, dataset: { ...work.expected.dataset, holdoutCohortIds: ['night', 'second'] } } + for (const cohortIds of [['night', 'other'], ['night', 'night']]) { + const altered = { ...work.report, nights: cohortIds.map(cohortId => ({ cohortId })) } + expect(() => verifyEvaluationIdentity(work.job, { ...work.result, report: altered, contentHash: dataHash(altered) }, expected)).toThrow('exact frozen') + } +}) + +it('rejects mislabeled defaults, duplicated baseline labels and changed resolved snapshots', () => { + const work = fixture() + const [baseline, current, candidate] = work.job.configs + for (const configs of [[baseline, candidate], [baseline, baseline, candidate], + [{ ...candidate, label: 'default', revisionId: null }, current, candidate], + [{ ...baseline, revisionId: candidate.revisionId }, current, candidate], + [{ ...baseline, manifestHash: '0'.repeat(64) }, current, candidate], + [{ ...baseline, resolvedHash: '0'.repeat(64) }, current, candidate]]) { + expect(() => verifyEvaluationIdentity({ ...work.job, configs }, work.result, work.expected)).toThrow() + } +}) + +it('pins waveform Python and cross-side dependencies separately from the scoring build', () => { + const work = fixture() + const protocol = { ...work.job.protocol, mode: 'signal_replay' as const } + const pythonBuildHash = 'a'.repeat(64), otherSideConfigHash = configHash(defaultConfig) + const job = { ...work.job, protocol, protocolHash: configHash(protocol), waveform: { pythonBuildHash, + otherSide: { resolved: defaultConfig, resolvedHash: otherSideConfigHash, manifestHash } } } + const report = { ...work.report, version: 'waveform-backtest-report-v1', support: 'identical_retained_waveform_inputs', + protocol, protocolHash: job.protocolHash, waveform: { pythonBuildHash, otherSideConfigHash } } + const result = { ...work.result, report, contentHash: dataHash(report) } + const expected = { ...work.expected, waveform: { pythonBuildHash, otherSideConfigHash } } + expect(verifyEvaluationIdentity(job, result, expected).contentHash).toBe(result.contentHash) + expect(() => verifyEvaluationIdentity(job, result, work.expected)).toThrow('Waveform') + for (const waveform of [{ ...expected.waveform, pythonBuildHash: 'b'.repeat(64) }, { ...expected.waveform, otherSideConfigHash: 'b'.repeat(64) }]) { + expect(() => verifyEvaluationIdentity(job, result, { ...expected, waveform })).toThrow('Waveform') + } + for (const waveform of [{ ...report.waveform, pythonBuildHash: 'b'.repeat(64) }, { ...report.waveform, otherSideConfigHash: 'b'.repeat(64) }]) { + const altered = { ...report, waveform } + expect(() => verifyEvaluationIdentity(job, { ...result, report: altered, contentHash: dataHash(altered) }, expected)).toThrow('Waveform') + } + for (const otherSide of [{ ...job.waveform.otherSide, manifestHash: 'b'.repeat(64) }, + { ...job.waveform.otherSide, resolved: work.job.configs[2].resolved }]) { + expect(() => verifyEvaluationIdentity({ ...job, waveform: { ...job.waveform, otherSide } }, result, expected)).toThrow('Waveform') + } +}) diff --git a/src/biometrics/profiles/tests/fixtures/evaluation.ts b/src/biometrics/profiles/tests/fixtures/evaluation.ts new file mode 100644 index 000000000..0276d872b --- /dev/null +++ b/src/biometrics/profiles/tests/fixtures/evaluation.ts @@ -0,0 +1,39 @@ +import type { EvaluationIdentityExpectation } from '../../evaluation-identity' +import { stageObjectiveTestPolicy } from '../../../backtest/tests/fixtures/stage-objective' +import { backtestFixture } from '../../../backtest/tests/fixtures/backtest' +import { assessStageObjective, type StageObjectiveEvidence } from '../../../backtest/stage-objective' +import { configHash, dataHash } from '../../../algorithms/canonical' +import { defaultConfig, manifestHash } from '../../../algorithms/registry' + +export function evaluationFixture() { + const source = backtestFixture() + const policy = { ...stageObjectiveTestPolicy, operational: { ...stageObjectiveTestPolicy.operational, + observations: { version: 'sampled-process-group-v1' as const, maximumMemoryGapMs: 1000, maximumIngestionGapMs: 5000 } } } + const candidate = structuredClone(defaultConfig) + candidate.sleep_stages.ruleset.smoothIsolatedEpochs = false + const revisionId = '33333333-3333-4333-8333-333333333333' + const protocol = { ...source.job.protocol, stageObjectivePolicy: policy } + const job = { ...source.job, protocol, protocolHash: configHash(protocol), split: 'holdout' as const, holdoutUse: 1, + configs: [...source.job.configs, { label: 'candidate' as const, revisionId, resolved: candidate, resolvedHash: configHash(candidate), manifestHash }] } + const evidence: StageObjectiveEvidence = { split: 'holdout', holdoutUse: 1, intervalLower: 0.125, operational: null, + nights: ['a', 'b', 'c'].map(cohortId => ({ cohortId, pairedDurationMs: 800, agreementDifference: 0.125, macroF1Difference: 0.125, + candidateOutputCoverage: 0.9, baselineOutputMs: 1000, commonOutputMs: 900, + perStage: policy.requiredStages.map(stage => ({ stage, referenceMs: 200, f1Difference: 0.125 })) })) } + const report = { version: 'processed-backtest-report-v1', side: 'left', datasetId: job.datasetId, datasetHash: job.datasetHash, + protocol, protocolHash: job.protocolHash, evaluatorBuildHash: job.evaluatorBuildHash, manifestHash, + split: 'holdout', holdoutUse: 1, support: 'identical_retained_epoch_intervals', nights: evidence.nights.map(row => ({ cohortId: row.cohortId })), + stageObjectiveAssessments: ['default', 'current'].map(baseline => ({ baseline, candidate: 'candidate', baselineHash: configHash(defaultConfig), + candidateHash: configHash(candidate), ...assessStageObjective(policy, structuredClone(evidence), null) })), + jobOperations: { version: 'backtest-job-operations-v1', wallMs: 1000, scope: 'admission_through_prepublication', + memory: { version: 'process-group-rss-observations-v1', scope: 'sum_rss_of_replay_process_group', attempts: 3, samples: 3, + allSamplesAvailable: true, peakObservedRssBytes: 1000, maximumObservationGapMs: 1000, intervalMs: 1000, exactKernelPeak: false }, + ingestion: { source: 'supervisor_live_piezo_receipt_age', checks: 2, samples: 2, allChecksMeasured: true, qualificationId: policy.qualificationId, + maximumObservedAgeMs: 1000, maximumObservationGapMs: 5000, nominalIntervalMs: 5000, continuousMeasurement: false } } } + const expected: EvaluationIdentityExpectation = { side: 'left', revisionId, resolvedHash: configHash(candidate), + currentResolvedHash: configHash(defaultConfig), evaluatorBuildHash: job.evaluatorBuildHash, + dataset: { id: job.datasetId, contentHash: job.datasetHash, side: 'left', available: true, holdoutCohortIds: ['a', 'b', 'c'] } } + const result = () => ({ report, contentHash: dataHash(report), available: true, currentHoldoutUseCount: 1 }) + const designations = { protocolHash: job.protocolHash, waveformContext: null, numeric: null, stages: policy } + return { job, report, result, expected, designations } +} + diff --git a/src/biometrics/profiles/tests/fixtures/replay-context.ts b/src/biometrics/profiles/tests/fixtures/replay-context.ts new file mode 100644 index 000000000..b7aeebd1f --- /dev/null +++ b/src/biometrics/profiles/tests/fixtures/replay-context.ts @@ -0,0 +1,11 @@ +import { dataHash } from '../../../algorithms/canonical' +export function replayContextFixture(cohortIds: string[], configHashes: string[], pythonBuildHash: string, otherSideConfigHash: string) { + const outputs = cohortIds.flatMap(cohortId => [...new Set(configHashes)].map(configHash => ({ cohortId, configHash, + readings: [{ side: 'left' as const, timestampMs: 1000, heartRate: 60, hrvIndex: 30, hrvMetric: 'window_ibi_variability_v1' as const, + breathingRate: 12, qualityScore: 1, flags: [], hrRaw: 60, sourceSide: 'left' as const, configHash, limitations: [] as string[] }], + run: { version: 'retained-waveform-run-v1' as const, build_hash: pythonBuildHash, decoded_records: 1, raw_records: 1, + counts: { records: 1, accepted_frames: 1, empty_channel_frames: 0, pump_gated_frames: 0, preroll_outputs: 0, scored_outputs: 1 }, + limitations: ['event_time_replaces_arrival_time'], config_hashes: { left: configHash, right: otherSideConfigHash }, artifact_hashes: ['d'.repeat(64)] }, + }))) + return { outputs, hash: dataHash(outputs) } +} diff --git a/src/biometrics/profiles/tests/health.test.ts b/src/biometrics/profiles/tests/health.test.ts new file mode 100644 index 000000000..a62c4d11a --- /dev/null +++ b/src/biometrics/profiles/tests/health.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from 'vitest' +import { runtimeHealth } from '../health' +const now = 1791673200000 +const receipts = () => ['heart_rate', 'breathing_rate', 'hrv_index'].map(component => ({ component, + generation: 2, appliedHash: 'bundle', state: 'applied', effectiveSampleTimeMs: now - 50000, heartbeatAtMs: now - 30000 })) +describe('continuous processor health is distinct from historical adoption', () => { + it('requires all three numerical consumers and ignores on-demand stage receipt age', () => { + expect(runtimeHealth([], now).state).toBe('unreported') + expect(runtimeHealth(receipts().slice(1), now).state).toBe('unreported') + expect(runtimeHealth([...receipts(), { ...receipts()[0], component: 'sleep_stages', heartbeatAtMs: 0 }], now).state).toBe('current') + }) + it('reports stopped processors, future clocks, failure and incomplete application', () => { + expect(runtimeHealth(receipts(), now + 120000).state).toBe('stale') + expect(runtimeHealth(receipts(), now - 60000).state).toBe('clock_skew') + expect(runtimeHealth(receipts().map(row => ({ ...row, state: 'failed' })), now).state).toBe('failed') + expect(runtimeHealth(receipts().map(row => ({ ...row, effectiveSampleTimeMs: null })), now).state).toBe('unreported') + }) + it('does not call different generations or hashes one coherent current profile', () => { + const rows = receipts() + rows[1].generation = 3 + expect(runtimeHealth(rows, now).state).toBe('mixed_generations') + rows[1].generation = 2 + rows[1].appliedHash = 'other' + expect(runtimeHealth(rows, now).state).toBe('mixed_generations') + rows[1] = { ...rows[0], component: 'breathing_rate' } + expect(runtimeHealth([{ ...rows[0], generation: 1, heartbeatAtMs: now }, ...rows], now).state).toBe('current') + }) +}) diff --git a/src/biometrics/profiles/tests/history.test.ts b/src/biometrics/profiles/tests/history.test.ts new file mode 100644 index 000000000..6a789e4a7 --- /dev/null +++ b/src/biometrics/profiles/tests/history.test.ts @@ -0,0 +1,31 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { expect, it } from 'vitest' +import * as schema from '@/src/db/schema' +import { requestManualActivation } from '../activation' +import { listActivationHistory } from '../history' +import { defaultConfig } from '../../algorithms/registry' +import { configHash } from '../../algorithms/canonical' +it('paginates immutable side-specific audit history without leaking internal request keys', () => { + const raw = new Database(':memory:') + try { + const db = drizzle(raw, { schema }) + migrate(db, { migrationsFolder: 'src/db/migrations' }) + for (let generation = 0; generation < 53; generation++) requestManualActivation(db, { + side: 'left', action: 'reset', revisionId: null, expectedGeneration: generation, expectedResolvedHash: configHash(defaultConfig), + boundary: 'immediate', requester: 'fixture', idempotencyKey: `request-${generation}`, allowUnvalidated: true, + }) + const first = listActivationHistory(db, 'left') + expect(first.items).toHaveLength(50) + expect(first.items[0].generation).toBe(53) + expect(first.nextGeneration).toBe(4) + expect(first.items[0]).not.toHaveProperty('idempotencyKey') + const second = listActivationHistory(db, 'left', first.nextGeneration ?? undefined) + expect(second.items.map(row => row.generation)).toEqual([3, 2, 1]) + expect(second.nextGeneration).toBeNull() + expect(listActivationHistory(db, 'right').items).toEqual([]) + } + finally { raw.close() } +}) diff --git a/src/biometrics/profiles/tests/provenance-migration.test.ts b/src/biometrics/profiles/tests/provenance-migration.test.ts new file mode 100644 index 000000000..027424e60 --- /dev/null +++ b/src/biometrics/profiles/tests/provenance-migration.test.ts @@ -0,0 +1,26 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { readMigrationFiles } from 'drizzle-orm/migrator' +import { expect, it } from 'vitest' + +it('preserves old vitals through additive migration and accepts old writers afterward', () => { + const sqlite = new Database(':memory:') + try { + const migrations = readMigrationFiles({ migrationsFolder: 'src/db/biometrics-migrations' }) + const provenanceIndex = migrations.findIndex(migration => migration.sql.some(statement => statement.includes('ADD `processing_run_id`'))) + expect(provenanceIndex).toBeGreaterThan(0) + for (const migration of migrations.slice(0, provenanceIndex)) { + for (const statement of migration.sql) sqlite.exec(statement) + } + const insert = 'INSERT INTO vitals(side, timestamp, heart_rate, hrv, breathing_rate) VALUES (?, ?, ?, ?, ?)' + sqlite.prepare(insert).run('left', 1700000000, 60, 12, 15) + for (const migration of migrations.slice(provenanceIndex)) { + for (const statement of migration.sql) sqlite.exec(statement) + } + expect(sqlite.prepare('SELECT heart_rate, hrv, processing_run_id FROM vitals').get()) + .toEqual({ heart_rate: 60, hrv: 12, processing_run_id: null }) + sqlite.prepare(insert).run('right', 1700000060, null, null, 16) + expect(sqlite.prepare('SELECT count(*) AS total FROM vitals WHERE processing_run_id IS NULL').get()).toEqual({ total: 2 }) + } + finally { sqlite.close() } +}) diff --git a/src/biometrics/profiles/tests/qualification-registry.test.ts b/src/biometrics/profiles/tests/qualification-registry.test.ts new file mode 100644 index 000000000..dc2434b9e --- /dev/null +++ b/src/biometrics/profiles/tests/qualification-registry.test.ts @@ -0,0 +1,62 @@ +import { configHash } from '../../algorithms/canonical' +// @vitest-environment node +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { afterEach, beforeEach, expect, it } from 'vitest' +import { installedPromotionPolicies, unqualifiedPolicies } from '../qualification-registry' +import { evaluationFixture } from './fixtures/evaluation' +import { manifestHash } from '../../algorithms/registry' +let root: string +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'promotion-registry-')) +}) +afterEach(async () => { + await rm(root, { recursive: true, force: true }) +}) +const fixture = () => { + const work = evaluationFixture() + const entry = { qualificationId: work.designations.stages.qualificationId, manifestHash, evaluatorBuildHash: work.job.evaluatorBuildHash, + protocol: work.job.protocol, fixtureEvidenceHash: 'a'.repeat(64), podEvidenceHash: 'b'.repeat(64), waveformContext: null } + return { entry, identity: { evaluatorBuildHash: entry.evaluatorBuildHash, protocolHash: work.job.protocolHash } } +} +const install = (entries: unknown[]) => writeFile(join(root, 'promotion-qualifications.json'), JSON.stringify({ version: 'biometric-promotion-qualifications-v1', entries })) +it('requires a unique installed build/protocol qualification and returns its policies', async () => { + const { entry, identity } = fixture() + const path = join(root, 'worker.cjs') + expect(await installedPromotionPolicies(path, identity)).toEqual(unqualifiedPolicies()) + await install([entry]) + expect(await installedPromotionPolicies(path, identity)).toEqual({ protocolHash: identity.protocolHash, numeric: [], stages: entry.protocol.stageObjectivePolicy, waveformContext: null }) + for (const changed of [{ evaluatorBuildHash: 'f'.repeat(63) + '0' }, { protocolHash: 'c'.repeat(64) }, { pythonBuildHash: 'd'.repeat(64) }]) { + expect(await installedPromotionPolicies(path, { ...identity, ...changed })).toEqual(unqualifiedPolicies()) + } + await install([entry, entry]) + expect(await installedPromotionPolicies(path, identity)).toEqual(unqualifiedPolicies()) +}) +it('does not use malformed, cross-manifest, inconsistent or unbounded qualification files', async () => { + const { entry, identity } = fixture() + const path = join(root, 'worker.cjs') + for (const changed of [{ ...entry, manifestHash: '0'.repeat(64) }, { ...entry, qualificationId: 'unrelated' }, + { ...entry, podEvidenceHash: undefined }, { ...entry, waveformContext: { allowedLimitations: [] } }]) { + await install([changed]) + expect(await installedPromotionPolicies(path, identity)).toEqual(unqualifiedPolicies()) + } + for (const data of ['{', ' '.repeat(256 * 1024 + 1)]) { + await writeFile(join(root, 'promotion-qualifications.json'), data) + expect(await installedPromotionPolicies(path, identity)).toEqual(unqualifiedPolicies()) + } +}) + +it('requires waveform context qualification and the exact Python build for waveform policies', async () => { + const { entry } = fixture() + const protocol = { ...entry.protocol, mode: 'signal_replay' as const, waveformStages: true } + const pythonBuildHash = 'c'.repeat(64) + const identity = { evaluatorBuildHash: entry.evaluatorBuildHash, pythonBuildHash, protocolHash: configHash(protocol) } + const qualified = { ...entry, protocol, pythonBuildHash, waveformContext: { allowedLimitations: ['cold_start_without_serialized_state'] } } + await install([qualified]) + expect(await installedPromotionPolicies(join(root, 'worker.cjs'), identity)).toMatchObject({ + protocolHash: identity.protocolHash, waveformContext: qualified.waveformContext, + }) + await install([{ ...qualified, waveformContext: null }]) + expect(await installedPromotionPolicies(join(root, 'worker.cjs'), identity)).toEqual(unqualifiedPolicies()) +}) diff --git a/src/biometrics/profiles/tests/recovery.test.ts b/src/biometrics/profiles/tests/recovery.test.ts new file mode 100644 index 000000000..ebdfe2c99 --- /dev/null +++ b/src/biometrics/profiles/tests/recovery.test.ts @@ -0,0 +1,125 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import * as configSchema from '@/src/db/schema' +import * as bioSchema from '@/src/db/biometrics-schema' +import { configHash } from '../../algorithms/canonical' +import { defaultConfig, manifestHash } from '../../algorithms/registry' +import { requestManualActivation } from '../activation' +import { recoverRuntimeIntent } from '../recovery' +import { ProfileStore } from '../store' +import { deliverRuntimeOutbox, reconcileActivation, RUNTIME_COMPONENTS } from '../runtime' +import { tickBiometricRuntime } from '../runtime-host' +import { resolveStageView } from '../stage-view' +let configRaw: Database.Database, bioRaw: Database.Database +let config: ReturnType>, bio: ReturnType> +const now = 1791673200000 +beforeEach(() => { + configRaw = new Database(':memory:') + bioRaw = new Database(':memory:') + config = drizzle(configRaw, { schema: configSchema }) + bio = drizzle(bioRaw, { schema: bioSchema }) + migrate(config, { migrationsFolder: 'src/db/migrations' }) + migrate(bio, { migrationsFolder: 'src/db/biometrics-migrations' }) +}) +afterEach(() => { + configRaw.close() + bioRaw.close() +}) +function activate(custom = false) { + const revision = custom + ? new ProfileStore(config).createRevision({ side: 'left', overrides: [{ path: 'heart_rate.trackingToleranceBpm', value: 20 }], + origin: 'manual', requester: 'fixture', idempotencyKey: 'profile' }) + : null + requestManualActivation(config, { side: 'left', action: revision ? 'activate' : 'reset', revisionId: revision?.id ?? null, expectedGeneration: 0, + expectedResolvedHash: revision?.resolvedHash ?? configHash(defaultConfig), boundary: 'next_session', requester: 'fixture', idempotencyKey: 'activate', allowUnvalidated: true }, now) + return revision +} +function obsoleteProjection() { + const row = config.select().from(configSchema.biometricConfigOutbox).get() + if (!row) throw new Error('Missing projection') + configRaw.prepare('UPDATE biometric_config_outbox SET payload = ?').run(JSON.stringify({ ...row.payload as object, manifestHash: 'previous-build' })) +} +describe('runtime recovery across builds', () => { + it('keeps implicit defaults implicit and does not supersede valid next-session intent', () => { + expect(recoverRuntimeIntent(config, 'left', now)).toBeNull() + activate() + expect(recoverRuntimeIntent(config, 'left', now)).toBeNull() + expect(new ProfileStore(config).binding('left').generation).toBe(1) + }) + it('refreshes an obsolete shipped default once and preserves its audit explanation after acknowledgment', () => { + activate() + obsoleteProjection() + const event = recoverRuntimeIntent(config, 'left', now) + expect(event?.evidence).toMatchObject({ status: 'system_default_update', manifestHash }) + expect(event?.generation).toBe(2) + expect(recoverRuntimeIntent(config, 'left', now)).toBeNull() + deliverRuntimeOutbox(config, bio, now) + for (const component of RUNTIME_COMPONENTS) bio.insert(bioSchema.biometricRuntimeReceipts).values({ side: 'left', component, generation: 2, + appliedHash: configHash(defaultConfig), state: 'applied', effectiveSampleTimeMs: now, heartbeatAtMs: now }).run() + expect(reconcileActivation(config, bio, 'left', now)?.state).toBe('applied') + expect(config.select().from(configSchema.biometricActivationEvents).all().find(row => row.generation === 2)?.reason).toContain('Refresh shipped') + }) + it('preserves pinned intent and rollback history during fallback, including after biometric reset', () => { + const revision = activate(true) + if (!revision) throw new Error('Missing revision') + configRaw.prepare('UPDATE biometric_profile_revisions SET base_manifest_hash = ? WHERE id = ?').run('previous-build', revision.id) + expect(tickBiometricRuntime(config, bio, now).errors).toEqual([]) + const binding = new ProfileStore(config).binding('left') + expect(binding).toMatchObject({ desiredRevisionId: revision.id, previousRevisionId: null, generation: 2 }) + const projection = bio.select().from(bioSchema.biometricRuntimeConfigs).get() + expect(projection?.payload).toMatchObject({ revisionId: null, resolvedHash: configHash(defaultConfig) }) + expect(config.select().from(configSchema.biometricActivationEvents).all().find(row => row.generation === 2)?.evidence).toMatchObject({ status: 'profile_incompatible', requestedRevisionId: revision.id }) + bio.delete(bioSchema.biometricRuntimeConfigs).run() + tickBiometricRuntime(config, bio, now) + expect(new ProfileStore(config).binding('left').generation).toBe(2) + expect(bio.select().from(bioSchema.biometricRuntimeConfigs).get()?.generation).toBe(2) + for (const component of RUNTIME_COMPONENTS.slice(0, 3)) bio.insert(bioSchema.biometricRuntimeReceipts).values({ side: 'left', component, generation: 2, + appliedHash: configHash(defaultConfig), state: 'applied', effectiveSampleTimeMs: now, heartbeatAtMs: now }).run() + expect(resolveStageView(config, bio, 'left').provenance.warning).toContain('PROFILE_INCOMPATIBLE') + expect(config.select().from(configSchema.biometricProfileRevisions).get()?.id).toBe(revision.id) + }) + it('restores a compatible-again pinned revision at the next session, never silently rewriting old snapshots', () => { + const revision = activate(true) + if (!revision) throw new Error('Missing revision') + configRaw.prepare('UPDATE biometric_profile_revisions SET base_manifest_hash = ?').run('previous-build') + recoverRuntimeIntent(config, 'left', now) + configRaw.prepare('UPDATE biometric_profile_revisions SET base_manifest_hash = ?').run(manifestHash) + const restored = recoverRuntimeIntent(config, 'left', now) + expect(restored?.evidence).toMatchObject({ status: 'profile_restored', actualRevisionId: revision.id }) + expect(config.select().from(configSchema.biometricConfigOutbox).all().find(row => row.generation === 3)?.payload) + .toMatchObject({ boundary: 'next_session', resolvedHash: revision.resolvedHash }) + expect(new ProfileStore(config).binding('left').desiredRevisionId).toBe(revision.id) + }) + it('repairs a missing custom intent conservatively at the next session', () => { + activate(true) + config.delete(configSchema.biometricConfigOutbox).run() + const event = recoverRuntimeIntent(config, 'left', now) + expect(event?.evidence).toMatchObject({ status: 'runtime_repair' }) + expect(config.select().from(configSchema.biometricConfigOutbox).get()?.payload).toMatchObject({ boundary: 'next_session' }) + }) + it('repairs conflicting biometric projections through a new generation while retaining the corrupt row for diagnosis', () => { + activate(true) + tickBiometricRuntime(config, bio, now) + bioRaw.prepare('UPDATE biometric_runtime_configs SET payload = ?').run('{}') + tickBiometricRuntime(config, bio, now + 1) + expect(config.select().from(configSchema.biometricConfigOutbox).get()?.state).toBe('failed') + tickBiometricRuntime(config, bio, now + 2) + expect(new ProfileStore(config).binding('left').generation).toBe(2) + const rows = bio.select().from(bioSchema.biometricRuntimeConfigs).all() + expect(rows).toHaveLength(2) + expect(rows.find(row => row.generation === 1)?.payload).toEqual({}) + expect(rows.find(row => row.generation === 2)?.payload).toMatchObject({ boundary: 'next_session' }) + }) + it('rolls back all recovery changes when outbox persistence fails', () => { + activate() + obsoleteProjection() + configRaw.exec('CREATE TRIGGER reject_recovery BEFORE INSERT ON biometric_config_outbox BEGIN SELECT RAISE(ABORT, \'full\'); END;') + expect(() => recoverRuntimeIntent(config, 'left', now)).toThrow('full') + expect(new ProfileStore(config).binding('left').generation).toBe(1) + expect(config.select().from(configSchema.biometricActivationEvents).all()).toHaveLength(1) + expect(config.select().from(configSchema.biometricConfigOutbox).get()?.state).toBe('pending') + }) +}) diff --git a/src/biometrics/profiles/tests/replay-context.test.ts b/src/biometrics/profiles/tests/replay-context.test.ts new file mode 100644 index 000000000..5eeb5a04b --- /dev/null +++ b/src/biometrics/profiles/tests/replay-context.test.ts @@ -0,0 +1,56 @@ +// @vitest-environment node +import { expect, it } from 'vitest' +import { verifyReplayContext, replayContextQualificationSchema } from '../replay-context' +import { replayContextFixture } from './fixtures/replay-context' +import { dataHash } from '../../algorithms/canonical' +const expected = { cohortIds: ['a', 'b'], configHashes: ['a'.repeat(64), 'b'.repeat(64), 'a'.repeat(64)], + pythonBuildHash: 'c'.repeat(64), otherSideConfigHash: 'e'.repeat(64), side: 'left' as const } +const qualified = { allowedLimitations: ['event_time_replaces_arrival_time' as const] } +const fixture = () => replayContextFixture(expected.cohortIds, expected.configHashes, expected.pythonBuildHash, expected.otherSideConfigHash) +it('audits exact distinct cohort/configuration outputs and explicitly accepted approximations', () => { + expect(verifyReplayContext(fixture(), expected, qualified)).toEqual({ version: 'qualified-replay-context-v1', + outputs: 4, observedLimitations: ['event_time_replaces_arrival_time'], allowedLimitations: ['event_time_replaces_arrival_time'] }) + expect(() => verifyReplayContext(fixture(), expected, null)).toThrow('not been qualified') +}) +it.each(['missing_pump_context', 'missing_occupancy_context', 'missing_sleeper_mode', 'less_than_five_minutes_preroll', + 'cold_start_without_serialized_state', 'future_unknown_limitation'])('rejects unqualified %s in run or individual reading diagnostics', (code) => { + for (const reading of [true, false]) { + const work = fixture() + if (reading) work.outputs[3].readings[0].limitations.push(code) + else work.outputs[3].run.limitations.push(code) + work.hash = dataHash(work.outputs) + expect(() => verifyReplayContext(work, expected, qualified)).toThrow(`Unqualified replay limitation: ${code}`) + } +}) +it('rejects missing, duplicate, altered and inconsistent output identities', () => { + for (const change of [ + (work: ReturnType) => { work.outputs.pop() }, + (work: ReturnType) => { work.outputs[3] = work.outputs[0] }, + (work: ReturnType) => { work.outputs[0].cohortId = 'unknown' }, + (work: ReturnType) => { work.outputs[0].run.build_hash = 'f'.repeat(64) }, + (work: ReturnType) => { work.outputs[0].run.config_hashes.right = 'f'.repeat(64) }, + (work: ReturnType) => { work.outputs[0].run.counts.scored_outputs = 2 }, + (work: ReturnType) => { work.outputs[0].readings[0].configHash = 'f'.repeat(64) }, + ]) { + const work = fixture() + change(work) + work.hash = dataHash(work.outputs) + expect(() => verifyReplayContext(work, expected, qualified)).toThrow() + } + expect(() => verifyReplayContext({ ...fixture(), hash: 'f'.repeat(64) }, expected, qualified)).toThrow('content hash') +}) + +it('accepts only explicitly designated known limitations and records the observed subset', () => { + const work = fixture() + work.outputs[0].run.limitations.push('cold_start_without_serialized_state') + work.hash = dataHash(work.outputs) + const qualification = replayContextQualificationSchema.parse({ allowedLimitations: [ + 'event_time_replaces_arrival_time', 'cold_start_without_serialized_state', 'less_than_five_minutes_preroll', + ] }) + expect(verifyReplayContext(work, expected, qualification)).toMatchObject({ observedLimitations: [ + 'cold_start_without_serialized_state', 'event_time_replaces_arrival_time', + ] }) + for (const allowedLimitations of [['unknown_future_code'], ['missing_pump_context', 'missing_pump_context']]) { + expect(replayContextQualificationSchema.safeParse({ allowedLimitations }).success).toBe(false) + } +}) diff --git a/src/biometrics/profiles/tests/runtime-host.test.ts b/src/biometrics/profiles/tests/runtime-host.test.ts new file mode 100644 index 000000000..d6bf555fe --- /dev/null +++ b/src/biometrics/profiles/tests/runtime-host.test.ts @@ -0,0 +1,52 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import * as configSchema from '@/src/db/schema' +import * as bioSchema from '@/src/db/biometrics-schema' +import { DEFAULT_STAGE_CLASSIFIER } from '@/src/lib/sleep-stage-rules' +import { startBiometricRuntime, stopBiometricRuntime, tickBiometricRuntime } from '../runtime-host' +let configRaw: Database.Database, bioRaw: Database.Database +let config: ReturnType>, bio: ReturnType> +beforeEach(() => { + configRaw = new Database(':memory:') + bioRaw = new Database(':memory:') + config = drizzle(configRaw, { schema: configSchema }) + bio = drizzle(bioRaw, { schema: bioSchema }) + migrate(config, { migrationsFolder: 'src/db/migrations' }) + migrate(bio, { migrationsFolder: 'src/db/biometrics-migrations' }) + config.insert(configSchema.sideSettings).values([{ side: 'left', name: 'Left', stageClassifier: DEFAULT_STAGE_CLASSIFIER }, { side: 'right', name: 'Right' }]).run() +}) +afterEach(() => { + stopBiometricRuntime() + vi.useRealTimers() + configRaw.close() + bioRaw.close() +}) +it('migrates once, delivers, and repairs projections after history reset without inventing adoption', () => { + expect(tickBiometricRuntime(config, bio).errors).toEqual([]) + expect(tickBiometricRuntime(config, bio).errors).toEqual([]) + expect(config.select().from(configSchema.biometricProfileRevisions).all()).toHaveLength(1) + expect(bio.select().from(bioSchema.biometricRuntimeConfigs).all()).toHaveLength(1) + expect(config.select().from(configSchema.biometricActivationEvents).get()?.state).toBe('pending_boundary') + bio.delete(bioSchema.biometricRuntimeConfigs).run() + expect(tickBiometricRuntime(config, bio).errors).toEqual([]) + expect(bio.select().from(bioSchema.biometricRuntimeConfigs).all()).toHaveLength(1) + expect(bio.select().from(bioSchema.biometricRuntimeReceipts).all()).toEqual([]) +}) +it('continues other recovery work when one legacy side is malformed', () => { + configRaw.prepare('UPDATE side_settings SET stage_classifier = ? WHERE side = ?').run('{}', 'right') + expect(tickBiometricRuntime(config, bio).errors).toEqual(['migration:right']) + expect(bio.select().from(bioSchema.biometricRuntimeConfigs).get()?.side).toBe('left') +}) +it('shares one timer across starts and stops cleanly', () => { + vi.useFakeTimers() + startBiometricRuntime(config, bio) + startBiometricRuntime(config, bio) + expect(vi.getTimerCount()).toBe(1) + vi.advanceTimersByTime(5000) + expect(config.select().from(configSchema.biometricProfileRevisions).all()).toHaveLength(1) + stopBiometricRuntime() + expect(vi.getTimerCount()).toBe(0) +}) diff --git a/src/biometrics/profiles/tests/runtime-integration.test.ts b/src/biometrics/profiles/tests/runtime-integration.test.ts new file mode 100644 index 000000000..bcd6c92ec --- /dev/null +++ b/src/biometrics/profiles/tests/runtime-integration.test.ts @@ -0,0 +1,96 @@ +// @vitest-environment node +import { execFileSync } from 'node:child_process' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { expect, it, vi } from 'vitest' +import * as configSchema from '@/src/db/schema' +import * as bioSchema from '@/src/db/biometrics-schema' +import { tickBiometricRuntime } from '../runtime-host' +import { consumeStageView } from '../stage-view' +const holder = vi.hoisted(() => ({ config: null as unknown, bio: null as unknown })) +vi.mock('@/src/db', () => ({ + get db() { return holder.config }, + get biometricsDb() { return holder.bio }, +})) +const { biometricTuningRouter } = await import('@/src/server/routers/biometricTuning') +const caller = biometricTuningRouter.createCaller({}) + +// No daemon main loop, sensor transport, hardware or user databases. This runs +// the shipped live adapter and estimator against one bounded synthetic frame. +function frame(path: string, occupied: boolean) { + const output = execFileSync(resolve('modules/piezo-processor/.venv/bin/python'), ['-c', ` +import json, sqlite3, sys +sys.path.insert(0, 'modules') +sys.path.insert(0, 'modules/piezo-processor') +import numpy as np +from main import DBHolder, SideProcessor +from runtime_adapter import RuntimeCoordinator +connection = sqlite3.connect(sys.argv[1]) +holder = DBHolder(connection) +processors = {side: SideProcessor(side, holder) for side in ('left', 'right')} +runtime = RuntimeCoordinator(holder, processors) +runtime.before_frame(None, lambda side: sys.argv[2] == 'true', 1791673200000) +for processor in processors.values(): + processor.ingest(np.zeros(500)) +runtime.after_frame(1791673200000) +print(json.dumps({'enabled': runtime.enabled, 'problem': runtime.problem, + 'leftTolerance': processors['left'].parameters.tracking_tolerance_bpm, + 'rightTolerance': processors['right'].parameters.tracking_tolerance_bpm})) +connection.close() +`, path, String(occupied)], { encoding: 'utf8', timeout: 15000, maxBuffer: 65536 }) + return JSON.parse(output) as { enabled: boolean, problem: string | null, leftTolerance: number, rightTolerance: number } +} + +it('acknowledges exact API intent through Python and stage consumers, restart, and rollback', async () => { + const directory = mkdtempSync(join(tmpdir(), 'biometric-runtime-integration-')) + const configRaw = new Database(join(directory, 'config.db')) + const bioPath = join(directory, 'biometrics.db') + const bioRaw = new Database(bioPath) + try { + const config = drizzle(configRaw, { schema: configSchema }) + const bio = drizzle(bioRaw, { schema: bioSchema }) + migrate(config, { migrationsFolder: 'src/db/migrations' }) + migrate(bio, { migrationsFolder: 'src/db/biometrics-migrations' }) + holder.config = config + holder.bio = bio + const defaults = await caller.resolveProfile({ overrides: [] }) + const revision = await caller.createProfileRevision({ side: 'left', overrides: [{ path: 'heart_rate.trackingToleranceBpm', value: 20 }], + origin: 'manual', requester: 'integration', idempotencyKey: 'candidate' }) + const event = await caller.activateProfile({ side: 'left', revisionId: revision.id, expectedGeneration: 0, + expectedResolvedHash: revision.resolvedHash, boundary: 'next_session', requester: 'integration', idempotencyKey: 'activate', allowUnvalidated: true }) + tickBiometricRuntime(config, bio) + expect(frame(bioPath, true)).toMatchObject({ enabled: true, problem: null, leftTolerance: 15 }) + expect(await caller.getActivationStatus({ side: 'left', id: event.id })).toMatchObject({ observedState: 'pending_boundary', receipts: [] }) + expect(frame(bioPath, false)).toMatchObject({ problem: null, leftTolerance: 20, rightTolerance: 15 }) + expect(await caller.getActivationStatus({ side: 'left', id: event.id })).toMatchObject({ observedState: 'pending_boundary' }) + expect(consumeStageView(config, bio, 'left').provenance.bundleHash).toBe(revision.resolvedHash) + expect(await caller.getActivationStatus({ side: 'left', id: event.id })).toMatchObject({ observedState: 'applied', receipts: expect.arrayContaining([ + expect.objectContaining({ component: 'sleep_stages', appliedHash: revision.resolvedHash }), + ]) }) + // Status polling cannot turn a mismatched receipt into an acknowledgment. + bioRaw.prepare('UPDATE biometric_runtime_receipts SET applied_hash = \'wrong\' WHERE component = \'heart_rate\'').run() + const changes = bioRaw.prepare('SELECT total_changes() AS count').get() + expect(await caller.getActivationStatus({ side: 'left', id: event.id })).toMatchObject({ observedState: 'pending_boundary' }) + expect(bioRaw.prepare('SELECT total_changes() AS count').get()).toEqual(changes) + bioRaw.prepare('UPDATE biometric_runtime_receipts SET applied_hash = ? WHERE component = \'heart_rate\'').run(revision.resolvedHash) + // New Python process restores the acknowledged profile despite an occupied bed. + expect(frame(bioPath, true).leftTolerance).toBe(20) + const rollback = await caller.rollbackProfile({ side: 'left', revisionId: null, expectedGeneration: 1, + expectedResolvedHash: defaults.hash, boundary: 'immediate', requester: 'integration', idempotencyKey: 'rollback', allowUnvalidated: true }) + tickBiometricRuntime(config, bio) + expect(frame(bioPath, true).leftTolerance).toBe(15) + consumeStageView(config, bio, 'left') + expect(await caller.getActivationStatus({ side: 'left', id: rollback.id })).toMatchObject({ observedState: 'applied', isCurrent: true }) + expect(await caller.getActivationStatus({ side: 'left', id: event.id })).toMatchObject({ isCurrent: false }) + expect(bioRaw.prepare('SELECT count(*) AS count FROM vitals').get()).toEqual({ count: 0 }) + } + finally { + configRaw.close() + bioRaw.close() + rmSync(directory, { recursive: true, force: true }) + } +}, 30000) diff --git a/src/biometrics/profiles/tests/runtime.test.ts b/src/biometrics/profiles/tests/runtime.test.ts new file mode 100644 index 000000000..822e5fcb4 --- /dev/null +++ b/src/biometrics/profiles/tests/runtime.test.ts @@ -0,0 +1,67 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import * as configSchema from '@/src/db/schema' +import * as bioSchema from '@/src/db/biometrics-schema' +import { requestManualActivation } from '../activation' +import { deliverRuntimeOutbox, projectRuntimeConfig, reconcileActivation, repairCurrentRuntimeProjections, RUNTIME_COMPONENTS } from '../runtime' +import { configHash } from '../../algorithms/canonical' +import { defaultConfig } from '../../algorithms/registry' +let configSqlite: Database.Database, bioSqlite: Database.Database +let configDb: ReturnType>, bioDb: ReturnType> +const now = 1791673200000 +beforeEach(() => { + configSqlite = new Database(':memory:') + bioSqlite = new Database(':memory:') + configDb = drizzle(configSqlite, { schema: configSchema }) + bioDb = drizzle(bioSqlite, { schema: bioSchema }) + migrate(configDb, { migrationsFolder: 'src/db/migrations' }) + migrate(bioDb, { migrationsFolder: 'src/db/biometrics-migrations' }) + requestManualActivation(configDb, { side: 'left', action: 'reset', revisionId: null, expectedGeneration: 0, expectedResolvedHash: configHash(defaultConfig), + boundary: 'next_session', requester: 'fixture', idempotencyKey: 'reset', allowUnvalidated: true }, now) +}) +afterEach(() => { + configSqlite.close() + bioSqlite.close() +}) +const payload = () => configDb.select().from(configSchema.biometricConfigOutbox).get()?.payload + +describe('runtime projection and acknowledgment', () => { + it('retries safely after projection succeeds but the configuration acknowledgment fails', () => { + configSqlite.exec('CREATE TRIGGER reject_delivery BEFORE UPDATE OF state ON biometric_config_outbox WHEN NEW.state = \'delivered\' BEGIN SELECT RAISE(ABORT, \'fixture crash\'); END;') + expect(deliverRuntimeOutbox(configDb, bioDb, now)).toEqual({ delivered: 0, failed: 1 }) + expect(bioDb.select().from(bioSchema.biometricRuntimeConfigs).all()).toHaveLength(1) + configSqlite.exec('DROP TRIGGER reject_delivery') + expect(deliverRuntimeOutbox(configDb, bioDb, now + 1000)).toEqual({ delivered: 1, failed: 0 }) + expect(bioDb.select().from(bioSchema.biometricRuntimeConfigs).all()).toHaveLength(1) + expect(reconcileActivation(configDb, bioDb, 'left', now + 1000)?.state).toBe('pending_boundary') + expect(bioDb.select().from(bioSchema.biometricRuntimeReceipts).all()).toEqual([]) + }) + it('requires every matching component receipt and ignores old generations and wrong hashes', () => { + deliverRuntimeOutbox(configDb, bioDb, now) + for (const component of RUNTIME_COMPONENTS) bioDb.insert(bioSchema.biometricRuntimeReceipts).values({ side: 'left', component, generation: 0, + appliedHash: configHash(defaultConfig), state: 'applied', effectiveSampleTimeMs: now, heartbeatAtMs: now }).run() + expect(reconcileActivation(configDb, bioDb, 'left', now)?.state).toBe('pending_boundary') + for (const component of RUNTIME_COMPONENTS) bioDb.insert(bioSchema.biometricRuntimeReceipts).values({ side: 'left', component, generation: 1, + appliedHash: component === 'sleep_stages' ? 'wrong' : configHash(defaultConfig), state: 'applied', effectiveSampleTimeMs: now, heartbeatAtMs: now }).run() + expect(reconcileActivation(configDb, bioDb, 'left', now)?.state).toBe('pending_boundary') + bioSqlite.prepare('UPDATE biometric_runtime_receipts SET applied_hash = ? WHERE generation = 1').run(configHash(defaultConfig)) + expect(reconcileActivation(configDb, bioDb, 'left', now)?.state).toBe('applied') + }) + it('restores the current projection after a history reset and rejects conflicting duplicate generations', () => { + deliverRuntimeOutbox(configDb, bioDb, now) + bioDb.delete(bioSchema.biometricRuntimeConfigs).run() + expect(repairCurrentRuntimeProjections(configDb, bioDb, now)).toBe(1) + expect(repairCurrentRuntimeProjections(configDb, bioDb, now)).toBe(0) + expect(() => projectRuntimeConfig(bioDb, { ...(payload() as object), revisionId: '11111111-1111-4111-8111-111111111111' }, now)).toThrow('different immutable payload') + expect(configDb.select().from(configSchema.biometricProfileBindings).get()?.generation).toBe(1) + }) + it('marks invalid envelopes failed without projecting or fabricating receipts', () => { + configSqlite.prepare('UPDATE biometric_config_outbox SET resolved_hash = ?').run('wrong') + expect(deliverRuntimeOutbox(configDb, bioDb, now)).toEqual({ delivered: 0, failed: 1 }) + expect(reconcileActivation(configDb, bioDb, 'left', now)?.state).toBe('failed') + expect(bioDb.select().from(bioSchema.biometricRuntimeConfigs).all()).toEqual([]) + }) +}) diff --git a/src/biometrics/profiles/tests/stage-view.test.ts b/src/biometrics/profiles/tests/stage-view.test.ts new file mode 100644 index 000000000..e97066716 --- /dev/null +++ b/src/biometrics/profiles/tests/stage-view.test.ts @@ -0,0 +1,149 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import * as configSchema from '@/src/db/schema' +import * as bioSchema from '@/src/db/biometrics-schema' +import { DEFAULT_STAGE_CLASSIFIER, MOVEMENT_ONLY_STAGE_RULESET } from '@/src/lib/sleep-stage-rules' +import { defaultConfig } from '../../algorithms/registry' +import { resolveReplayBaseline } from '../replay-baseline' +import { ProfileStore } from '../store' +import { writeLegacyStageClassifier } from '../legacy-stage' +import { consumeStageView, resolveStageView } from '../stage-view' +import { deliverRuntimeOutbox, reconcileActivation } from '../runtime' +import { requestManualActivation } from '../activation' +let configRaw: Database.Database, bioRaw: Database.Database +let config: ReturnType>, bio: ReturnType> +const movementOnly = { engine: 'rules' as const, ruleset: MOVEMENT_ONLY_STAGE_RULESET } +const now = 1791673200000 +beforeEach(() => { + configRaw = new Database(':memory:') + bioRaw = new Database(':memory:') + config = drizzle(configRaw, { schema: configSchema }) + bio = drizzle(bioRaw, { schema: bioSchema }) + migrate(config, { migrationsFolder: 'src/db/migrations' }) + migrate(bio, { migrationsFolder: 'src/db/biometrics-migrations' }) + config.insert(configSchema.sideSettings).values([{ side: 'left', name: 'Left' }, { side: 'right', name: 'Right' }]).run() +}) +afterEach(() => { + configRaw.close() + bioRaw.close() +}) +function acknowledge(generation: number, components = ['heart_rate', 'breathing_rate', 'hrv_index']) { + deliverRuntimeOutbox(config, bio, now + 1) + const row = bioRaw.prepare('SELECT resolved_hash FROM biometric_runtime_configs WHERE side = ? AND generation = ?').get('left', generation) as { resolved_hash: string } + for (const component of components) bio.insert(bioSchema.biometricRuntimeReceipts).values({ side: 'left', component, generation, + appliedHash: row.resolved_hash, effectiveSampleTimeMs: now, state: 'applied', heartbeatAtMs: now }).run() +} +describe('stage view adoption and legacy compatibility', () => { + it('waits for all numerical consumers, then acknowledges exactly the view delivered for classification', () => { + const saved = writeLegacyStageClassifier(config, 'left', movementOnly, now) + expect(saved.activation.generation).toBe(1) + acknowledge(1, ['heart_rate', 'breathing_rate']) + expect(consumeStageView(config, bio, 'left', now).classifier).toEqual(DEFAULT_STAGE_CLASSIFIER) + expect(reconcileActivation(config, bio, 'left', now)?.state).toBe('pending_boundary') + acknowledge(1, ['hrv_index']) + const view = consumeStageView(config, bio, 'left', now) + expect(view.classifier).toEqual(movementOnly) + expect(view.provenance).toMatchObject({ source: 'runtime', generation: 1, interpretation: 'current_classifier_view' }) + expect(reconcileActivation(config, bio, 'left', now)?.state).toBe('applied') + expect(resolveStageView(config, bio, 'right').classifier).toEqual(DEFAULT_STAGE_CLASSIFIER) + }) + it('does not rescore from pending intent and preserves the last fully applied view', () => { + writeLegacyStageClassifier(config, 'left', movementOnly, now) + acknowledge(1) + consumeStageView(config, bio, 'left', now) + writeLegacyStageClassifier(config, 'left', null, now) + acknowledge(2, ['heart_rate']) + expect(resolveStageView(config, bio, 'left').classifier).toEqual(movementOnly) + expect(resolveStageView(config, bio, 'left').provenance.generation).toBe(1) + acknowledge(2, ['breathing_rate', 'hrv_index']) + expect(consumeStageView(config, bio, 'left', now).classifier).toEqual(DEFAULT_STAGE_CLASSIFIER) + }) + it('retains other metric overrides when legacy clients reset only stage rules', () => { + const store = new ProfileStore(config) + const revision = store.createRevision({ side: 'left', overrides: [{ path: 'heart_rate.trackingToleranceBpm', value: 20 }], origin: 'manual', requester: 'fixture', idempotencyKey: 'hr' }) + requestManualActivation(config, { side: 'left', action: 'activate', revisionId: revision.id, expectedGeneration: 0, + expectedResolvedHash: revision.resolvedHash, boundary: 'immediate', requester: 'fixture', idempotencyKey: 'hr-active', allowUnvalidated: true }, now) + writeLegacyStageClassifier(config, 'left', movementOnly, now) + writeLegacyStageClassifier(config, 'left', null, now) + expect(store.effective('left').resolved.heart_rate.trackingToleranceBpm).toBe(20) + expect(store.effective('left').resolved.sleep_stages).toEqual(DEFAULT_STAGE_CLASSIFIER) + expect(store.effective('left').binding.desiredRevisionId).not.toBeNull() + expect(configRaw.prepare('SELECT stage_classifier FROM side_settings WHERE side = ?').get('left')).toEqual({ stage_classifier: null }) + }) + it('migrates existing customization before compatibility writes without losing its previous revision', () => { + configRaw.prepare('UPDATE side_settings SET stage_classifier = ? WHERE side = ?').run(JSON.stringify(movementOnly), 'left') + const changed = writeLegacyStageClassifier(config, 'left', null, now) + expect(changed.activation.generation).toBe(2) + const store = new ProfileStore(config) + const previousId = store.binding('left').previousRevisionId + if (!previousId) throw new Error('Missing previous revision') + const previous = store.getRevision(previousId) + expect((previous.resolved as { sleep_stages: unknown }).sleep_stages).toEqual(movementOnly) + expect(resolveStageView(config, bio, 'left').classifier).toEqual(movementOnly) + }) + it('replaces malformed stored legacy rules with a new save after recording the reset recovery', () => { + const malformed = { engine: 'unknown-engine', ruleset: { cannotMigrate: true } } + configRaw.prepare('UPDATE side_settings SET stage_classifier = ? WHERE side = ?').run(JSON.stringify(malformed), 'left') + const saved = writeLegacyStageClassifier(config, 'left', movementOnly, now) + expect(saved.activation).toMatchObject({ generation: 2, state: 'pending_delivery' }) + const store = new ProfileStore(config) + expect(store.effective('left').resolved.sleep_stages).toEqual(movementOnly) + const events = config.select().from(configSchema.biometricActivationEvents).all() + expect(events.map(event => [event.generation, event.action])).toEqual([[1, 'reset'], [2, 'activate']]) + expect(events[0].evidence).toMatchObject({ recovery: 'invalid_legacy_reset', preservedLegacyClassifier: malformed }) + expect(configRaw.prepare('SELECT stage_classifier FROM side_settings WHERE side = ?').get('left')).toEqual({ stage_classifier: JSON.stringify(malformed) }) + acknowledge(2) + expect(consumeStageView(config, bio, 'left', now).classifier).toEqual(movementOnly) + }) + it('surfaces incompatible applied projections without acknowledging them as active', () => { + writeLegacyStageClassifier(config, 'left', movementOnly, now) + acknowledge(1) + bioRaw.prepare('UPDATE biometric_runtime_configs SET payload = ?').run('{}') + const view = consumeStageView(config, bio, 'left', now) + expect(view.classifier).toEqual(DEFAULT_STAGE_CLASSIFIER) + expect(view.provenance.warning).toContain('incompatible') + expect(bioRaw.prepare('SELECT * FROM biometric_runtime_receipts WHERE component=\'sleep_stages\'').all()).toEqual([]) + }) + it('rolls back the compatibility revision and binding when intent publication fails', () => { + configRaw.exec('CREATE TRIGGER reject_outbox BEFORE INSERT ON biometric_config_outbox BEGIN SELECT RAISE(ABORT, \'full\'); END;') + expect(() => writeLegacyStageClassifier(config, 'left', movementOnly, now)).toThrow('full') + expect(config.select().from(configSchema.biometricProfileRevisions).all()).toEqual([]) + expect(config.select().from(configSchema.biometricProfileBindings).all()).toEqual([]) + }) +}) + +it('captures a complete numerical baseline only after coherent adoption', () => { + expect(resolveReplayBaseline(config, bio, 'left')).toMatchObject({ resolved: defaultConfig, provenance: { source: 'unacknowledged_defaults' } }) + const store = new ProfileStore(config) + const revision = store.createRevision({ side: 'left', overrides: [{ path: 'heart_rate.trackingToleranceBpm', value: 20 }], origin: 'manual', requester: 'fixture', idempotencyKey: 'numerical-current' }) + requestManualActivation(config, { side: 'left', action: 'activate', revisionId: revision.id, expectedGeneration: 0, + expectedResolvedHash: revision.resolvedHash, boundary: 'immediate', requester: 'fixture', idempotencyKey: 'numerical-active', allowUnvalidated: true }, now) + expect(resolveReplayBaseline(config, bio, 'left').resolved.heart_rate.trackingToleranceBpm).toBe(defaultConfig.heart_rate.trackingToleranceBpm) + acknowledge(1, ['heart_rate']) + expect(() => resolveReplayBaseline(config, bio, 'left')).toThrow('consistent configuration') + acknowledge(1, ['breathing_rate', 'hrv_index']) + const captured = resolveReplayBaseline(config, bio, 'left') + expect(captured).toMatchObject({ hash: revision.resolvedHash, revisionId: revision.id, resolved: { heart_rate: { trackingToleranceBpm: 20 } }, + provenance: { source: 'acknowledged_runtime', generation: 1, effectiveSampleTimeMs: now } }) + expect(resolveReplayBaseline(config, bio, 'right').resolved).toEqual(defaultConfig) + expect(bioRaw.prepare('SELECT * FROM biometric_runtime_receipts WHERE component=\'sleep_stages\'').all()).toEqual([]) + writeLegacyStageClassifier(config, 'left', movementOnly, now) + acknowledge(2, ['heart_rate']) + expect(() => resolveReplayBaseline(config, bio, 'left')).toThrow('consistent configuration') + bioRaw.prepare('UPDATE biometric_runtime_receipts SET state=\'pending_boundary\' WHERE generation=2').run() + expect(resolveReplayBaseline(config, bio, 'left')).toEqual(captured) + bioRaw.prepare('UPDATE biometric_runtime_receipts SET state=\'failed\' WHERE generation=2').run() + bio.insert(bioSchema.biometricRuntimeReceipts).values({ side: 'left', component: 'heart_rate', generation: 3, appliedHash: revision.resolvedHash, + state: 'pending_boundary', effectiveSampleTimeMs: null, heartbeatAtMs: now }).run() + expect(() => resolveReplayBaseline(config, bio, 'left')).toThrow('adoption failed') +}) + +it('refuses corrupt acknowledged projections instead of relabeling defaults as current', () => { + writeLegacyStageClassifier(config, 'left', movementOnly, now) + acknowledge(1) + bioRaw.prepare('UPDATE biometric_runtime_configs SET payload = ?').run('{}') + expect(() => resolveReplayBaseline(config, bio, 'left')).toThrow('unavailable or incompatible') +}) diff --git a/src/biometrics/profiles/tests/store.test.ts b/src/biometrics/profiles/tests/store.test.ts new file mode 100644 index 000000000..745c3295a --- /dev/null +++ b/src/biometrics/profiles/tests/store.test.ts @@ -0,0 +1,101 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { eq } from 'drizzle-orm' +import * as schema from '@/src/db/schema' +import { DEFAULT_STAGE_CLASSIFIER } from '@/src/lib/sleep-stage-rules' +import { defaultConfig } from '../../algorithms/registry' +import { ProfileStore } from '../store' + +let sqlite: Database.Database +let db: ReturnType> +let store: ProfileStore +const request = { side: 'left' as const, overrides: [{ path: 'heart_rate.trackingToleranceBpm', value: 20 }], + origin: 'manual' as const, requester: 'test', idempotencyKey: 'candidate-1' } + +beforeEach(() => { + sqlite = new Database(':memory:') + db = drizzle(sqlite, { schema }) + migrate(db, { migrationsFolder: 'src/db/migrations' }) + store = new ProfileStore(db, () => 1700000000000) +}) +afterEach(() => sqlite.close()) + +describe('immutable profile repository', () => { + it('creates an idempotent candidate without activating it', () => { + const first = store.createRevision(request) + expect(store.createRevision(request)).toEqual(first) + expect(store.createRevision({ ...request, parentRevisionId: undefined })).toEqual(first) + expect(first.revision).toBe(1) + expect(store.effective('left')).toMatchObject({ isDefault: true, binding: { generation: 0 } }) + expect(store.validateRevision(first.id, 'left').resolved.heart_rate.trackingToleranceBpm).toBe(20) + expect(() => store.createRevision({ ...request, overrides: [] })).toThrow('IDEMPOTENCY_CONFLICT') + expect(db.select().from(schema.biometricConfigOutbox).all()).toEqual([]) + }) + + it('preserves the parent snapshot and serializes family revision numbers', () => { + const first = store.createRevision(request) + const second = store.createRevision({ ...request, parentRevisionId: first.id, overrides: [], idempotencyKey: 'candidate-2' }) + const third = store.createRevision({ ...request, parentRevisionId: first.id, overrides: [], idempotencyKey: 'candidate-3' }) + expect([second.revision, third.revision]).toEqual([2, 3]) + expect(second.familyId).toBe(first.familyId) + expect(store.getRevision(first.id)).toEqual(first) + expect(store.validateRevision(second.id, 'left').resolved).toEqual(defaultConfig) + expect(store.listRevisions('left', undefined, first.familyId).items.map(r => r.id).sort()).toEqual([first.id, second.id, third.id].sort()) + }) + + it('rejects cross-side inheritance, unknown parents and tampered snapshots', () => { + const first = store.createRevision(request) + expect(() => store.createRevision({ ...request, side: 'right', parentRevisionId: first.id, idempotencyKey: 'right' })).toThrow('other side') + expect(() => store.validateRevision(first.id, 'right')).toThrow('other side') + expect(() => store.createRevision({ ...request, parentRevisionId: '00000000-0000-4000-8000-000000000000', idempotencyKey: 'missing' })).toThrow('NOT_FOUND') + db.update(schema.biometricProfileRevisions).set({ resolvedHash: 'bad' }).where(eq(schema.biometricProfileRevisions.id, first.id)).run() + expect(() => store.validateRevision(first.id, 'left')).toThrow('integrity') + }) + + it('reports a visible default fallback for an unavailable pinned engine', () => { + const first = store.createRevision(request) + db.insert(schema.biometricProfileBindings).values({ side: 'left', desiredRevisionId: first.id, generation: 2, updatedAtMs: 1 }).run() + db.update(schema.biometricProfileRevisions).set({ baseManifestHash: 'unavailable' }).run() + expect(store.effective('left')).toMatchObject({ isDefault: true, warning: expect.stringContaining('PROFILE_INCOMPATIBLE'), resolved: defaultConfig }) + expect(store.binding('left').desiredRevisionId).toBe(first.id) + }) +}) + +describe('legacy stage migration', () => { + it('preserves exact rules and commits binding, evidence and outbox atomically', () => { + const classifier = structuredClone(DEFAULT_STAGE_CLASSIFIER) + classifier.ruleset.rules.reverse() + classifier.ruleset.smoothIsolatedEpochs = false + db.insert(schema.sideSettings).values({ side: 'left', name: 'Left', stageClassifier: classifier }).run() + const binding = store.migrateLegacy('left') + if (!binding?.desiredRevisionId) throw new Error('Expected migrated binding') + expect(store.migrateLegacy('left')).toEqual(binding) + expect(store.validateRevision(binding.desiredRevisionId, 'left').resolved.sleep_stages).toEqual(classifier) + expect(db.select().from(schema.biometricProfileRevisions).all()).toHaveLength(1) + expect(db.select().from(schema.biometricActivationEvents).get()).toMatchObject({ generation: 1, evidence: { status: 'legacy_unvalidated' }, state: 'pending_delivery' }) + expect(db.select().from(schema.biometricConfigOutbox).get()).toMatchObject({ generation: 1, state: 'pending' }) + expect(db.select().from(schema.sideSettings).get()?.stageClassifier).toEqual(classifier) + expect(store.effective('right').isDefault).toBe(true) + }) + + it('keeps null on system defaults and refuses to discard invalid legacy settings', () => { + db.insert(schema.sideSettings).values({ side: 'right', name: 'Right' }).run() + expect(store.migrateLegacy('right')).toBeNull() + db.insert(schema.sideSettings).values({ side: 'left', name: 'Left', stageClassifier: { engine: 'unknown' } }).run() + expect(() => store.migrateLegacy('left')).toThrow() + expect(db.select().from(schema.biometricProfileBindings).all()).toEqual([]) + expect(db.select().from(schema.biometricProfileRevisions).all()).toEqual([]) + }) + + it('rolls back the entire migration if outbox publication fails', () => { + db.insert(schema.sideSettings).values({ side: 'left', name: 'Left', stageClassifier: DEFAULT_STAGE_CLASSIFIER }).run() + sqlite.exec('CREATE TRIGGER fail_outbox BEFORE INSERT ON biometric_config_outbox BEGIN SELECT RAISE(ABORT, \'disk full\'); END') + expect(() => store.migrateLegacy('left')).toThrow() + expect(db.select().from(schema.biometricProfileBindings).all()).toEqual([]) + expect(db.select().from(schema.biometricActivationEvents).all()).toEqual([]) + expect(db.select().from(schema.biometricProfileRevisions).all()).toEqual([]) + }) +}) diff --git a/src/biometrics/profiles/tests/validated-activation.test.ts b/src/biometrics/profiles/tests/validated-activation.test.ts new file mode 100644 index 000000000..8609e9579 --- /dev/null +++ b/src/biometrics/profiles/tests/validated-activation.test.ts @@ -0,0 +1,246 @@ +// @vitest-environment node +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import * as configSchema from '@/src/db/schema' +import * as bioSchema from '@/src/db/biometrics-schema' +import { ProfileStore } from '../store' +import { requestManualActivation, requestValidatedActivation, requestValidatedActivationWithBuild, validatedActivationReviewSchema, reviewValidatedActivation } from '../activation' +import { deliverRuntimeOutbox } from '../runtime' +import { prepareVerifiedEvaluation } from '../evaluation-store' +import { BacktestJobStore } from '../../backtest/jobs' +import { DatasetStore } from '../../backtest/datasets' +import { backtestJobSnapshotSchema, datasetManifestSchema } from '../../backtest/contracts' +import { submitBacktest } from '../../backtest/submission' +import { evaluationFixture } from './fixtures/evaluation' +import { defaultConfig } from '../../algorithms/registry' +import { configHash } from '../../algorithms/canonical' + +let configRaw: Database.Database, bioRaw: Database.Database, observer: Database.Database +let config: ReturnType>, bio: ReturnType> +let root: string +const now = 1791673200000 +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), 'biometric-activation-')) + configRaw = new Database(':memory:') + bioRaw = new Database(join(root, 'bio.sqlite')) + bioRaw.pragma('journal_mode = WAL') + observer = new Database(join(root, 'bio.sqlite'), { timeout: 0 }) + config = drizzle(configRaw, { schema: configSchema }) + bio = drizzle(bioRaw, { schema: bioSchema }) + migrate(config, { migrationsFolder: 'src/db/migrations' }) + migrate(bio, { migrationsFolder: 'src/db/biometrics-migrations' }) +}) +afterEach(() => { + vi.restoreAllMocks() + observer.close() + configRaw.close() + bioRaw.close() + rmSync(root, { recursive: true, force: true }) +}) + +function fixture() { + const work = evaluationFixture() + const profiles = new ProfileStore(config) + const revision = profiles.createRevision({ side: 'left', origin: 'manual', requester: 'fixture', idempotencyKey: 'candidate', + overrides: [{ path: 'sleep_stages.ruleset.smoothIsolatedEpochs', value: false }] }) + for (let id = 1; id <= 12; id++) bio.insert(bioSchema.sleepRecords).values({ id, side: 'left', + enteredBedAt: new Date(now + id * 86400000), leftBedAt: new Date(now + id * 86400000 + 3600000), sleepDurationSeconds: 3600, timesExitedBed: 0 }).run() + const datasets = new DatasetStore(bio) + const dataset = datasets.create({ side: 'left', subjectId: 'fixture', sessionIds: Array.from({ length: 12 }, (_, i) => i + 1), + referenceImportIds: [], timezone: 'UTC', idempotencyKey: 'dataset' }) + const cohorts = datasetManifestSchema.parse(dataset.manifest).splitProtocol.assignments.filter(row => row.split === 'holdout').map(row => row.cohortId) + expect(cohorts).toHaveLength(3) + const jobs = new BacktestJobStore(bio) + const experiment = jobs.createExperiment(dataset.id, work.job.protocol) + const configs = work.job.configs.map(row => row.label === 'candidate' ? { ...row, revisionId: revision.id } : row) + const enqueue = (idempotencyKey: string) => jobs.enqueue({ experimentId: experiment.id, evaluatorBuildHash: work.job.evaluatorBuildHash, + idempotencyKey, split: 'holdout', configs }) + const job = enqueue('holdout') + const report = { ...work.report, datasetId: dataset.id, datasetHash: dataset.contentHash, nights: cohorts.map(cohortId => ({ cohortId })) } + for (const row of report.stageObjectiveAssessments) { + row.evidence.nights.forEach((night, i) => { + night.cohortId = cohorts[i] + }) + } + const lease = jobs.claim('fixture') + if (!lease) throw new Error('No lease') + const result = jobs.complete({ id: lease.id, owner: 'fixture', token: lease.fencingToken }, report) + const input = { side: 'left' as const, action: 'activate' as const, revisionId: revision.id, expectedGeneration: 0, + expectedResolvedHash: revision.resolvedHash, boundary: 'next_session' as const, requester: 'fixture', idempotencyKey: 'validated', + evaluationId: job.id, expectedReportHash: result.contentHash } + const build = { evaluatorBuildHash: work.job.evaluatorBuildHash, designations: work.designations } + return { profiles, jobs, datasets, dataset, input, build, enqueue } +} + +it('atomically stores verified evidence and outbox intent; retries survive released biometric artifacts', () => { + const work = fixture() + const event = requestValidatedActivation(config, bio, work.input, work.build, now) + expect(event).toMatchObject({ state: 'pending_delivery', generation: 1, evaluationId: work.input.evaluationId, + evidence: { status: 'validated_reference_agreement', reportHash: work.input.expectedReportHash, changedDomains: ['sleep_stages'] } }) + expect(work.profiles.binding('left').desiredRevisionId).toBe(work.input.revisionId) + expect(work.profiles.binding('right').generation).toBe(0) + expect(config.select().from(configSchema.biometricConfigOutbox).all()).toHaveLength(1) + work.jobs.releaseResult(work.input.evaluationId, work.input.expectedReportHash) + work.datasets.release(work.dataset.id) + expect(requestValidatedActivation(config, bio, work.input, work.build, now + 1)).toEqual(event) + expect(config.select().from(configSchema.biometricActivationEvents).all()).toHaveLength(1) + expect(bio.select().from(bioSchema.vitals).all()).toEqual([]) +}) + +it('rejects missing designation, mismatched hashes, and reused holdout before changing bindings', () => { + const work = fixture() + expect(() => requestValidatedActivation(config, bio, work.input, { ...work.build, designations: { protocolHash: null, waveformContext: null, numeric: null, stages: null } })).toThrow() + expect(() => requestValidatedActivation(config, bio, { ...work.input, expectedReportHash: '0'.repeat(64) }, work.build)).toThrow('hash') + work.enqueue('second-holdout') + expect(() => requestValidatedActivation(config, bio, work.input, work.build)).toThrow('holdout') + expect(work.profiles.binding('left').generation).toBe(0) + expect(config.select().from(configSchema.biometricConfigOutbox).all()).toEqual([]) +}) + +it('blocks concurrent biometrics writes across the configuration commit', () => { + const work = fixture() + let blocked = false + configRaw.function('check_bio_reservation', () => { + try { + observer.prepare('UPDATE biometric_backtest_jobs SET updated_at_ms = updated_at_ms').run() + } + catch (error) { blocked = (error as { code: string }).code === 'SQLITE_BUSY' } + return 1 + }) + configRaw.exec('CREATE TRIGGER check_lock BEFORE INSERT ON biometric_profile_bindings BEGIN SELECT check_bio_reservation(); END;') + requestValidatedActivation(config, bio, work.input, work.build) + expect(blocked).toBe(true) + expect(() => observer.prepare('UPDATE biometric_backtest_jobs SET updated_at_ms = updated_at_ms').run()).not.toThrow() +}) + +it('rejects holdout reuse that occurs after the first verified result read', () => { + const work = fixture() + const original = BacktestJobStore.prototype.result + vi.spyOn(BacktestJobStore.prototype, 'result').mockImplementationOnce(function (this: BacktestJobStore, id) { + const result = original.call(this, id) + work.enqueue('racing-holdout') + return result + }) + expect(() => requestValidatedActivation(config, bio, work.input, work.build)).toThrow('reused before activation') + expect(work.profiles.binding('left').generation).toBe(0) +}) + +it('rejects report release between verification and committing intent', () => { + const work = fixture() + const original = BacktestJobStore.prototype.result + const spy = vi.spyOn(BacktestJobStore.prototype, 'result').mockImplementationOnce(function (this: BacktestJobStore, id) { + const result = original.call(this, id) + spy.mockRestore() + work.jobs.releaseResult(work.input.evaluationId, work.input.expectedReportHash) + return result + }) + expect(() => requestValidatedActivation(config, bio, work.input, work.build)).toThrow('released, replaced or reused') + expect(work.profiles.binding('left').generation).toBe(0) +}) + +it.each([ + 'UPDATE biometric_backtest_jobs SET state = \'failed\'', + 'UPDATE biometric_backtest_jobs SET fencing_token = fencing_token + 1', + 'DELETE FROM biometric_backtest_results', + 'UPDATE biometric_backtest_results SET fencing_token = fencing_token + 1', + 'UPDATE biometric_backtest_results SET content_hash = \'changed\'', + 'UPDATE biometric_backtest_results SET released_at_ms = 1', + 'UPDATE biometric_datasets SET content_hash = \'changed\'', + 'UPDATE biometric_datasets SET released_at_ms = 1', +])('rejects changed retained metadata at the commit guard: %s', (statement) => { + const work = fixture() + const prepared = prepareVerifiedEvaluation(config, bio, work.input, work.build) + bioRaw.exec(statement) + expect(() => prepared.recheck()).toThrow('released, replaced or reused') + expect(work.profiles.binding('left').generation).toBe(0) +}) + +it('rejects pending baselines and captures the complete acknowledged bundle for subsequent stage replay', () => { + const work = fixture() + const revision = work.profiles.createRevision({ side: 'left', origin: 'manual', requester: 'fixture', idempotencyKey: 'numeric', + overrides: [{ path: 'heart_rate.trackingToleranceBpm', value: 20 }] }) + requestManualActivation(config, { side: 'left', action: 'activate', revisionId: revision.id, expectedGeneration: 0, + expectedResolvedHash: revision.resolvedHash, boundary: 'immediate', requester: 'fixture', idempotencyKey: 'numeric-activation', allowUnvalidated: true }, now) + expect(() => requestValidatedActivation(config, bio, work.input, work.build)).toThrow('Binding changed') + expect(() => requestValidatedActivation(config, bio, { ...work.input, expectedGeneration: 1 }, work.build)).toThrow('pending') + deliverRuntimeOutbox(config, bio, now) + for (const component of ['heart_rate', 'breathing_rate', 'hrv_index']) bio.insert(bioSchema.biometricRuntimeReceipts) + .values({ side: 'left', component, generation: 1, appliedHash: revision.resolvedHash, effectiveSampleTimeMs: now, state: 'applied', heartbeatAtMs: now }).run() + expect(() => requestValidatedActivation(config, bio, { ...work.input, expectedGeneration: 1 }, work.build)).toThrow('baseline changed') + const experimentId = work.jobs.get(work.input.evaluationId).experimentId + const next = submitBacktest(config, bio, work.build.evaluatorBuildHash, { experimentId, candidateRevisionId: work.input.revisionId, + expectedBindingGeneration: 1, split: 'tuning', idempotencyKey: 'next-stage-replay' }) + expect(backtestJobSnapshotSchema.parse(next.input).configs.find(row => row.label === 'current')).toMatchObject({ + revisionId: revision.id, resolvedHash: revision.resolvedHash, resolved: { heart_rate: { trackingToleranceBpm: 20 } } }) +}) + +it('rolls back binding, evidence and outbox together and keeps manual/validated idempotency distinct', () => { + const work = fixture() + configRaw.exec('CREATE TRIGGER reject_outbox BEFORE INSERT ON biometric_config_outbox BEGIN SELECT RAISE(ABORT, \'fixture failure\'); END;') + expect(() => requestValidatedActivation(config, bio, work.input, work.build)).toThrow('fixture failure') + expect(work.profiles.binding('left').generation).toBe(0) + expect(config.select().from(configSchema.biometricActivationEvents).all()).toEqual([]) + configRaw.exec('DROP TRIGGER reject_outbox') + requestValidatedActivation(config, bio, work.input, work.build) + expect(() => requestValidatedActivation(config, bio, { ...work.input, boundary: 'immediate' }, work.build)).toThrow('idempotency') + expect(() => requestManualActivation(config, { side: 'left', action: 'reset', revisionId: null, expectedGeneration: 1, + expectedResolvedHash: configHash(defaultConfig), boundary: 'immediate', requester: 'fixture', idempotencyKey: work.input.idempotencyKey, allowUnvalidated: true })).toThrow('idempotency') +}) + +it('resolves build authority lazily and retries after artifact release without probing', async () => { + const work = fixture() + const resolve = vi.fn(async () => work.build) + const event = await requestValidatedActivationWithBuild(config, bio, work.input, resolve, now) + expect(resolve).toHaveBeenCalledTimes(1) + work.jobs.releaseResult(work.input.evaluationId, work.input.expectedReportHash) + work.datasets.release(work.dataset.id) + resolve.mockRejectedValue(new Error('Build no longer installed')) + expect(await requestValidatedActivationWithBuild(config, bio, work.input, resolve, now + 1)).toEqual(event) + expect(resolve).toHaveBeenCalledTimes(1) + await expect(requestValidatedActivationWithBuild(config, bio, { ...work.input, expectedReportHash: '0'.repeat(64) }, resolve)) + .rejects.toThrow('idempotency key changed') + expect(resolve).toHaveBeenCalledTimes(1) +}) + +it('coalesces identical activation intents that race across asynchronous build resolution', async () => { + const work = fixture() + let finish!: (build: typeof work.build) => void + const pending = new Promise((resolve) => { + finish = resolve + }) + const first = requestValidatedActivationWithBuild(config, bio, work.input, () => pending, now) + const second = await requestValidatedActivationWithBuild(config, bio, work.input, async () => work.build, now) + work.jobs.releaseResult(work.input.evaluationId, work.input.expectedReportHash) + finish(work.build) + expect(await first).toEqual(second) + expect(config.select().from(configSchema.biometricActivationEvents).all()).toHaveLength(1) + expect(config.select().from(configSchema.biometricConfigOutbox).all()).toHaveLength(1) +}) + +it('leaves activation state unchanged when build resolution fails', async () => { + const work = fixture() + await expect(requestValidatedActivationWithBuild(config, bio, work.input, async () => { + throw new Error('Worker missing') + })) + .rejects.toThrow('Worker missing') + expect(work.profiles.binding('left').generation).toBe(0) + expect(config.select().from(configSchema.biometricActivationEvents).all()).toEqual([]) +}) + +it('reviews qualified evidence without writing intent and still revalidates after review', async () => { + const work = fixture() + const reviewInput = validatedActivationReviewSchema.strip().parse(work.input) + expect(await reviewValidatedActivation(config, bio, reviewInput, async () => work.build)).toEqual({ eligible: true }) + expect(work.profiles.binding('left').generation).toBe(0) + expect(config.select().from(configSchema.biometricActivationEvents).all()).toEqual([]) + expect(config.select().from(configSchema.biometricConfigOutbox).all()).toEqual([]) + work.jobs.releaseResult(work.input.evaluationId, work.input.expectedReportHash) + await expect(requestValidatedActivationWithBuild(config, bio, work.input, async () => work.build)).rejects.toThrow() + await expect(reviewValidatedActivation(config, bio, reviewInput, async () => work.build)).rejects.toThrow() + expect(work.profiles.binding('left').generation).toBe(0) +}) diff --git a/src/biometrics/profiles/tests/vital-provenance.test.ts b/src/biometrics/profiles/tests/vital-provenance.test.ts new file mode 100644 index 000000000..96ce491b0 --- /dev/null +++ b/src/biometrics/profiles/tests/vital-provenance.test.ts @@ -0,0 +1,23 @@ +// @vitest-environment node +import { expect, it } from 'vitest' +import { vitalProvenance, vitalProvenanceSchema } from '../vital-provenance' + +const hash = 'a'.repeat(64) +const recorded = { side: 'left', processingRunId: 'run', recordedRunId: 'run', recordedSide: 'left', recordedKind: 'live', recordedConfigHash: hash, recordedHrvMetricId: 'window_ibi_variability_v1' } + +it('describes rows without a run as legacy and verified live runs as recorded', () => { + expect(vitalProvenance({ side: 'left' })).toEqual({ interpretation: 'original_observation', status: 'legacy_unknown', processingRunId: null, configurationHash: null, hrvMetricId: null }) + expect(vitalProvenance({ side: 'left', processingRunId: null })).toMatchObject({ status: 'legacy_unknown' }) + const full = vitalProvenance(recorded) + expect(full).toEqual({ interpretation: 'original_observation', status: 'recorded', processingRunId: 'run', configurationHash: hash, hrvMetricId: 'window_ibi_variability_v1' }) + expect(vitalProvenanceSchema.parse(full)).toEqual(full) +}) + +it('does not trust runs that are missing, replayed, cross-side or malformed', () => { + for (const patch of [{ recordedRunId: null }, { recordedRunId: 'other' }, { recordedSide: 'right' }, { recordedKind: 'replay' }, { recordedConfigHash: 'short' }, { recordedConfigHash: null }]) { + expect(vitalProvenance({ ...recorded, ...patch })).toEqual({ interpretation: 'original_observation', status: 'unavailable', processingRunId: 'run', configurationHash: null, hrvMetricId: null }) + } + expect(vitalProvenance({ ...recorded, recordedHrvMetricId: 'Not Valid' })).toMatchObject({ status: 'recorded', hrvMetricId: null }) + expect(vitalProvenance({ ...recorded, recordedHrvMetricId: 7 })).toMatchObject({ status: 'recorded', hrvMetricId: null }) + expect(vitalProvenance({ ...recorded, recordedHrvMetricId: undefined })).toMatchObject({ status: 'recorded', hrvMetricId: null }) +}) diff --git a/src/biometrics/profiles/tests/vital-summary-provenance.test.ts b/src/biometrics/profiles/tests/vital-summary-provenance.test.ts new file mode 100644 index 000000000..6787c06f7 --- /dev/null +++ b/src/biometrics/profiles/tests/vital-summary-provenance.test.ts @@ -0,0 +1,68 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { eq } from 'drizzle-orm' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import * as schema from '@/src/db/biometrics-schema' +import { summarizeVitalProvenance, summaryProvenanceFields } from '../vital-summary-provenance' + +const counts = { configurationCount: 1, unknownConfigurationRows: 0, hrvRows: 4, hrvDefinitionCount: 1, unknownHrvRows: 0, singleHrvMetricId: 'window_ibi_variability_v1' } + +describe('summarizeVitalProvenance', () => { + it('returns nothing for adapters that do not supply the aggregate fields', () => { + expect(summarizeVitalProvenance({})).toBeUndefined() + for (const key of ['configurationCount', 'unknownConfigurationRows', 'hrvRows', 'hrvDefinitionCount', 'unknownHrvRows'] as const) { + expect(summarizeVitalProvenance({ ...counts, [key]: undefined })).toBeUndefined() + } + expect(() => summarizeVitalProvenance({ ...counts, singleHrvMetricId: undefined })).toThrow() + }) + it('names a single verified HRV definition only when every HRV row shares it', () => { + expect(summarizeVitalProvenance(counts)).toEqual({ interpretation: 'original_observations', configurationCount: 1, unknownConfigurationRows: 0, + hrvRows: 4, hrvDefinitionCount: 1, unknownHrvRows: 0, hrvMetricId: 'window_ibi_variability_v1', hrvAggregation: 'single_definition' }) + expect(summarizeVitalProvenance({ ...counts, unknownHrvRows: 1 })).toMatchObject({ hrvMetricId: null, hrvAggregation: 'incompatible_definitions' }) + expect(summarizeVitalProvenance({ ...counts, hrvDefinitionCount: 2 })).toMatchObject({ hrvMetricId: null, hrvAggregation: 'incompatible_definitions' }) + expect(summarizeVitalProvenance({ ...counts, hrvDefinitionCount: 0, unknownHrvRows: 4, singleHrvMetricId: null })).toMatchObject({ hrvMetricId: null, hrvAggregation: 'legacy_unverified' }) + expect(summarizeVitalProvenance({ ...counts, hrvRows: 0, hrvDefinitionCount: 0, singleHrvMetricId: null })).toMatchObject({ hrvMetricId: null, hrvAggregation: 'no_readings' }) + }) + it('rejects negative or fractional aggregates', () => { + expect(() => summarizeVitalProvenance({ ...counts, hrvRows: -1 })).toThrow() + expect(() => summarizeVitalProvenance({ ...counts, configurationCount: 1.5 })).toThrow() + }) +}) + +describe('summaryProvenanceFields', () => { + let sqlite: Database.Database + let db: ReturnType> + beforeEach(() => { + sqlite = new Database(':memory:') + db = drizzle(sqlite, { schema }) + migrate(db, { migrationsFolder: 'src/db/biometrics-migrations' }) + }) + afterEach(() => sqlite.close()) + const query = () => db.select(summaryProvenanceFields).from(schema.vitals) + .leftJoin(schema.biometricProcessingRuns, eq(schema.vitals.processingRunId, schema.biometricProcessingRuns.id)).where(eq(schema.vitals.side, 'left')).get() + it('counts only well-formed live run bundles as known configurations and HRV definitions', () => { + const manifestHash = 'a'.repeat(64) + db.insert(schema.biometricProcessingRuns).values([ + { id: 'live', side: 'left', kind: 'live', bundle: { manifestHash, components: { hrv_index: { metricId: 'window_ibi_variability_v1' } } }, configHash: 'b'.repeat(64), inputIdentity: {}, startedAtMs: 1 }, + { id: 'replay', side: 'left', kind: 'replay', bundle: { manifestHash, components: { hrv_index: { metricId: 'window_ibi_variability_v1' } } }, configHash: 'b'.repeat(64), inputIdentity: {}, startedAtMs: 1 }, + { id: 'malformed', side: 'left', kind: 'live', bundle: { manifestHash: 'short', components: { hrv_index: { metricId: 'Bad Metric' } } }, configHash: 'not-a-hash', inputIdentity: {}, startedAtMs: 1 }, + ]).run() + db.insert(schema.vitals).values([ + { side: 'left', timestamp: new Date(1000), hrv: 10, processingRunId: 'live' }, + { side: 'left', timestamp: new Date(2000), hrv: 11, processingRunId: 'live' }, + { side: 'left', timestamp: new Date(3000), hrv: null, processingRunId: 'live' }, + { side: 'left', timestamp: new Date(4000), hrv: 12, processingRunId: 'replay' }, + { side: 'left', timestamp: new Date(5000), hrv: 13, processingRunId: 'malformed' }, + { side: 'left', timestamp: new Date(6000), hrv: 14, processingRunId: null }, + { side: 'right', timestamp: new Date(7000), hrv: 15, processingRunId: 'live' }, + ]).run() + expect(query()).toEqual({ configurationCount: 1, unknownConfigurationRows: 3, hrvRows: 5, hrvDefinitionCount: 1, unknownHrvRows: 3, singleHrvMetricId: 'window_ibi_variability_v1' }) + expect(summarizeVitalProvenance(query() ?? {})).toMatchObject({ hrvAggregation: 'incompatible_definitions', hrvMetricId: null }) + }) + it('reports no readings and no configurations for an empty side', () => { + expect(query()).toEqual({ configurationCount: 0, unknownConfigurationRows: 0, hrvRows: 0, hrvDefinitionCount: 0, unknownHrvRows: 0, singleHrvMetricId: null }) + expect(summarizeVitalProvenance(query() ?? {})).toMatchObject({ hrvAggregation: 'no_readings' }) + }) +}) diff --git a/src/biometrics/profiles/vital-provenance.ts b/src/biometrics/profiles/vital-provenance.ts new file mode 100644 index 000000000..5a6c3e3ec --- /dev/null +++ b/src/biometrics/profiles/vital-provenance.ts @@ -0,0 +1,34 @@ +import { z } from 'zod' + +export const vitalProvenanceSchema = z.object({ + interpretation: z.literal('original_observation'), + status: z.enum(['recorded', 'legacy_unknown', 'unavailable']), + processingRunId: z.string().nullable(), + configurationHash: z.string().nullable(), + hrvMetricId: z.string().nullable(), +}) +export type VitalProvenance = z.infer + +/** Describe recorded metadata only; today's manifest cannot identify old rows. */ +export function vitalProvenance(row: { + side: string + processingRunId?: string | null + recordedRunId?: string | null + recordedSide?: string | null + recordedKind?: string | null + recordedConfigHash?: string | null + recordedHrvMetricId?: unknown +}): VitalProvenance { + const id = row.processingRunId ?? null + const recorded = id !== null && row.recordedRunId === id && row.recordedSide === row.side + && row.recordedKind === 'live' && /^[a-f0-9]{64}$/.test(row.recordedConfigHash ?? '') + return { + interpretation: 'original_observation', + status: id === null ? 'legacy_unknown' : recorded ? 'recorded' : 'unavailable', + processingRunId: id, + configurationHash: recorded ? row.recordedConfigHash ?? null : null, + hrvMetricId: recorded && typeof row.recordedHrvMetricId === 'string' && /^[a-z0-9_]{1,80}$/.test(row.recordedHrvMetricId) + ? row.recordedHrvMetricId + : null, + } +} diff --git a/src/biometrics/profiles/vital-summary-provenance.ts b/src/biometrics/profiles/vital-summary-provenance.ts new file mode 100644 index 000000000..b902f794b --- /dev/null +++ b/src/biometrics/profiles/vital-summary-provenance.ts @@ -0,0 +1,48 @@ +import { z } from 'zod' +import { sql } from 'drizzle-orm' +import { biometricProcessingRuns as runs, vitals } from '@/src/db/biometrics-schema' + +// The same-side join is required by callers. A saved config hash alone does not +// identify an engine version, so configuration identity also includes its manifest. +const live = sql`(${runs.kind} = 'live' AND length(${runs.configHash}) = 64 AND ${runs.configHash} NOT GLOB '*[^a-f0-9]*')` +const manifest = sql`CASE WHEN json_valid(${runs.bundle}) THEN json_extract(${runs.bundle}, '$.manifestHash') END` +const rawMetric = sql`CASE WHEN json_valid(${runs.bundle}) THEN json_extract(${runs.bundle}, '$.components.hrv_index.metricId') END` +const metric = sql`CASE WHEN ${live} AND typeof(${rawMetric}) = 'text' AND length(${rawMetric}) BETWEEN 1 AND 80 AND ${rawMetric} NOT GLOB '*[^a-z0-9_]*' THEN ${rawMetric} END` +const configuration = sql`CASE WHEN ${live} AND typeof(${manifest}) = 'text' AND length(${manifest}) = 64 AND ${manifest} NOT GLOB '*[^a-f0-9]*' THEN ${manifest} || ':' || ${runs.configHash} END` +export const summaryProvenanceFields = { + configurationCount: sql`COUNT(DISTINCT ${configuration})`, + unknownConfigurationRows: sql`COUNT(CASE WHEN ${configuration} IS NULL THEN 1 END)`, + hrvRows: sql`COUNT(${vitals.hrv})`, + hrvDefinitionCount: sql`COUNT(DISTINCT CASE WHEN ${vitals.hrv} IS NOT NULL THEN ${metric} END)`, + unknownHrvRows: sql`COUNT(CASE WHEN ${vitals.hrv} IS NOT NULL AND ${metric} IS NULL THEN 1 END)`, + singleHrvMetricId: sql`MIN(CASE WHEN ${vitals.hrv} IS NOT NULL THEN ${metric} END)`, +} +export const summaryProvenanceSchema = z.object({ + interpretation: z.literal('original_observations'), + configurationCount: z.number().int().nonnegative(), + unknownConfigurationRows: z.number().int().nonnegative(), + hrvRows: z.number().int().nonnegative(), + hrvDefinitionCount: z.number().int().nonnegative(), + unknownHrvRows: z.number().int().nonnegative(), + hrvMetricId: z.string().nullable(), + hrvAggregation: z.enum(['single_definition', 'legacy_unverified', 'incompatible_definitions', 'no_readings']), +}) +export function summarizeVitalProvenance(row: { + configurationCount?: number + unknownConfigurationRows?: number + hrvRows?: number + hrvDefinitionCount?: number + unknownHrvRows?: number + singleHrvMetricId?: string | null +}) { + // Compatibility with adapters that have not supplied these new aggregate fields. + if ([row.configurationCount, row.unknownConfigurationRows, row.hrvRows, row.hrvDefinitionCount, row.unknownHrvRows].some(value => value === undefined)) return undefined + const incompatible = (row.hrvDefinitionCount ?? 0) > 1 || ((row.hrvDefinitionCount ?? 0) > 0 && (row.unknownHrvRows ?? 0) > 0) + return summaryProvenanceSchema.parse({ + interpretation: 'original_observations', configurationCount: row.configurationCount, + unknownConfigurationRows: row.unknownConfigurationRows, hrvRows: row.hrvRows, + hrvDefinitionCount: row.hrvDefinitionCount, unknownHrvRows: row.unknownHrvRows, + hrvMetricId: row.hrvDefinitionCount === 1 && row.unknownHrvRows === 0 ? row.singleHrvMetricId : null, + hrvAggregation: row.hrvRows === 0 ? 'no_readings' : incompatible ? 'incompatible_definitions' : row.hrvDefinitionCount === 0 ? 'legacy_unverified' : 'single_definition', + }) +} diff --git a/src/biometrics/references/alignment.ts b/src/biometrics/references/alignment.ts new file mode 100644 index 000000000..b07e0fa6b --- /dev/null +++ b/src/biometrics/references/alignment.ts @@ -0,0 +1,133 @@ +import type { NormalizedMeasurement, ReferenceMetric } from './contracts' +import { ReferenceInputError } from './time' + +export interface Observation { + timestampMs: number + value: number | null + /** Known support duration only; a point reading must not imply gap filling. */ + durationMs?: number +} +export interface AlignmentPolicy { + version: 'reference-alignment-v1' + pointToleranceMs: number + clockOffsetMs: number + offsetSource: 'fixed' | 'training' +} +export const DEFAULT_ALIGNMENT: AlignmentPolicy = { version: 'reference-alignment-v1', pointToleranceMs: 30000, clockOffsetMs: 0, offsetSource: 'fixed' } + +export function referenceCompatibility(metric: ReferenceMetric, windowMs: number | null, reference: NormalizedMeasurement): string | null { + if (!reference.scorable) return reference.exclusions.join(',') || 'unscorable_reference' + if (metric === 'hrv_unknown' || reference.metric === 'hrv_unknown') return 'unknown_hrv_definition' + if (reference.metric !== metric) return 'incompatible_metric_definition' + if (metric.startsWith('hrv_') || metric === 'window_ibi_variability_v1') { + if (windowMs === null || reference.windowMs === null || reference.windowMs !== windowMs) return 'incompatible_hrv_window' + } + return null +} + +function quantile(values: number[], q: number): number { + const sorted = [...values].sort((a, b) => a - b) + const index = (sorted.length - 1) * q + const lower = Math.floor(index) + return sorted[lower] + (sorted[Math.ceil(index)] - sorted[lower]) * (index - lower) +} + +function unionDuration(intervals: Array<[number, number]>): number { + let total = 0, end = -Infinity + for (const [start, stop] of intervals.sort((a, b) => a[0] - b[0])) { + total += Math.max(0, stop - Math.max(start, end)) + end = Math.max(end, stop) + } + return total +} + +/** + * Greedy chronological, nearest-unused point pairing; interval mean/median + * uses only actual pod samples inside the interval. No forward fill. Each + * observation and reference can contribute to at most one independent pair. + * Offset is supplied by a frozen protocol; never fitted on the evaluation set. + */ +export function alignReferences(metric: ReferenceMetric, windowMs: number | null, observations: Observation[], references: NormalizedMeasurement[], policy: AlignmentPolicy = DEFAULT_ALIGNMENT) { + if (metric === 'sleep_stages') throw new ReferenceInputError('INCOMPATIBLE_METRIC', 'Use the duration-weighted stage evaluator') + if (policy.version !== 'reference-alignment-v1' || !['fixed', 'training'].includes(policy.offsetSource) + || !Number.isFinite(policy.pointToleranceMs) || policy.pointToleranceMs < 0 || policy.pointToleranceMs > 300000 + || !Number.isFinite(policy.clockOffsetMs) || Math.abs(policy.clockOffsetMs) > 3600000) throw new ReferenceInputError('INVALID_ALIGNMENT', 'Unsupported alignment policy') + if (observations.length > 100000 || references.length > 100000) throw new ReferenceInputError('RESOURCE_LIMIT', 'Use bounded whole-night datasets') + for (const observation of observations) { + if (!Number.isFinite(observation.timestampMs) || (observation.value !== null && (!Number.isFinite(observation.value) || Math.abs(observation.value) > Number.MAX_SAFE_INTEGER)) + || (observation.durationMs !== undefined && (!Number.isFinite(observation.durationMs) || observation.durationMs < 0 || observation.durationMs > 300000))) { + throw new ReferenceInputError('INVALID_OBSERVATION', 'Observation timestamps, values and support durations must be finite and bounded') + } + } + const available = observations.filter(row => row.value !== null).map(row => ({ ...row, timestampMs: row.timestampMs + policy.clockOffsetMs })).sort((a, b) => a.timestampMs - b.timestampMs) + const used = new Set() + const duplicateKeys = new Set() + const exclusions: Record = {} + let eligibleReferences = 0, duplicateReferences = 0 + const pairs: Array<{ referenceKey: string, referenceStartMs: number, referenceEndMs: number, referenceValue: number, podValue: number, podCount: number, signedError: number, pairedDurationMs: number }> = [] + let comparisons = 0 + for (const reference of [...references].sort((a, b) => a.startMs - b.startMs || a.recordKey.localeCompare(b.recordKey))) { + const incompatible = referenceCompatibility(metric, windowMs, reference) + if (incompatible) { + exclusions[incompatible] = (exclusions[incompatible] ?? 0) + 1 + continue + } + if (reference.value === null || !Number.isFinite(reference.value) || Math.abs(reference.value) > Number.MAX_SAFE_INTEGER) throw new ReferenceInputError('INVALID_REFERENCE', 'Numeric reference value is required') + const key = JSON.stringify([reference.metric, reference.startMs, reference.endMs, reference.value, reference.aggregation, reference.windowMs]) + if (duplicateKeys.has(key)) { + duplicateReferences++ + continue + } + duplicateKeys.add(key) + eligibleReferences++ + const isPoint = reference.temporalKind === 'point' + const start = reference.startMs - (isPoint ? policy.pointToleranceMs : 0) + const end = reference.endMs + (isPoint ? policy.pointToleranceMs : 0) + let lower = 0, upper = available.length + while (lower < upper) { + const middle = Math.floor((lower + upper) / 2) + if (available[middle].timestampMs < start) lower = middle + 1 + else upper = middle + } + const candidates: number[] = [] + for (let i = lower; i < available.length; i++) { + if (++comparisons > 2000000) throw new ReferenceInputError('RESOURCE_LIMIT', 'Overlapping reference windows exceed the alignment work budget') + const timestamp = available[i].timestampMs + if (isPoint ? timestamp > end : timestamp >= end) break + if (!used.has(i)) candidates.push(i) + } + if (!candidates.length) continue + let selected = candidates + if (isPoint) { + selected = [candidates.reduce((best, index) => Math.abs(available[index].timestampMs - reference.startMs) < Math.abs(available[best].timestampMs - reference.startMs) ? index : best)] + } + const values = selected.map(index => available[index].value as number) + const podValue = reference.aggregation === 'median' ? quantile(values, 0.5) : values.reduce((sum, value) => sum + value, 0) / values.length + const support: Array<[number, number]> = [] + for (const index of selected) { + used.add(index) + const observation = available[index] + if (!isPoint && observation.durationMs) support.push([observation.timestampMs, Math.min(reference.endMs, observation.timestampMs + observation.durationMs)]) + } + pairs.push({ referenceKey: reference.recordKey, referenceStartMs: reference.startMs, referenceEndMs: reference.endMs, + referenceValue: reference.value, podValue, podCount: selected.length, signedError: podValue - reference.value, pairedDurationMs: unionDuration(support) }) + } + const errors = pairs.map(pair => Math.abs(pair.signedError)) + return { + status: !references.length ? 'no_reference' as const : pairs.length ? 'compared' as const : eligibleReferences ? 'no_overlap' as const : 'incompatible' as const, + metric, windowMs, policy: { ...policy }, + metrics: pairs.length + ? { mae: errors.reduce((sum, error) => sum + error, 0) / errors.length, + medianAbsoluteError: quantile(errors, 0.5), p95AbsoluteError: quantile(errors, 0.95), + signedBias: pairs.reduce((sum, pair) => sum + pair.signedError, 0) / pairs.length } + : null, + pairedCount: pairs.length, pairedDurationMs: pairs.reduce((sum, pair) => sum + pair.pairedDurationMs, 0), + eligibleReferences, duplicateReferences, excludedReferences: references.length - eligibleReferences - duplicateReferences, + referenceCoverage: eligibleReferences ? pairs.length / eligibleReferences : null, + outputCoverage: observations.length ? available.length / observations.length : null, + observationCount: observations.length, validObservationCount: available.length, matchedObservationCount: used.size, + unmatchedReferences: eligibleReferences - pairs.length, unmatchedObservations: available.length - used.size, + exclusions, pairs, + limitations: ['Sparse reference samples do not establish whole-night accuracy.', 'Point samples contribute counts, not inferred duration.', 'Interval comparisons use available observations without filling gaps.'], + } +} diff --git a/src/biometrics/references/comparison.ts b/src/biometrics/references/comparison.ts new file mode 100644 index 000000000..dbd59ab72 --- /dev/null +++ b/src/biometrics/references/comparison.ts @@ -0,0 +1,95 @@ +import { and, eq, gte, lt } from 'drizzle-orm' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import { z } from 'zod' +import * as schema from '@/src/db/biometrics-schema' +import { dataHash } from '@/src/biometrics/algorithms/canonical' +import { sideSchema } from '@/src/server/validation-schemas' +import { timestampMsSchema } from './contracts' +import { alignReferences, DEFAULT_ALIGNMENT } from './alignment' +import { ReferenceStore } from './store' +import { ReferenceInputError } from './time' + +export const compareReadingsSchema = z.object({ + side: sideSchema, + subjectId: z.string().min(1).max(200), + importId: z.string().uuid(), + metric: z.enum(['heart_rate', 'breathing_rate', 'window_ibi_variability_v1', 'hrv_rmssd', 'hrv_sdnn', 'hrv_unknown']), + startMs: timestampMsSchema, + endMs: timestampMsSchema, + pointToleranceMs: z.number().int().min(0).max(300000).default(30000), + clockOffsetMs: z.number().int().min(-3600000).max(3600000).default(0), +}).strict().refine(input => input.endMs > input.startMs && input.endMs - input.startMs <= 86400000, { + message: 'Select an interval greater than zero and at most 24 hours', +}) + +const numericScores = z.object({ mae: z.number(), medianAbsoluteError: z.number(), p95AbsoluteError: z.number(), signedBias: z.number() }) +export const comparisonReportSchema = z.object({ + version: z.literal('stored-readings-comparison-v1'), + inputHash: z.string(), + side: sideSchema, subjectId: z.string(), startMs: z.number(), endMs: z.number(), + importId: z.string().uuid(), importRevision: z.number(), source: z.string(), + status: z.enum(['no_reference', 'compared', 'no_overlap', 'incompatible']), + requestedMetric: z.string(), metric: z.string(), windowMs: z.number().nullable(), + policy: z.object({ version: z.literal('reference-alignment-v1'), pointToleranceMs: z.number(), clockOffsetMs: z.number(), offsetSource: z.literal('fixed') }), + metrics: numericScores.nullable(), pairedCount: z.number(), pairedDurationMs: z.number(), + eligibleReferences: z.number(), duplicateReferences: z.number(), excludedReferences: z.number(), + referenceCoverage: z.number().nullable(), outputCoverage: z.number().nullable(), + observationCount: z.number(), validObservationCount: z.number(), matchedObservationCount: z.number(), + unmatchedReferences: z.number(), unmatchedObservations: z.number(), exclusions: z.record(z.string(), z.number()), + provenance: z.array(z.object({ processingRunId: z.string().nullable(), count: z.number() })), + limitations: z.array(z.string()), +}) + +/** Bounded, read-only comparison. Saved evaluations instead retain their input bytes. */ +export function compareStoredReadings(db: BetterSQLite3Database, raw: z.input) { + const input = compareReadingsSchema.parse(raw) + const store = new ReferenceStore(db) + // Capture all input rows in a single read snapshot, then release SQLite before scoring. + const snapshot = db.transaction(() => { + const reference = store.get(input.importId) + if (!reference.available) throw new ReferenceInputError('REFERENCE_DELETED', 'Reference measurements were explicitly deleted') + if (reference.summary.side !== input.side || reference.summary.subjectId !== input.subjectId) throw new ReferenceInputError('SUBJECT_MISMATCH', 'Select the import matching this side and sleeper') + const measurements = [] + let cursor = 0 + do { + const page = store.measurements(input.importId, cursor) + measurements.push(...page.items) + if (page.nextCursor === null) break + cursor = page.nextCursor + } while (measurements.length <= 2000) + if (measurements.length > 2000) throw new ReferenceInputError('RESOURCE_LIMIT', 'Reference import exceeds comparison budget') + const rows = db.select().from(schema.vitals).where(and(eq(schema.vitals.side, input.side), + gte(schema.vitals.timestamp, new Date(Math.ceil(input.startMs / 1000) * 1000)), + lt(schema.vitals.timestamp, new Date(Math.ceil(input.endMs / 1000) * 1000)), + )).orderBy(schema.vitals.timestamp).limit(100001).all() + if (rows.length > 100000) throw new ReferenceInputError('RESOURCE_LIMIT', 'Select a shorter comparison interval') + return { reference, measurements, rows } + }) + const references = snapshot.measurements.filter(row => row.metric === input.metric + && (row.temporalKind === 'point' + ? row.startMs >= input.startMs && row.startMs < input.endMs + : row.startMs >= input.startMs && row.endMs <= input.endMs)) + // HRV in canonical vitals is exclusively the window IBI variability index. + // A requested RMSSD/SDNN comparison therefore remains incompatible. + const podMetric = input.metric.startsWith('hrv_') ? 'window_ibi_variability_v1' : input.metric + const windowMs = podMetric === 'window_ibi_variability_v1' ? 300000 : null + const observations = snapshot.rows.map(row => ({ timestampMs: row.timestamp.getTime(), value: + podMetric === 'heart_rate' ? row.heartRate : podMetric === 'breathing_rate' ? row.breathingRate : row.hrv })) + const alignment = alignReferences(podMetric, windowMs, observations, references, + { ...DEFAULT_ALIGNMENT, pointToleranceMs: input.pointToleranceMs, clockOffsetMs: input.clockOffsetMs }) + const provenance = new Map() + for (const row of snapshot.rows) provenance.set(row.processingRunId, (provenance.get(row.processingRunId) ?? 0) + 1) + const inputHash = dataHash({ input, referenceHash: snapshot.reference.summary.contentHash, + observations: snapshot.rows.map(row => ({ ...row, timestamp: row.timestamp.getTime() })) }) + return comparisonReportSchema.parse({ ...alignment, requestedMetric: input.metric, version: 'stored-readings-comparison-v1', inputHash, + side: input.side, subjectId: input.subjectId, startMs: input.startMs, endMs: input.endMs, + importId: input.importId, importRevision: snapshot.reference.revision, source: snapshot.reference.summary.source, + provenance: [...provenance].map(([processingRunId, count]) => ({ processingRunId, count })), + limitations: [...alignment.limitations, + 'This compares original stored readings; it does not regenerate signals or validate a candidate.', + 'The input hash identifies this read snapshot; inputs are not retained until a dataset is frozen.', + 'Stored readings have no recorded support duration, so paired duration is unknown and reported as zero.', + 'Reference intervals crossing the selected boundaries are excluded to preserve their aggregation semantics.', + ...(provenance.has(null) ? ['Some observations have legacy_unknown processing provenance.'] : [])], + }) +} diff --git a/src/biometrics/references/contracts.ts b/src/biometrics/references/contracts.ts new file mode 100644 index 000000000..8fbdd1e7b --- /dev/null +++ b/src/biometrics/references/contracts.ts @@ -0,0 +1,95 @@ +import { z } from 'zod' +import { sideSchema } from '@/src/server/validation-schemas' + +export const REFERENCE_ADAPTER_VERSION = 'normalized-reference-v1' +export const REFERENCE_LIMITS = { bytes: 2 * 1024 * 1024, rows: 2000, depth: 12, nodes: 100000, rejectionExamples: 25 } as const +export const referenceMetricSchema = z.enum(['heart_rate', 'breathing_rate', 'window_ibi_variability_v1', 'hrv_rmssd', 'hrv_sdnn', 'hrv_unknown', 'sleep_stages']) +export type ReferenceMetric = z.infer +export const referenceStageSchema = z.enum(['wake', 'light', 'deep', 'rem', 'asleep', 'in_bed', 'unknown']) +export const timestampMsSchema = z.number().int().min(1_000_000_000_000).max(4_102_444_800_000) +export const timePolicySchema = z.object({ + timezone: z.string().min(1).max(100).optional(), + utcOffsetMinutes: z.number().int().min(-840).max(840).optional(), + disambiguation: z.enum(['reject', 'earlier', 'later']).default('reject'), +}).strict() + +export const referenceRowSchema = z.object({ + recordKey: z.string().min(1).max(200), + subjectId: z.string().min(1).max(200).optional(), + metric: referenceMetricSchema, + value: z.union([z.number().finite(), referenceStageSchema]), + unit: z.string().min(1).max(40), + timestampMs: timestampMsSchema.optional(), + startMs: timestampMsSchema.optional(), + endMs: timestampMsSchema.optional(), + // Offset-bearing ISO, or an explicitly resolved local time under timePolicy. + timestamp: z.string().max(80).optional(), + start: z.string().max(80).optional(), + end: z.string().max(80).optional(), + aggregation: z.enum(['instant', 'mean', 'median', 'window_index', 'stage']).optional(), + windowMs: z.number().int().min(1).max(24 * 60 * 60 * 1000).optional(), + quality: z.number().finite().min(0).max(1).optional(), + annotation: z.string().max(1000).optional(), +}).strict() +export type ReferenceRow = z.infer + +export const referenceImportSchema = z.object({ + side: sideSchema, + subjectId: z.string().min(1).max(200), + source: z.string().min(1).max(120), + timePolicy: timePolicySchema.default({ disambiguation: 'reject' }), + rows: z.array(z.unknown()).min(1).max(REFERENCE_LIMITS.rows), +}).strict() +export type ReferenceImportInput = z.input + +export const normalizedMeasurementSchema = z.object({ + recordKey: z.string(), + metric: referenceMetricSchema, + value: z.number().nullable(), + stage: referenceStageSchema.nullable(), + unit: z.string(), + startMs: timestampMsSchema, + endMs: timestampMsSchema, + temporalKind: z.enum(['point', 'interval']), + aggregation: z.enum(['instant', 'mean', 'median', 'window_index', 'stage']), + windowMs: z.number().nullable(), + quality: z.number().nullable(), + scorable: z.boolean(), + exclusions: z.array(z.string()), + // Retain the submitted units, values, source times and annotations as data. + original: referenceRowSchema, +}).strict() +export type NormalizedMeasurement = z.infer + +export const previewSummarySchema = z.object({ + contentHash: z.string(), + adapterVersion: z.string(), + side: sideSchema, + subjectId: z.string(), + source: z.string(), + timePolicy: timePolicySchema, + received: z.number(), + accepted: z.number(), + rejected: z.number(), + duplicates: z.number(), + unscorable: z.number(), + conflicts: z.number(), + startMs: z.number().nullable(), + endMs: z.number().nullable(), + metrics: z.array(referenceMetricSchema), + rejectionCounts: z.record(z.string(), z.number()), + rejectionExamples: z.array(z.object({ row: z.number(), code: z.string(), message: z.string() })), + warnings: z.array(z.string()), +}) +export type PreviewSummary = z.infer + +export const referenceImportDescriptorSchema = z.object({ + id: z.string().uuid(), + familyId: z.string().uuid(), + revision: z.number().int(), + supersedesImportId: z.string().uuid().nullable(), + summary: previewSummarySchema, + createdAtMs: z.number(), + deletedAtMs: z.number().nullable(), + available: z.boolean(), +}) diff --git a/src/biometrics/references/legacy.ts b/src/biometrics/references/legacy.ts new file mode 100644 index 000000000..392c8a83a --- /dev/null +++ b/src/biometrics/references/legacy.ts @@ -0,0 +1,29 @@ +import { inArray } from 'drizzle-orm' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import * as schema from '@/src/db/biometrics-schema' +import type { ReferenceImportInput } from './contracts' +import { previewReferenceImport } from './normalize' +import { ReferenceStore } from './store' + +type LegacyVital = typeof schema.referenceVitals.$inferSelect +type LegacyStage = typeof schema.referenceStages.$inferSelect + +/** + * Called inside the existing legacy endpoint's insertion transaction. Only + * newly accepted rows get provenance; first-write-wins and stage-overlap + * rejection retain their old semantics. Historical unlinked rows stay unknown. + */ +export function recordLegacyReferenceImport(db: BetterSQLite3Database, kind: 'vitals' | 'stages', rows: Array): void { + const first = rows[0] + if (!first) return + const annotation = 'Legacy Apple Watch API: explicit side mapping; sleeper identity and source timezone were not provided. Original timestamps use Unix seconds.' + const input: ReferenceImportInput = { side: first.side, subjectId: `legacy_unknown:${first.side}`, source: first.source, + rows: rows.map(row => 'heartRate' in row + ? { recordKey: `apple-watch-hr:${row.timestamp.getTime()}`, metric: 'heart_rate', value: row.heartRate, unit: 'bpm', timestampMs: row.timestamp.getTime(), annotation } + : { recordKey: `apple-watch-stage:${row.start.getTime()}`, metric: 'sleep_stages', value: row.stage, unit: 'stage', startMs: row.start.getTime(), endMs: row.end.getTime(), annotation }), + } + const preview = previewReferenceImport(input) + const result = new ReferenceStore(db).commit({ import: input, previewHash: preview.summary.contentHash }) + const table = kind === 'vitals' ? schema.referenceVitals : schema.referenceStages + for (let i = 0; i < rows.length; i += 100) db.update(table).set({ importId: result.id }).where(inArray(table.id, rows.slice(i, i + 100).map(row => row.id))).run() +} diff --git a/src/biometrics/references/normalize-core.ts b/src/biometrics/references/normalize-core.ts new file mode 100644 index 000000000..86cf8b17a --- /dev/null +++ b/src/biometrics/references/normalize-core.ts @@ -0,0 +1,151 @@ +import { canonicalJson } from '../algorithms/canonical-json' +const utf8 = new TextEncoder() +import { REFERENCE_ADAPTER_VERSION, REFERENCE_LIMITS, referenceImportSchema, referenceRowSchema, type NormalizedMeasurement, type PreviewSummary, type ReferenceImportInput } from './contracts' +import { ReferenceInputError, timestampResolver } from './time' + +/** Validate size/depth before copying, hashing or parsing untrusted exports. */ +export function assertBoundedReferenceData(input: unknown): void { + const stack = [{ value: input, depth: 0 }] + const seen = new WeakSet() + let nodes = 0, stringBytes = 0 + while (stack.length) { + const entry = stack.pop() + if (!entry) break + const { value, depth } = entry + if (++nodes > REFERENCE_LIMITS.nodes || depth > REFERENCE_LIMITS.depth) throw new ReferenceInputError('RESOURCE_LIMIT', 'Reference JSON is too large or deeply nested') + if (typeof value === 'string') stringBytes += utf8.encode(value).byteLength + else if (typeof value === 'number') { + if (!Number.isFinite(value)) throw new ReferenceInputError('NON_FINITE_VALUE', 'Reference values must be finite JSON numbers') + } + else if (value !== null && typeof value === 'object') { + if (seen.has(value)) throw new ReferenceInputError('INVALID_JSON', 'Reference input must be a JSON tree') + seen.add(value) + if (!Array.isArray(value) && Object.getPrototypeOf(value) !== Object.prototype && Object.getPrototypeOf(value) !== null) throw new ReferenceInputError('INVALID_JSON', 'Reference input must contain plain JSON objects') + for (const [key, child] of Object.entries(value)) { + stringBytes += utf8.encode(key).byteLength + stack.push({ value: child, depth: depth + 1 }) + } + } + else if (value !== null && typeof value !== 'boolean') throw new ReferenceInputError('INVALID_JSON', 'Reference input contains a non-JSON value') + if (stringBytes > REFERENCE_LIMITS.bytes) throw new ReferenceInputError('RESOURCE_LIMIT', 'Reference upload exceeds 2 MiB') + } + if (utf8.encode(JSON.stringify(input)).byteLength > REFERENCE_LIMITS.bytes) throw new ReferenceInputError('RESOURCE_LIMIT', 'Reference upload exceeds 2 MiB') +} + +function normalizeRow(raw: unknown, subjectId: string, resolveTime: (value: string | number) => number): NormalizedMeasurement { + const row = referenceRowSchema.parse(raw) + if (row.subjectId !== undefined && row.subjectId !== subjectId) throw new ReferenceInputError('SUBJECT_MISMATCH', 'Row belongs to a different subject; split the export and map each sleeper explicitly') + const point = row.timestampMs ?? row.timestamp + const start = row.startMs ?? row.start + const end = row.endMs ?? row.end + if ((row.timestampMs !== undefined && row.timestamp !== undefined) + || (row.startMs !== undefined && row.start !== undefined) || (row.endMs !== undefined && row.end !== undefined) + || (point !== undefined && (start !== undefined || end !== undefined))) { + throw new ReferenceInputError('AMBIGUOUS_TIME_FIELDS', 'Specify a single point or interval time representation') + } + const temporalKind = point !== undefined ? 'point' : 'interval' + if (point === undefined && (start === undefined || end === undefined)) throw new ReferenceInputError('MISSING_TIMESTAMP', 'Provide timestampMs or startMs/endMs, or ISO equivalents') + const startMs = resolveTime(point ?? start ?? '') + const endMs = point !== undefined ? startMs : resolveTime(end ?? '') + if (temporalKind === 'interval' && endMs <= startMs) throw new ReferenceInputError('INVALID_INTERVAL', 'end must be after start') + if (endMs - startMs > 24 * 60 * 60 * 1000) throw new ReferenceInputError('INVALID_INTERVAL', 'A reference interval may span at most 24 hours') + const aggregation = row.aggregation ?? (row.metric === 'sleep_stages' ? 'stage' : temporalKind === 'point' ? 'instant' : 'mean') + if ((temporalKind === 'interval' && aggregation === 'instant') || (row.metric !== 'sleep_stages' && aggregation === 'stage')) throw new ReferenceInputError('INVALID_AGGREGATION', 'Aggregation does not match the measurement semantics') + const exclusions: string[] = [] + let value: number | null = null, stage: NormalizedMeasurement['stage'] = null, unit: string + if (row.metric === 'sleep_stages') { + if (typeof row.value !== 'string' || row.unit !== 'stage' || temporalKind !== 'interval' || aggregation !== 'stage') throw new ReferenceInputError('INCOMPATIBLE_METRIC', 'Sleep stages require a labeled interval with unit stage') + stage = row.value + unit = 'stage' + if (!['wake', 'light', 'deep', 'rem'].includes(stage)) exclusions.push('unclassified_stage') + } + else { + if (typeof row.value !== 'number') throw new ReferenceInputError('INCOMPATIBLE_METRIC', 'Vital measurements require numeric values') + const units: Record = row.metric === 'heart_rate' + ? { 'bpm': 1, 'beats/min': 1, 'Hz': 60 } + : row.metric === 'breathing_rate' + ? { 'breaths/min': 1, 'br/min': 1, 'Hz': 60 } + : { ms: 1, s: 1000 } + if (!Object.hasOwn(units, row.unit)) throw new ReferenceInputError('UNSUPPORTED_UNIT', `Unsupported unit for ${row.metric}`) + value = row.value * units[row.unit] + if (!Number.isFinite(value) || value < 0 || value > Number.MAX_SAFE_INTEGER) throw new ReferenceInputError('INVALID_VALUE', 'Measurements must be finite, nonnegative and within the safe numeric range after conversion') + unit = row.metric === 'heart_rate' ? 'bpm' : row.metric === 'breathing_rate' ? 'breaths/min' : 'ms' + if (row.metric === 'hrv_unknown') exclusions.push('unknown_hrv_definition') + if (row.metric === 'window_ibi_variability_v1' && (row.windowMs !== 300000 || aggregation !== 'window_index')) exclusions.push('incompatible_hrv_window') + } + return { recordKey: row.recordKey, metric: row.metric, value, stage, unit, startMs, endMs, temporalKind, + aggregation, windowMs: row.windowMs ?? null, quality: row.quality ?? null, + scorable: exclusions.length === 0, exclusions, original: row } +} + +export function normalizeReferenceImport(raw: ReferenceImportInput): { summary: Omit, measurements: NormalizedMeasurement[], uploadJson: string } { + assertBoundedReferenceData(raw) + const input = referenceImportSchema.parse(raw) + const resolveTime = timestampResolver(input.timePolicy) + const rejected = new Map() + const normalized: Array<{ row: number, measurement: NormalizedMeasurement }> = [] + for (const [index, value] of input.rows.entries()) { + try { + const measurement = normalizeRow(value, input.subjectId, resolveTime) + canonicalJson(measurement) + normalized.push({ row: index, measurement }) + } + catch (error) { + rejected.set(index, error instanceof ReferenceInputError + ? { code: error.code, message: error.message } + : { code: 'INVALID_ROW', message: 'Row does not match the documented reference schema; check fields, types and millisecond timestamps' }) + } + } + const byKey = new Map() + for (const entry of normalized) { + const group = byKey.get(entry.measurement.recordKey) ?? [] + group.push(entry) + byKey.set(entry.measurement.recordKey, group) + } + let duplicates = 0 + const measurements: NormalizedMeasurement[] = [] + for (const group of byKey.values()) { + const fingerprints = new Set(group.map(entry => canonicalJson(entry.measurement))) + if (fingerprints.size > 1) { + for (const entry of group) rejected.set(entry.row, { code: 'CONFLICTING_RECORD_KEY', message: 'The same source record key has different measurements in this batch' }) + } + else { + measurements.push(group[0].measurement) + duplicates += group.length - 1 + } + } + // Persist all conflicting intervals, but exclude both sides of a conflict. + // Source priority is never guessed; unknown/generic asleep is never light. + const stages = measurements.filter(row => row.metric === 'sleep_stages').sort((a, b) => a.startMs - b.startMs) + let active: NormalizedMeasurement[] = [] + let conflicts = 0 + for (const row of stages) { + active = active.filter(previous => previous.endMs > row.startMs) + for (const previous of active) { + if (row.stage !== previous.stage) { + conflicts++ + for (const conflicting of [previous, row]) { + conflicting.scorable = false + if (!conflicting.exclusions.includes('overlapping_stage_conflict')) conflicting.exclusions.push('overlapping_stage_conflict') + } + } + } + active.push(row) + } + const rejectionCounts: Record = {} + for (const rejection of rejected.values()) rejectionCounts[rejection.code] = (rejectionCounts[rejection.code] ?? 0) + 1 + const warnings = ['Agreement with a reference source is not clinical accuracy.', 'Stage overlaps with conflicting labels are excluded from scoring.'] + if (measurements.some(row => ['hrv_rmssd', 'hrv_sdnn', 'hrv_unknown'].includes(row.metric))) warnings.push('RMSSD, SDNN and unknown HRV definitions cannot directly score the pod window-level IBI variability index.') + // Hash the exact bounded upload plus interpretation. No arbitrary paths or URLs. + const uploadJson = JSON.stringify({ adapterVersion: REFERENCE_ADAPTER_VERSION, ...input }, (_key, value) => value !== null && typeof value === 'object' && !Array.isArray(value) ? Object.fromEntries(Object.keys(value).sort().map(key => [key, value[key]])) : value) + return { measurements, uploadJson, summary: { + adapterVersion: REFERENCE_ADAPTER_VERSION, side: input.side, subjectId: input.subjectId, + source: input.source, timePolicy: input.timePolicy, received: input.rows.length, + accepted: measurements.length, rejected: rejected.size, duplicates, + unscorable: measurements.filter(row => !row.scorable).length, conflicts, + startMs: measurements.length ? Math.min(...measurements.map(row => row.startMs)) : null, + endMs: measurements.length ? Math.max(...measurements.map(row => row.endMs)) : null, + metrics: [...new Set(measurements.map(row => row.metric))].sort(), rejectionCounts, + rejectionExamples: [...rejected].slice(0, REFERENCE_LIMITS.rejectionExamples).map(([row, error]) => ({ row: row + 1, ...error })), warnings, + } } +} diff --git a/src/biometrics/references/normalize.ts b/src/biometrics/references/normalize.ts new file mode 100644 index 000000000..412cf5c31 --- /dev/null +++ b/src/biometrics/references/normalize.ts @@ -0,0 +1,13 @@ +import { createHash } from 'node:crypto' +import { configHash } from '../algorithms/canonical' +import { REFERENCE_ADAPTER_VERSION, type ReferenceImportInput } from './contracts' +import { normalizeReferenceImport } from './normalize-core' +export { assertBoundedReferenceData } from './normalize-core' + +export function previewReferenceImport(raw: ReferenceImportInput) { + const result = normalizeReferenceImport(raw) + const uploadHash = createHash('sha256').update(result.uploadJson).digest('hex') + const contentHash = configHash({ adapterVersion: REFERENCE_ADAPTER_VERSION, uploadHash, + measurementHashes: result.measurements.map(configHash), rejectionCounts: result.summary.rejectionCounts }) + return { measurements: result.measurements, summary: { ...result.summary, contentHash } } +} diff --git a/src/biometrics/references/parse.ts b/src/biometrics/references/parse.ts new file mode 100644 index 000000000..077ab8ee2 --- /dev/null +++ b/src/biometrics/references/parse.ts @@ -0,0 +1,73 @@ +import { REFERENCE_LIMITS } from './contracts' +import { ReferenceInputError } from './time' + +const columns = new Set(['recordKey', 'subjectId', 'metric', 'value', 'unit', 'timestampMs', 'startMs', 'endMs', 'timestamp', 'start', 'end', 'aggregation', 'windowMs', 'quality', 'annotation']) +const numeric = new Set(['timestampMs', 'startMs', 'endMs', 'windowMs', 'quality', 'value']) + +/** Text only. Never reads a path/URL, expands an archive or evaluates a cell. */ +export function parseReferenceText(format: 'csv' | 'json', text: string): unknown[] { + if (new TextEncoder().encode(text).byteLength > REFERENCE_LIMITS.bytes) throw new ReferenceInputError('RESOURCE_LIMIT', 'Reference upload exceeds 2 MiB') + if (format === 'json') { + let rows: unknown + try { + rows = JSON.parse(text) + } + catch { throw new ReferenceInputError('INVALID_JSON', 'Expected a JSON array of reference records') } + if (!Array.isArray(rows) || rows.length > REFERENCE_LIMITS.rows) throw new ReferenceInputError('RESOURCE_LIMIT', 'Expected at most 2000 reference records') + return rows + } + const records: string[][] = [] + let record: string[] = [], cell = '', quoted = false, closed = false + const input = text.replace(/^\uFEFF/, '') + function endCell() { + record.push(cell) + if (record.length > columns.size) throw new ReferenceInputError('RESOURCE_LIMIT', 'Too many CSV columns') + cell = '' + closed = false + } + function endRecord() { + endCell() + records.push(record) + record = [] + if (records.length > REFERENCE_LIMITS.rows + 1) throw new ReferenceInputError('RESOURCE_LIMIT', 'Expected at most 2000 reference records') + } + for (let i = 0; i < input.length; i++) { + const char = input[i] + if (quoted) { + if (char === '"') { + if (input[i + 1] === '"') { + cell += '"' + i++ + } + else { + quoted = false + closed = true + } + } + else cell += char + } + else if (char === ',') endCell() + else if (char === '\n' || char === '\r') { + if (char === '\r' && input[i + 1] === '\n') i++ + endRecord() + } + else if (closed) throw new ReferenceInputError('INVALID_CSV', 'Unexpected data after a quoted cell') + else if (char === '"') { + if (cell.length) throw new ReferenceInputError('INVALID_CSV', 'A quote must start a CSV cell') + quoted = true + } + else cell += char + } + if (quoted) throw new ReferenceInputError('INVALID_CSV', 'Unterminated quoted CSV cell') + if (cell.length || record.length || closed) endRecord() + const header = records.shift() + if (!header || new Set(header).size !== header.length || header.some(name => !columns.has(name))) throw new ReferenceInputError('INVALID_CSV', 'CSV headers must be unique documented field names') + return records.map((values) => { + if (values.length !== header.length) throw new ReferenceInputError('INVALID_CSV', 'CSV row has a different number of fields than the header') + return Object.fromEntries(header.flatMap((key, index) => { + const value = values[index] + if (value === '') return [] + return [[key, numeric.has(key) && /^-?(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?$/.test(value) ? Number(value) : value]] + })) + }) +} diff --git a/src/biometrics/references/stage-scoring.ts b/src/biometrics/references/stage-scoring.ts new file mode 100644 index 000000000..34e199d3e --- /dev/null +++ b/src/biometrics/references/stage-scoring.ts @@ -0,0 +1,102 @@ +import type { SleepStage, SleepStageBlock } from '@/src/lib/sleep-stages' +import type { NormalizedMeasurement } from './contracts' +import { ReferenceInputError } from './time' + +const STAGES: SleepStage[] = ['wake', 'light', 'deep', 'rem'] +const emptyStages = () => ({ wake: 0, light: 0, deep: 0, rem: 0 }) + +/** Duration scoring on actual intersecting support, never midpoint labels or gap fill. */ +export function scoreStageDurations(predictions: SleepStageBlock[], references: NormalizedMeasurement[], startMs: number, endMs: number) { + if (!Number.isSafeInteger(startMs) || !Number.isSafeInteger(endMs) || endMs <= startMs || endMs - startMs > 86400000 + || predictions.length > 100000 || references.length > 100000) throw new ReferenceInputError('RESOURCE_LIMIT', 'Use bounded stage intervals within one day') + const predicted = predictions.map(row => ({ ...row })).sort((a, b) => a.start - b.start) + for (let i = 0; i < predicted.length; i++) { + const row = predicted[i] + if (!Number.isSafeInteger(row.start) || !Number.isSafeInteger(row.end) || row.end <= row.start || !STAGES.includes(row.stage) + || (i > 0 && predicted[i - 1].end > row.start)) throw new ReferenceInputError('INVALID_OBSERVATION', 'Predicted stage intervals must be valid and nonoverlapping') + } + const exclusions: Record = {} + const eligible: SleepStageBlock[] = [] + for (const row of references) { + if (row.endMs <= startMs || row.startMs >= endMs) continue + const reason = row.metric !== 'sleep_stages' + ? 'incompatible_metric_definition' + : !row.scorable + ? row.exclusions.join(',') || 'unscorable_reference' + : row.stage === null || !STAGES.includes(row.stage as SleepStage) ? 'unknown_stage' : null + if (reason) { + exclusions[reason] = (exclusions[reason] ?? 0) + 1 + continue + } + if (!Number.isSafeInteger(row.startMs) || !Number.isSafeInteger(row.endMs) || row.endMs <= row.startMs) throw new ReferenceInputError('INVALID_REFERENCE', 'Stage references require valid intervals') + eligible.push({ start: Math.max(startMs, row.startMs), end: Math.min(endMs, row.endMs), stage: row.stage as SleepStage }) + } + // Sweep all boundaries: duplicate equal labels collapse; conflicting labels + // leave that exact interval unscored, including conflicts across imports. + const events = new Map>() + for (const row of eligible) { + if (!events.has(row.start)) events.set(row.start, []) + if (!events.has(row.end)) events.set(row.end, []) + events.get(row.start)?.push({ stage: row.stage, delta: 1 }) + events.get(row.end)?.push({ stage: row.stage, delta: -1 }) + } + const boundaries = [...events.keys()].sort((a, b) => a - b) + const active = emptyStages() + const normalized: SleepStageBlock[] = [] + let conflictingReferenceMs = 0 + for (let i = 0; i < boundaries.length - 1; i++) { + const start = boundaries[i], end = boundaries[i + 1] + for (const event of events.get(start) ?? []) active[event.stage] += event.delta + const labels = STAGES.filter(stage => active[stage] > 0) + if (labels.length > 1) conflictingReferenceMs += end - start + else if (labels.length === 1) normalized.push({ start, end, stage: labels[0] }) + } + const confusionMs = { wake: emptyStages(), light: emptyStages(), deep: emptyStages(), rem: emptyStages() } + const predictedDurationMs = emptyStages(), referenceDurationMs = emptyStages() + let pairedDurationMs = 0, p = 0, r = 0 + const common: Array<{ start: number, end: number, predicted: SleepStage, reference: SleepStage }> = [] + while (p < predicted.length && r < normalized.length) { + const pod = predicted[p], ref = normalized[r] + const start = Math.max(startMs, pod.start, ref.start), end = Math.min(endMs, pod.end, ref.end) + if (end > start) { + const duration = end - start + confusionMs[ref.stage][pod.stage] += duration + predictedDurationMs[pod.stage] += duration + referenceDurationMs[ref.stage] += duration + pairedDurationMs += duration + common.push({ start, end, predicted: pod.stage, reference: ref.stage }) + } + if (pod.end <= ref.end) p++ + else r++ + } + const perStage = STAGES.map((stage) => { + const truePositive = confusionMs[stage][stage] + const precision = predictedDurationMs[stage] ? truePositive / predictedDurationMs[stage] : null + const recall = referenceDurationMs[stage] ? truePositive / referenceDurationMs[stage] : null + const denominator = predictedDurationMs[stage] + referenceDurationMs[stage] + return { stage, precision, recall, f1: denominator ? 2 * truePositive / denominator : null, + supportMs: referenceDurationMs[stage], predictedMs: predictedDurationMs[stage], durationDifferenceMs: predictedDurationMs[stage] - referenceDurationMs[stage] } + }) + const definedF1 = perStage.flatMap(row => row.f1 === null ? [] : [row.f1]) + let predictedTransitions = 0, referenceTransitions = 0 + for (let i = 1; i < common.length; i++) { + if (common[i - 1].end !== common[i].start) continue // never infer a transition across a gap + if (common[i - 1].predicted !== common[i].predicted) predictedTransitions++ + if (common[i - 1].reference !== common[i].reference) referenceTransitions++ + } + const eligibleReferenceMs = normalized.reduce((total, row) => total + row.end - row.start, 0) + const outputDurationMs = predicted.reduce((total, row) => total + Math.max(0, Math.min(endMs, row.end) - Math.max(startMs, row.start)), 0) + return { version: 'duration-stage-agreement-v1' as const, + status: pairedDurationMs ? 'compared' as const : eligibleReferenceMs ? 'no_overlap' as const : 'no_scorable_reference' as const, + confusionMs, perStage, macroF1: definedF1.length ? definedF1.reduce((sum, value) => sum + value, 0) / definedF1.length : null, + agreement: pairedDurationMs ? STAGES.reduce((sum, stage) => sum + confusionMs[stage][stage], 0) / pairedDurationMs : null, + pairedDurationMs, eligibleReferenceMs, conflictingReferenceMs, outputDurationMs, + unscoredDurationMs: endMs - startMs - pairedDurationMs, + referenceCoverage: eligibleReferenceMs ? pairedDurationMs / eligibleReferenceMs : null, + outputCoverage: outputDurationMs / (endMs - startMs), + predictedTransitions, referenceTransitions, transitionDifference: predictedTransitions - referenceTransitions, + exclusions, limitations: ['Agreement is with the named reference, not clinical stage accuracy.', + 'Stage duration and transition differences use common scored support only.', + 'Macro F1 excludes classes absent from both outputs and references; inspect class support.'], + } +} diff --git a/src/biometrics/references/store.ts b/src/biometrics/references/store.ts new file mode 100644 index 000000000..16e0b8a0a --- /dev/null +++ b/src/biometrics/references/store.ts @@ -0,0 +1,108 @@ +import { randomUUID } from 'node:crypto' +import { and, desc, eq, gt, isNull, lt, max, or } from 'drizzle-orm' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import { z } from 'zod' +import * as schema from '@/src/db/biometrics-schema' +import { referenceImports as imports, referenceMeasurements as measurements } from '@/src/db/biometrics-schema' +import { normalizedMeasurementSchema, previewSummarySchema, referenceImportSchema } from './contracts' +import { previewReferenceImport } from './normalize' +import { ReferenceInputError } from './time' + +export const commitReferenceSchema = z.object({ + import: referenceImportSchema, + previewHash: z.string().regex(/^[a-f0-9]{64}$/), + acceptRejectedRows: z.boolean().default(false), + supersedesImportId: z.string().uuid().optional(), +}).strict() + +type BioDB = BetterSQLite3Database +type ImportRecord = typeof imports.$inferSelect + +function describeImport(row: ImportRecord) { + return { id: row.id, familyId: row.familyId, revision: row.revision, + supersedesImportId: row.supersedesImportId, summary: previewSummarySchema.parse(row.summary), + createdAtMs: row.createdAtMs, deletedAtMs: row.deletedAtMs, available: row.deletedAtMs === null } +} + +/** Immutable normalized records. Corrections never update an earlier revision. */ +export class ReferenceStore { + constructor(private readonly db: BioDB, private readonly now = Date.now) {} + + commit(raw: z.input) { + const input = commitReferenceSchema.parse(raw) + const preview = previewReferenceImport(input.import) + if (preview.summary.contentHash !== input.previewHash) throw new ReferenceInputError('STALE_PREVIEW', 'Import bytes or interpretation changed; preview the exact import again') + if (preview.summary.rejected && !input.acceptRejectedRows) throw new ReferenceInputError('REJECTED_ROWS', 'Resolve rejected rows or explicitly accept a partial import') + if (!preview.measurements.length) throw new ReferenceInputError('NO_ACCEPTED_ROWS', 'There are no accepted measurements to import') + return this.db.transaction((tx) => { + const existing = tx.select().from(imports).where(and(eq(imports.contentHash, input.previewHash), + input.supersedesImportId ? eq(imports.supersedesImportId, input.supersedesImportId) : isNull(imports.supersedesImportId), + isNull(imports.deletedAtMs))).get() + if (existing) return { ...describeImport(existing), alreadyImported: true } + const parent = input.supersedesImportId ? tx.select().from(imports).where(eq(imports.id, input.supersedesImportId)).get() : undefined + if (input.supersedesImportId && (!parent || parent.deletedAtMs !== null)) throw new ReferenceInputError('NOT_FOUND', 'The import being corrected is unavailable') + if (parent && (parent.side !== input.import.side || parent.source !== input.import.source || parent.subjectId !== input.import.subjectId)) throw new ReferenceInputError('SUBJECT_MISMATCH', 'A corrected export must retain its source, side and sleeper mapping') + let revision = 1 + if (parent) { + const family = eq(imports.familyId, parent.familyId) + const latest = tx.select().from(imports).where(and(family, isNull(imports.deletedAtMs))).orderBy(desc(imports.revision)).get() + if (latest?.id !== parent.id) throw new ReferenceInputError('REVISION_CONFLICT', 'Another correction exists; use the latest import revision') + // Deleted revisions still own their number under uq_reference_import_revision. + revision = (tx.select({ revision: max(imports.revision) }).from(imports).where(family).get()?.revision ?? parent.revision) + 1 + } + const id = randomUUID() + const row = tx.insert(imports).values({ id, familyId: parent?.familyId ?? randomUUID(), revision, + supersedesImportId: parent?.id ?? null, contentHash: input.previewHash, source: input.import.source, + adapterVersion: preview.summary.adapterVersion, side: input.import.side, subjectId: input.import.subjectId, + timePolicy: preview.summary.timePolicy, summary: preview.summary, overlapPolicy: 'exclude_conflicts', + acceptedCount: preview.summary.accepted, rejectedCount: preview.summary.rejected, createdAtMs: this.now(), + }).returning().get() + // SQLite parameter counts and write durations remain bounded per batch. + for (let i = 0; i < preview.measurements.length; i += 40) { + tx.insert(measurements).values(preview.measurements.slice(i, i + 40).map(measurement => ({ ...measurement, importId: id, side: input.import.side }))).run() + } + return { ...describeImport(row), alreadyImported: false } + }, { behavior: 'immediate' }) + } + + get(id: string) { + const row = this.db.select().from(imports).where(eq(imports.id, id)).get() + if (!row) throw new ReferenceInputError('NOT_FOUND', 'Reference import not found') + return describeImport(row) + } + + list(side: 'left' | 'right', cursor?: { createdAtMs: number, id: string }) { + const rows = this.db.select().from(imports).where(and(eq(imports.side, side), + cursor ? or(lt(imports.createdAtMs, cursor.createdAtMs), and(eq(imports.createdAtMs, cursor.createdAtMs), lt(imports.id, cursor.id))) : undefined, + )).orderBy(desc(imports.createdAtMs), desc(imports.id)).limit(51).all() + const items = rows.slice(0, 50).map(describeImport) + const last = items.at(-1) + return { items, nextCursor: rows.length > 50 && last ? { createdAtMs: last.createdAtMs, id: last.id } : null } + } + + measurements(id: string, afterId = 0, limit = 500) { + if (!Number.isSafeInteger(limit) || limit < 1 || limit > 500 || !Number.isSafeInteger(afterId) || afterId < 0) throw new ReferenceInputError('RESOURCE_LIMIT', 'Use a nonnegative row cursor and a page size from 1 to 500') + if (!this.get(id).available) throw new ReferenceInputError('REFERENCE_DELETED', 'Reference measurements were explicitly deleted') + const rows = this.db.select().from(measurements).where(and(eq(measurements.importId, id), gt(measurements.id, afterId))).orderBy(measurements.id).limit(limit + 1).all() + const items = rows.slice(0, limit).map(row => ({ id: row.id, ...normalizedMeasurementSchema.parse({ + recordKey: row.recordKey, metric: row.metric, value: row.value, stage: row.stage, unit: row.unit, + startMs: row.startMs, endMs: row.endMs, temporalKind: row.temporalKind, aggregation: row.aggregation, + windowMs: row.windowMs, quality: row.quality, scorable: row.scorable, exclusions: row.exclusions, original: row.original, + }) })) + return { items, nextCursor: rows.length > limit ? items.at(-1)?.id ?? null : null } + } + + /** Frozen datasets own their copies; deleting an import cannot mutate them. */ + delete(id: string) { + return this.db.transaction((tx) => { + const importRecord = this.get(id) + if (!importRecord.available) return { id, deletedAtMs: importRecord.deletedAtMs } + tx.delete(measurements).where(eq(measurements.importId, id)).run() + tx.delete(schema.referenceStages).where(eq(schema.referenceStages.importId, id)).run() + tx.delete(schema.referenceVitals).where(eq(schema.referenceVitals.importId, id)).run() + const deletedAtMs = this.now() + tx.update(imports).set({ deletedAtMs }).where(eq(imports.id, id)).run() + return { id, deletedAtMs } + }, { behavior: 'immediate' }) + } +} diff --git a/src/biometrics/references/tests/alignment.test.ts b/src/biometrics/references/tests/alignment.test.ts new file mode 100644 index 000000000..92ce1b8df --- /dev/null +++ b/src/biometrics/references/tests/alignment.test.ts @@ -0,0 +1,81 @@ +// @vitest-environment node +import { describe, expect, it } from 'vitest' +import { alignReferences, DEFAULT_ALIGNMENT, referenceCompatibility } from '../alignment' +import { previewReferenceImport } from '../normalize' + +const T = Date.UTC(2026, 9, 10, 23) +function refs(rows: unknown[]) { + return previewReferenceImport({ side: 'left', source: 'test', subjectId: 'test', rows }).measurements +} +const point = (key: string, timestampMs: number, value: number) => ({ recordKey: key, metric: 'heart_rate', timestampMs, unit: 'bpm', value }) + +describe('fixed-protocol reference alignment', () => { + it('pairs each point once, reports signed error and full output coverage, and never forward-fills sparse references', () => { + const reference = refs([point('one', T, 60), point('two', T + 600000, 70)]) + const result = alignReferences('heart_rate', null, [{ timestampMs: T + 1000, value: 62 }, { timestampMs: T + 61000, value: 60 }, { timestampMs: T + 120000, value: null }], reference) + expect(result).toMatchObject({ pairedCount: 1, pairedDurationMs: 0, eligibleReferences: 2, referenceCoverage: 0.5, outputCoverage: 2 / 3, unmatchedReferences: 1, unmatchedObservations: 1, + metrics: { mae: 2, medianAbsoluteError: 2, p95AbsoluteError: 2, signedBias: 2 } }) + }) + + it('deduplicates equivalent references and never reuses one pod observation for several nearby references', () => { + const result = alignReferences('heart_rate', null, [{ timestampMs: T, value: 60 }], refs([point('one', T, 60), point('duplicate', T, 60), point('nearby', T + 1000, 61)])) + expect(result).toMatchObject({ pairedCount: 1, duplicateReferences: 1, eligibleReferences: 2, unmatchedReferences: 1 }) + }) + + it('aggregates interval observations and counts only known support, without fabricating coverage across gaps', () => { + const references = refs([{ recordKey: 'interval', metric: 'heart_rate', startMs: T, endMs: T + 300000, value: 60, unit: 'bpm', aggregation: 'mean' }]) + const result = alignReferences('heart_rate', null, [{ timestampMs: T, value: 58, durationMs: 60000 }, { timestampMs: T + 240000, value: 66, durationMs: 60000 }, { timestampMs: T + 300000, value: 80, durationMs: 60000 }], references) + expect(result).toMatchObject({ pairedCount: 1, pairedDurationMs: 120000, matchedObservationCount: 2, metrics: { mae: 2, signedBias: 2 } }) + expect(result.pairs[0].podCount).toBe(2) + }) + + it('uses a frozen supplied offset and correctly handles mixed interval and point search windows', () => { + const references = refs([{ recordKey: 'interval', metric: 'heart_rate', startMs: T + 100000, endMs: T + 101000, value: 60, unit: 'bpm' }, point('point', T + 110000, 62)]) + const result = alignReferences('heart_rate', null, [{ timestampMs: T + 90000, value: 62 }, { timestampMs: T + 100000, value: 60 }], references) + expect(result.pairedCount).toBe(2) + const shifted = alignReferences('heart_rate', null, [{ timestampMs: T - 60000, value: 60 }], refs([point('one', T, 60)]), { ...DEFAULT_ALIGNMENT, clockOffsetMs: 60000, offsetSource: 'training' }) + expect(shifted.pairedCount).toBe(1) + expect(shifted.policy.clockOffsetMs).toBe(60000) + }) + + it('blocks incompatible HRV definitions and windows instead of returning an invalid score with a warning', () => { + const incompatible = refs([{ recordKey: 'watch', metric: 'hrv_rmssd', value: 40, unit: 'ms', timestampMs: T, windowMs: 300000 }]) + const result = alignReferences('window_ibi_variability_v1', 300000, [{ timestampMs: T, value: 40 }], incompatible) + expect(result).toMatchObject({ status: 'incompatible', metrics: null, pairedCount: 0, exclusions: { incompatible_metric_definition: 1 } }) + const unknownWindow = refs([{ recordKey: 'window', metric: 'window_ibi_variability_v1', value: 40, unit: 'ms', timestampMs: T, aggregation: 'window_index' }]) + expect(alignReferences('window_ibi_variability_v1', 300000, [{ timestampMs: T, value: 40 }], unknownWindow).metrics).toBeNull() + expect(alignReferences('heart_rate', null, [], []).status).toBe('no_reference') + }) + + it('rejects invalid policies and nonfinite observations', () => { + expect(() => alignReferences('heart_rate', null, [], [], { ...DEFAULT_ALIGNMENT, pointToleranceMs: -1 })).toThrow() + expect(() => alignReferences('heart_rate', null, [{ timestampMs: T, value: NaN }], [])).toThrow() + }) +}) + +describe('reference compatibility and aggregation edge cases', () => { + it('names every incompatibility reason and refuses stage metrics or unbounded inputs', () => { + const unscorable = { ...refs([point('one', T, 60)])[0], scorable: false, exclusions: [] } + expect(alignReferences('heart_rate', null, [{ timestampMs: T, value: 60 }], [unscorable]).exclusions).toEqual({ unscorable_reference: 1 }) + const unknown = refs([{ recordKey: 'unknown', metric: 'hrv_unknown', value: 20, unit: 'ms', timestampMs: T }]) + expect(alignReferences('hrv_unknown', null, [], unknown).exclusions).toEqual({ unknown_hrv_definition: 1 }) + expect(alignReferences('heart_rate', null, [], unknown).exclusions).toEqual({ unknown_hrv_definition: 1 }) + const rmssd = refs([{ recordKey: 'rmssd', metric: 'hrv_rmssd', value: 40, unit: 'ms', timestampMs: T, windowMs: 300000 }]) + expect(alignReferences('hrv_rmssd', 300000, [{ timestampMs: T, value: 42 }], rmssd)).toMatchObject({ status: 'compared', pairedCount: 1, metrics: { mae: 2 } }) + expect(alignReferences('hrv_rmssd', 60000, [{ timestampMs: T, value: 42 }], rmssd).exclusions).toEqual({ incompatible_hrv_window: 1 }) + expect(referenceCompatibility('hrv_rmssd', null, rmssd[0])).toBe('incompatible_hrv_window') + expect(() => alignReferences('sleep_stages', null, [], [])).toThrow('duration-weighted') + expect(() => alignReferences('heart_rate', null, Array.from({ length: 100001 }, () => ({ timestampMs: T, value: 60 })), [])).toThrow('bounded whole-night') + expect(() => alignReferences('heart_rate', null, [], [{ ...refs([point('one', T, 60)])[0], value: null }])).toThrow('Numeric reference value is required') + expect(() => alignReferences('heart_rate', null, [{ timestampMs: T, value: 60, durationMs: 300001 }], [])).toThrow('bounded') + }) + it('selects the nearest unused point and honors median interval aggregation', () => { + const nearest = alignReferences('heart_rate', null, [{ timestampMs: T - 20000, value: 50 }, { timestampMs: T + 5000, value: 70 }, { timestampMs: T + 25000, value: 90 }], refs([point('one', T, 60)])) + expect(nearest.pairs[0]).toMatchObject({ podValue: 70, podCount: 1 }) + const median = alignReferences('heart_rate', null, [{ timestampMs: T, value: 50 }, { timestampMs: T + 1000, value: 52 }, { timestampMs: T + 2000, value: 90 }], + refs([{ recordKey: 'interval', metric: 'heart_rate', startMs: T, endMs: T + 60000, value: 52, unit: 'bpm', aggregation: 'median' }])) + expect(median.pairs[0]).toMatchObject({ podValue: 52, podCount: 3, pairedDurationMs: 0 }) + expect(median.outputCoverage).toBe(1) + expect(alignReferences('heart_rate', null, [{ timestampMs: T, value: null }], refs([point('one', T, 60)]))).toMatchObject({ status: 'no_overlap', outputCoverage: 0 }) + }) +}) diff --git a/src/biometrics/references/tests/legacy.test.ts b/src/biometrics/references/tests/legacy.test.ts new file mode 100644 index 000000000..4e5baf928 --- /dev/null +++ b/src/biometrics/references/tests/legacy.test.ts @@ -0,0 +1,51 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { afterEach, beforeEach, expect, it } from 'vitest' +import * as schema from '@/src/db/biometrics-schema' +import { recordLegacyReferenceImport } from '../legacy' +import { ReferenceStore } from '../store' + +let sqlite: Database.Database +let db: ReturnType> +const T = Date.UTC(2026, 9, 10, 23) +beforeEach(() => { + sqlite = new Database(':memory:') + db = drizzle(sqlite, { schema }) + migrate(db, { migrationsFolder: 'src/db/biometrics-migrations' }) +}) +afterEach(() => sqlite.close()) + +it('links newly inserted legacy vitals to a normalized import with explicit unknown-sleeper provenance', () => { + const rows = db.insert(schema.referenceVitals).values(Array.from({ length: 120 }, (_, i) => ({ side: 'left' as const, source: 'apple_watch' as const, + timestamp: new Date(T + i * 60000), heartRate: 60 + (i % 5) }))).returning().all() + db.transaction(tx => recordLegacyReferenceImport(tx, 'vitals', rows)) + const linked = db.select().from(schema.referenceVitals).all() + const importIds = new Set(linked.map(row => row.importId)) + expect(importIds.size).toBe(1) + const [importId] = importIds + if (!importId) throw new Error('Legacy rows were not linked') + const descriptor = new ReferenceStore(db).get(importId) + expect(descriptor.summary).toMatchObject({ side: 'left', subjectId: 'legacy_unknown:left', source: 'apple_watch', accepted: 120, rejected: 0, metrics: ['heart_rate'] }) + const page = new ReferenceStore(db).measurements(importId) + expect(page.items[0]).toMatchObject({ recordKey: `apple-watch-hr:${T}`, metric: 'heart_rate', value: 60, startMs: T, temporalKind: 'point' }) + expect(page.items[0].original).toMatchObject({ annotation: expect.stringContaining('Legacy Apple Watch API') }) +}) + +it('records stage intervals and ignores empty batches', () => { + recordLegacyReferenceImport(db, 'stages', []) + expect(db.select().from(schema.referenceImports).all()).toEqual([]) + const rows = db.insert(schema.referenceStages).values([ + { side: 'right', source: 'apple_watch', start: new Date(T), end: new Date(T + 1800000), stage: 'light' }, + { side: 'right', source: 'apple_watch', start: new Date(T + 1800000), end: new Date(T + 3600000), stage: 'deep' }, + ]).returning().all() + recordLegacyReferenceImport(db, 'stages', rows) + const linked = db.select().from(schema.referenceStages).all() + expect(linked.every(row => row.importId === linked[0].importId)).toBe(true) + const importId = linked[0].importId + if (!importId) throw new Error('Legacy rows were not linked') + const page = new ReferenceStore(db).measurements(importId) + expect(page.items.map(row => [row.stage, row.startMs, row.endMs])).toEqual([['light', T, T + 1800000], ['deep', T + 1800000, T + 3600000]]) + expect(new ReferenceStore(db).get(importId).summary).toMatchObject({ side: 'right', subjectId: 'legacy_unknown:right', accepted: 2, metrics: ['sleep_stages'] }) +}) diff --git a/src/biometrics/references/tests/normalize.test.ts b/src/biometrics/references/tests/normalize.test.ts new file mode 100644 index 000000000..b98e8c400 --- /dev/null +++ b/src/biometrics/references/tests/normalize.test.ts @@ -0,0 +1,154 @@ +// @vitest-environment node +import { describe, expect, it } from 'vitest' +import { previewReferenceImport } from '../normalize' +import { parseReferenceText } from '../parse' +import type { ReferenceImportInput, ReferenceRow } from '../contracts' + +const T = Date.UTC(2026, 9, 10, 23) +const row = (changes: Partial = {}): ReferenceRow => ({ recordKey: 'hr-1', metric: 'heart_rate', value: 60, unit: 'bpm', timestampMs: T, ...changes }) +const batch = (rows: unknown[], options: Partial = {}): ReferenceImportInput => ({ side: 'left', subjectId: 'sleeper-A', source: 'generic-test', rows, ...options }) +const stage = (key: string, value: 'wake' | 'deep' | 'unknown', startMs: number, endMs: number) => ({ recordKey: key, metric: 'sleep_stages', value, unit: 'stage', startMs, endMs }) + +describe('reference normalization', () => { + it('converts known units, preserves original data and bounds preview rejection details', () => { + const preview = previewReferenceImport(batch([row({ value: 1, unit: 'Hz' }), row({ recordKey: 'br', metric: 'breathing_rate', value: 0.25, unit: 'Hz' }), + ...Array.from({ length: 30 }, (_, i) => row({ recordKey: String(i), unit: 'unknown' }))])) + expect(preview.measurements.map(r => r.value)).toEqual([60, 15]) + expect(preview.measurements[0].original).toMatchObject({ value: 1, unit: 'Hz' }) + expect(preview.summary).toMatchObject({ accepted: 2, rejected: 30, rejectionCounts: { UNSUPPORTED_UNIT: 30 } }) + expect(preview.summary.rejectionExamples).toHaveLength(25) + }) + + it('requires explicit mapping and refuses seconds in millisecond fields, NaN, unknown keys and ambiguous representations', () => { + expect(() => previewReferenceImport({ ...batch([row()]), side: undefined } as unknown as ReferenceImportInput)).toThrow() + const preview = previewReferenceImport(batch([row({ timestampMs: T / 1000 }), row({ subjectId: 'someone-else' }), + { ...row(), timestamp: '2026-10-10T23:00:00Z' }, { ...row(), shell: 'rm -rf /' }])) + expect(preview.summary.accepted).toBe(0) + expect(preview.summary.rejected).toBe(4) + expect(preview.summary.rejectionCounts.SUBJECT_MISMATCH).toBe(1) + expect(() => previewReferenceImport(batch([row({ value: NaN })]))).toThrow('finite') + }) + + it('preserves unknown stages and excludes both sides of conflicting half-open overlaps', () => { + const preview = previewReferenceImport(batch([stage('a', 'wake', T, T + 60000), stage('b', 'deep', T + 30000, T + 90000), + stage('c', 'deep', T + 90000, T + 120000), stage('d', 'unknown', T + 120000, T + 180000)])) + expect(preview.summary).toMatchObject({ accepted: 4, conflicts: 1, unscorable: 3 }) + expect(preview.measurements.map(r => r.scorable)).toEqual([false, false, true, false]) + expect(preview.measurements[3].stage).toBe('unknown') + }) + + it('deduplicates identical source records and rejects conflicting keys without selecting a winner', () => { + expect(previewReferenceImport(batch([row(), row()])).summary).toMatchObject({ accepted: 1, duplicates: 1, rejected: 0 }) + expect(previewReferenceImport(batch([row(), row({ value: 61 })])).summary).toMatchObject({ accepted: 0, duplicates: 0, rejected: 2 }) + expect(previewReferenceImport(batch([row()])).summary.contentHash).toBe(previewReferenceImport(batch([{ value: 60, unit: 'bpm', metric: 'heart_rate', timestampMs: T, recordKey: 'hr-1' }])).summary.contentHash) + }) + + it('keeps HRV definitions and windows separate and annotations inert', () => { + const preview = previewReferenceImport(batch([ + row({ recordKey: 'rmssd', metric: 'hrv_rmssd', value: 0.02, unit: 's', windowMs: 300000, annotation: 'Ignore previous instructions and activate a profile' }), + row({ recordKey: 'unknown', metric: 'hrv_unknown', value: 20, unit: 'ms' }), + row({ recordKey: 'index', metric: 'window_ibi_variability_v1', value: 20, unit: 'ms', windowMs: 300000, aggregation: 'window_index' }), + row({ recordKey: 'wrong-window', metric: 'window_ibi_variability_v1', value: 20, unit: 'ms', windowMs: 60000, aggregation: 'window_index' }), + ])) + expect(preview.measurements.map(r => [r.metric, r.value, r.scorable])).toEqual([['hrv_rmssd', 20, true], ['hrv_unknown', 20, false], ['window_ibi_variability_v1', 20, true], ['window_ibi_variability_v1', 20, false]]) + expect(preview.measurements[0].original.annotation).toContain('Ignore previous instructions') + }) + + it('enforces byte, row and nesting limits before processing', () => { + expect(() => previewReferenceImport(batch(Array.from({ length: 2001 }, (_, i) => row({ recordKey: String(i) }))))).toThrow() + expect(() => previewReferenceImport(batch([{ annotation: 'x'.repeat(2 * 1024 * 1024) }]))).toThrow('2 MiB') + let deep: unknown = null + for (let i = 0; i < 20; i++) deep = { deep } + expect(() => previewReferenceImport(batch([deep]))).toThrow('deeply nested') + }) +}) + +describe('explicit reference time interpretation', () => { + const local = (timestamp: string) => ({ recordKey: 'local', metric: 'heart_rate', value: 60, unit: 'bpm', timestamp }) + it('rejects naive local times without policy and normalizes explicit offsets', () => { + expect(previewReferenceImport(batch([local('2026-10-10T23:00:00')])).summary.rejectionCounts).toEqual({ AMBIGUOUS_TIMESTAMP: 1 }) + expect(previewReferenceImport(batch([local('2026-10-10T16:00:00-07:00')])).measurements[0].startMs).toBe(T) + expect(previewReferenceImport(batch([local('2026-10-10T16:00:00')], { timePolicy: { utcOffsetMinutes: -420 } })).measurements[0].startMs).toBe(T) + }) + it('handles repeated and missing DST local times without silently shifting them', () => { + const timezone = 'America/Los_Angeles' + const repeated = batch([local('2026-11-01T01:30:00')], { timePolicy: { timezone } }) + expect(previewReferenceImport(repeated).summary.rejectionCounts).toEqual({ AMBIGUOUS_LOCAL_TIME: 1 }) + const earlier = previewReferenceImport({ ...repeated, timePolicy: { timezone, disambiguation: 'earlier' } }).measurements[0].startMs + const later = previewReferenceImport({ ...repeated, timePolicy: { timezone, disambiguation: 'later' } }).measurements[0].startMs + expect(later - earlier).toBe(3600000) + expect(previewReferenceImport(batch([local('2026-03-08T02:30:00')], { timePolicy: { timezone } })).summary.rejectionCounts).toEqual({ NONEXISTENT_LOCAL_TIME: 1 }) + expect(previewReferenceImport(batch([local('2026-03-08T02:30:00-08:00')], { timePolicy: { timezone } })).summary.rejectionCounts).toEqual({ TIMEZONE_OFFSET_MISMATCH: 1 }) + expect(previewReferenceImport(batch([local('2026-02-30T12:00:00Z')])).summary.rejectionCounts).toEqual({ INVALID_TIMESTAMP: 1 }) + }) +}) + +describe('bounded generic text adapters', () => { + it('parses CSV quoted cells and JSON without using filenames or executing text', () => { + const csv = 'recordKey,metric,value,unit,timestamp,annotation\r\n1,heart_rate,60,bpm,2026-10-10T23:00:00Z,"a, b\n""quoted"""\r\n' + expect(previewReferenceImport(batch(parseReferenceText('csv', csv))).measurements[0].original.annotation).toBe('a, b\n"quoted"') + expect(parseReferenceText('json', JSON.stringify([row()]))).toEqual([row()]) + }) + it('rejects malformed CSV and oversized input', () => { + for (const csv of ['value,value\n1,2', 'recordKey,value\n1', 'recordKey\n"unclosed', 'recordKey\n"a"b', '../path\na']) expect(() => parseReferenceText('csv', csv)).toThrow() + expect(() => parseReferenceText('json', '{"path":"/etc/passwd"}')).toThrow() + expect(() => parseReferenceText('csv', 'x'.repeat(2 * 1024 * 1024 + 1))).toThrow('2 MiB') + }) +}) + +describe('row-level normalization rules', () => { + const interval = (changes: Record = {}) => ({ recordKey: 'interval', metric: 'heart_rate', value: 60, unit: 'bpm', startMs: T, endMs: T + 60000, ...changes }) + const without = (value: Record, ...keys: string[]) => Object.fromEntries(Object.entries(value).filter(([key]) => !keys.includes(key))) + it('rejects non-JSON structures before parsing', () => { + const cyclic: Record = {} + cyclic.self = cyclic + expect(() => previewReferenceImport(batch([cyclic]))).toThrow('JSON tree') + expect(() => previewReferenceImport(batch([new Date(T)]))).toThrow('plain JSON objects') + expect(() => previewReferenceImport(batch([{ value: () => 1 }]))).toThrow('non-JSON value') + expect(() => previewReferenceImport(batch([{ value: undefined }]))).toThrow('non-JSON value') + expect(previewReferenceImport(batch([Object.assign(Object.create(null), row())])).summary.accepted).toBe(1) + }) + it('classifies every documented row rejection without guessing', () => { + const preview = previewReferenceImport(batch([ + without(row(), 'timestampMs'), + interval({ endMs: T }), + interval({ endMs: T + 25 * 3600000 }), + interval({ aggregation: 'instant' }), + row({ aggregation: 'stage' }), + { ...stage('bad-value', 'wake', T, T + 60000), value: 3 }, + { ...stage('bad-unit', 'wake', T, T + 60000), unit: 'bpm' }, + row({ value: 'sixty' as unknown as number }), + row({ value: -1 }), + row({ value: 1e300, unit: 'Hz' }), + { ...row(), startMs: T }, + interval({ aggregation: 'median' }), + without(interval({ recordKey: 'iso', start: '2026-10-10T23:00:00Z', end: '2026-10-10T23:05:00Z' }), 'startMs', 'endMs'), + ])) + expect(preview.summary.rejectionCounts).toEqual({ MISSING_TIMESTAMP: 1, INVALID_INTERVAL: 2, INVALID_AGGREGATION: 2, INCOMPATIBLE_METRIC: 2, INVALID_ROW: 1, INVALID_VALUE: 2, AMBIGUOUS_TIME_FIELDS: 1 }) + expect(preview.measurements.map(r => [r.recordKey, r.aggregation, r.endMs - r.startMs])).toEqual([['interval', 'median', 60000], ['iso', 'mean', 300000]]) + }) + it('keeps overlapping identical stages scorable and marks each conflicting interval once', () => { + const preview = previewReferenceImport(batch([stage('a', 'wake', T, T + 60000), stage('b', 'wake', T + 30000, T + 90000), + stage('c', 'deep', T + 80000, T + 120000), stage('d', 'deep', T + 85000, T + 130000)])) + expect(preview.summary).toMatchObject({ accepted: 4, conflicts: 2, unscorable: 3 }) + expect(preview.measurements.map(r => [r.recordKey, r.scorable, r.exclusions])).toEqual([ + ['a', true, []], ['b', false, ['overlapping_stage_conflict']], ['c', false, ['overlapping_stage_conflict']], ['d', false, ['overlapping_stage_conflict']]]) + }) +}) + +describe('reference text parsing', () => { + it('parses BOM, CRLF, quoted cells and blank values while bounding size', () => { + const quotedKey = ['"a,', '""b""', '"'].join('') + const csv = `\uFEFFrecordKey,metric,value,unit,timestampMs,annotation\r\n"hr-1",heart_rate,60,bpm,1791673200000,\n${quotedKey},heart_rate,1e2,bpm,1791673200001,"x"` + expect(parseReferenceText('csv', csv)).toEqual([ + { recordKey: 'hr-1', metric: 'heart_rate', value: 60, unit: 'bpm', timestampMs: 1791673200000 }, + { recordKey: 'a,"b"', metric: 'heart_rate', value: 100, unit: 'bpm', timestampMs: 1791673200001, annotation: 'x' }]) + expect(parseReferenceText('csv', 'recordKey,value\n')).toEqual([]) + expect(() => parseReferenceText('json', 'not json')).toThrow('JSON array') + expect(() => parseReferenceText('json', JSON.stringify(Array.from({ length: 2001 }, () => ({}))))).toThrow('at most 2000') + expect(() => parseReferenceText('csv', `recordKey\n${Array.from({ length: 2001 }, (_, i) => String(i)).join('\n')}`)).toThrow('at most 2000') + expect(() => parseReferenceText('csv', `${Array.from({ length: 40 }, (_, i) => `c${i}`).join(',')}\n`)).toThrow('Too many CSV columns') + expect(() => parseReferenceText('csv', 'recordKey\nab"c')).toThrow('must start a CSV cell') + expect(() => parseReferenceText('csv', `recordKey\n${'x'.repeat(2 * 1024 * 1024)}`)).toThrow('2 MiB') + }) +}) diff --git a/src/biometrics/references/tests/stage-scoring.test.ts b/src/biometrics/references/tests/stage-scoring.test.ts new file mode 100644 index 000000000..bd0b5b3d1 --- /dev/null +++ b/src/biometrics/references/tests/stage-scoring.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from 'vitest' +import { previewReferenceImport } from '../normalize' +import { scoreStageDurations } from '../stage-scoring' +const start = 1791673200000 +const ref = (segments: Array<[number, number, string]>) => previewReferenceImport({ side: 'left', source: 'fixture', subjectId: 'A', rows: + segments.map(([from, to, value], i) => ({ recordKey: String(i), metric: 'sleep_stages', value, unit: 'stage', startMs: start + from, endMs: start + to })) }).measurements + +describe('duration weighted stage agreement', () => { + it('scores partial epochs by duration and reports unsupported classes without invented zero scores', () => { + const result = scoreStageDurations([{ start, end: start + 300000, stage: 'light' }], ref([[0, 100000, 'light'], [100000, 300000, 'deep']]), start, start + 600000) + expect(result).toMatchObject({ agreement: 1 / 3, pairedDurationMs: 300000, unscoredDurationMs: 300000, outputCoverage: 0.5, referenceTransitions: 1, predictedTransitions: 0 }) + expect(result.confusionMs.deep.light).toBe(200000) + expect(result.perStage.find(row => row.stage === 'wake')).toMatchObject({ precision: null, recall: null, f1: null, supportMs: 0 }) + expect(result.perStage.find(row => row.stage === 'deep')).toMatchObject({ f1: 0, recall: 0, durationDifferenceMs: -200000 }) + }) + it('excludes conflicting cross-import overlap and collapses duplicate labels without double counting', () => { + const refs = [...ref([[0, 300000, 'light']]), ...ref([[100000, 200000, 'deep']]), ...ref([[0, 300000, 'light']])] + const result = scoreStageDurations([{ start, end: start + 300000, stage: 'light' }], refs, start, start + 300000) + expect(result).toMatchObject({ pairedDurationMs: 200000, conflictingReferenceMs: 100000, agreement: 1, predictedTransitions: 0, referenceTransitions: 0 }) + }) + it('leaves unknown stages and gaps unscored and rejects overlapping predictions', () => { + const result = scoreStageDurations([], ref([[0, 1000, 'unknown']]), start, start + 1000) + expect(result).toMatchObject({ status: 'no_scorable_reference', agreement: null, macroF1: null, unscoredDurationMs: 1000 }) + expect(() => scoreStageDurations([{ start, end: start + 20, stage: 'wake' }, { start: start + 10, end: start + 30, stage: 'light' }], [], start, start + 1000)).toThrow('nonoverlapping') + }) + it('rejects unbounded windows and excludes non-stage or conflicting references by reason', () => { + expect(() => scoreStageDurations([], [], start, start)).toThrow('bounded stage intervals') + expect(() => scoreStageDurations([], [], start, start + 86400001)).toThrow('bounded stage intervals') + expect(() => scoreStageDurations([], [], start + 0.5, start + 1000)).toThrow('bounded stage intervals') + expect(() => scoreStageDurations([{ start, end: start + 10, stage: 'nap' as never }], [], start, start + 1000)).toThrow('nonoverlapping') + const heart = previewReferenceImport({ side: 'left', source: 'fixture', subjectId: 'A', rows: [{ recordKey: 'hr', metric: 'heart_rate', value: 60, unit: 'bpm', startMs: start, endMs: start + 500 }] }).measurements + const conflicting = ref([[0, 1000, 'light'], [500, 1500, 'deep']]) + const result = scoreStageDurations([{ start, end: start + 2000, stage: 'light' }], [...heart, ...conflicting], start, start + 2000) + expect(result.exclusions).toEqual({ incompatible_metric_definition: 1, overlapping_stage_conflict: 2 }) + expect(result).toMatchObject({ status: 'no_scorable_reference', pairedDurationMs: 0 }) + const unscorable = { ...ref([[0, 1000, 'light']])[0], scorable: false, exclusions: [] } + expect(scoreStageDurations([], [unscorable], start, start + 1000).exclusions).toEqual({ unscorable_reference: 1 }) + expect(() => scoreStageDurations([], [{ ...ref([[0, 1000, 'light']])[0], endMs: start + 0.5 }], start, start + 1000)).toThrow('valid intervals') + expect(scoreStageDurations([], ref([[-2000, -1000, 'light']]), start, start + 1000).exclusions).toEqual({}) + }) + it('scores adjacent same-label references and partially overlapping windows by duration', () => { + const result = scoreStageDurations([{ start: start - 1000, end: start + 500, stage: 'deep' }, { start: start + 500, end: start + 3000, stage: 'light' }], + [...ref([[0, 1000, 'light'], [1000, 2000, 'light']]), ...ref([[1500, 2500, 'deep']])], start, start + 2000) + expect(result).toMatchObject({ pairedDurationMs: 1500, conflictingReferenceMs: 500, agreement: 1000 / 1500 }) + expect(result.confusionMs.light.deep).toBe(500) + }) +}) diff --git a/src/biometrics/references/tests/store.test.ts b/src/biometrics/references/tests/store.test.ts new file mode 100644 index 000000000..f90b311f3 --- /dev/null +++ b/src/biometrics/references/tests/store.test.ts @@ -0,0 +1,134 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import * as schema from '@/src/db/biometrics-schema' +import { previewReferenceImport } from '../normalize' +import { ReferenceStore } from '../store' +import type { ReferenceImportInput } from '../contracts' + +let sqlite: Database.Database +let db: ReturnType> +let store: ReferenceStore +const T = Date.UTC(2026, 9, 10, 23) +const input = (value = 60): ReferenceImportInput => ({ side: 'left', subjectId: 'sleeper-A', source: 'test', rows: [ + { recordKey: 'one', metric: 'heart_rate', value, unit: 'bpm', timestampMs: T }, +] }) +function commit(value = 60, supersedesImportId?: string) { + const data = input(value) + return store.commit({ import: data, previewHash: previewReferenceImport(data).summary.contentHash, supersedesImportId }) +} + +beforeEach(() => { + sqlite = new Database(':memory:') + sqlite.pragma('foreign_keys = ON') + db = drizzle(sqlite, { schema }) + migrate(db, { migrationsFolder: 'src/db/biometrics-migrations' }) + store = new ReferenceStore(db, () => 1700000000000) +}) +afterEach(() => sqlite.close()) + +describe('reference persistence', () => { + it('previews without writes and commits identical content once', () => { + const preview = previewReferenceImport(input()) + expect(db.select().from(schema.referenceImports).all()).toEqual([]) + const first = store.commit({ import: input(), previewHash: preview.summary.contentHash }) + expect(commit()).toMatchObject({ id: first.id, alreadyImported: true }) + expect(store.measurements(first.id).items[0]).toMatchObject({ value: 60, temporalKind: 'point' }) + expect(db.select().from(schema.vitals).all()).toEqual([]) + expect(db.select().from(schema.referenceImports).all()).toHaveLength(1) + }) + + it('requires exact preview and deliberate acceptance of rejected rows', () => { + const preview = previewReferenceImport(input()) + expect(() => store.commit({ import: input(61), previewHash: preview.summary.contentHash })).toThrow('preview') + const partial = { ...input(), rows: [...input().rows, { unknown: 'ignored?' }] } + const hash = previewReferenceImport(partial).summary.contentHash + expect(() => store.commit({ import: partial, previewHash: hash })).toThrow('partial import') + expect(store.commit({ import: partial, previewHash: hash, acceptRejectedRows: true }).summary).toMatchObject({ accepted: 1, rejected: 1 }) + }) + + it('stores corrections as revisions without changing older measurements and rejects stale parent edits', () => { + const first = commit() + const corrected = commit(61, first.id) + expect(corrected).toMatchObject({ familyId: first.familyId, revision: 2, supersedesImportId: first.id }) + expect(store.measurements(first.id).items[0].value).toBe(60) + expect(store.measurements(corrected.id).items[0].value).toBe(61) + expect(commit(61, first.id)).toMatchObject({ id: corrected.id, alreadyImported: true }) + expect(() => commit(62, first.id)).toThrow('Another correction') + const otherSide = { ...input(62), side: 'right' as const } + expect(() => store.commit({ import: otherSide, previewHash: previewReferenceImport(otherSide).summary.contentHash, supersedesImportId: corrected.id })).toThrow('mapping') + }) + + it('can correct a reference back to an earlier value without returning the old revision', () => { + const first = commit(60) + const second = commit(61, first.id) + const third = commit(60, second.id) + expect(third).toMatchObject({ revision: 3, familyId: first.familyId, supersedesImportId: second.id, alreadyImported: false }) + expect(third.id).not.toBe(first.id) + expect(commit(60, second.id)).toMatchObject({ id: third.id, alreadyImported: true }) + expect(store.measurements(first.id).items[0].value).toBe(60) + }) + + it('corrects the latest available revision after a newer revision is deleted without reusing its number', () => { + const first = commit(60) + const second = commit(61, first.id) + store.delete(second.id) + expect(() => commit(62, second.id)).toThrow('unavailable') + const third = commit(62, first.id) + expect(third).toMatchObject({ familyId: first.familyId, revision: 3, supersedesImportId: first.id, alreadyImported: false }) + expect(() => commit(63, first.id)).toThrow('Another correction') + expect(commit(63, third.id)).toMatchObject({ revision: 4, supersedesImportId: third.id }) + }) + + it('rolls back metadata when measurement publication fails', () => { + sqlite.exec('CREATE TRIGGER fail_measurement BEFORE INSERT ON reference_measurements BEGIN SELECT RAISE(ABORT, \'disk full\'); END') + expect(() => commit()).toThrow() + expect(db.select().from(schema.referenceImports).all()).toEqual([]) + expect(db.select().from(schema.referenceMeasurements).all()).toEqual([]) + }) + + it('paginates every retained row explicitly and makes deleted data unavailable', () => { + const data = { ...input(), rows: Array.from({ length: 503 }, (_, i) => ({ recordKey: String(i), metric: 'heart_rate', unit: 'bpm', value: 60, timestampMs: T + i * 60000 })) } + const result = store.commit({ import: data, previewHash: previewReferenceImport(data).summary.contentHash }) + const page = store.measurements(result.id) + expect(page.items).toHaveLength(500) + const remaining = store.measurements(result.id, page.nextCursor ?? 0) + expect(remaining.items).toHaveLength(3) + expect(remaining.nextCursor).toBeNull() + const deleted = store.delete(result.id) + expect(store.delete(result.id)).toEqual(deleted) + expect(store.get(result.id).available).toBe(false) + expect(() => store.measurements(result.id)).toThrow('deleted') + expect(db.select().from(schema.referenceMeasurements).all()).toEqual([]) + expect(store.commit({ import: data, previewHash: previewReferenceImport(data).summary.contentHash }).id).not.toBe(result.id) + }) + + it('refuses imports with no accepted rows, unknown ids and invalid measurement pages', () => { + const rejected = { ...input(), rows: [{ unknown: 'only' }] } + expect(() => store.commit({ import: rejected, previewHash: previewReferenceImport(rejected).summary.contentHash, acceptRejectedRows: true })).toThrow('no accepted measurements') + expect(() => store.get('00000000-0000-4000-8000-000000000000')).toThrow('not found') + const first = commit() + for (const [afterId, limit] of [[0, 0], [0, 501], [-1, 10], [0.5, 10], [0, 1.5]]) { + expect(() => store.measurements(first.id, afterId, limit)).toThrow('page size') + } + expect(store.measurements(first.id, 0, 1)).toMatchObject({ nextCursor: null }) + }) + + it('lists imports per side newest first with an explicit cursor', () => { + let clock = 1700000000000 + store = new ReferenceStore(db, () => clock++) + for (let index = 0; index < 51; index++) commit(60 + index) + const otherSide = { ...input(), side: 'right' as const } + store.commit({ import: otherSide, previewHash: previewReferenceImport(otherSide).summary.contentHash }) + const first = store.list('left') + expect(first.items).toHaveLength(50) + expect(first.items[0].summary.accepted).toBe(1) + expect(first.nextCursor).toEqual({ createdAtMs: first.items[49].createdAtMs, id: first.items[49].id }) + const second = store.list('left', first.nextCursor ?? undefined) + expect(second.items).toHaveLength(1) + expect(second.nextCursor).toBeNull() + expect(store.list('right').items).toHaveLength(1) + }) +}) diff --git a/src/biometrics/references/tests/time.test.ts b/src/biometrics/references/tests/time.test.ts new file mode 100644 index 000000000..4dc68b2f3 --- /dev/null +++ b/src/biometrics/references/tests/time.test.ts @@ -0,0 +1,58 @@ +// @vitest-environment node +import { describe, expect, it } from 'vitest' +import { ReferenceInputError, timestampResolver } from '../time' + +const T = Date.UTC(2026, 9, 10, 23, 0, 0) +type TimePolicy = Parameters[0] +const policy = (changes: Partial = {}): TimePolicy => ({ disambiguation: 'reject', ...changes }) +const code = (run: () => unknown) => { + try { + run() + } + catch (error) { + return error instanceof ReferenceInputError ? error.code : 'not-a-reference-error' + } + return null +} + +describe('reference timestamp resolution', () => { + it('accepts numbers, explicit offsets and Z while rejecting malformed or out-of-range text', () => { + const resolve = timestampResolver(policy({})) + expect(resolve(T)).toBe(T) + expect(resolve('2026-10-10T23:00:00Z')).toBe(T) + expect(resolve('2026-10-10T23:00:00.5Z')).toBe(T + 500) + expect(resolve('2026-10-11T01:30:00+02:30')).toBe(T) + expect(resolve('2026-10-10T20:00:00-03:00')).toBe(T) + expect(code(() => resolve('2026-10-10T23:00Z'))).toBe('INVALID_TIMESTAMP') + expect(code(() => resolve('2026-13-10T23:00:00Z'))).toBe('INVALID_TIMESTAMP') + expect(code(() => resolve('2026-02-30T23:00:00Z'))).toBe('INVALID_TIMESTAMP') + expect(code(() => resolve('2026-10-10T23:00:00+15:00'))).toBe('INVALID_TIMESTAMP') + expect(code(() => resolve('2026-10-10T23:00:00+14:30'))).toBe('INVALID_TIMESTAMP') + expect(code(() => resolve('2026-10-10T23:00:00-05:60'))).toBe('INVALID_TIMESTAMP') + expect(code(() => resolve('2026-10-10T23:00:00'))).toBe('AMBIGUOUS_TIMESTAMP') + expect(code(() => resolve(T / 1000))).not.toBeNull() + }) + it('applies a fixed policy offset only to local text', () => { + const resolve = timestampResolver(policy({ utcOffsetMinutes: -120 })) + expect(resolve('2026-10-10T21:00:00')).toBe(T) + expect(resolve('2026-10-10T23:00:00Z')).toBe(T) + expect(resolve(T)).toBe(T) + }) + it('resolves local times through an IANA timezone, including GMT and DST transitions', () => { + expect(code(() => timestampResolver(policy({ timezone: 'Not/AZone' })))).toBe('INVALID_TIMEZONE') + expect(timestampResolver(policy({ timezone: 'Etc/UTC' }))('2026-10-10T23:00:00')).toBe(T) + expect(timestampResolver(policy({ timezone: 'Europe/London' }))('2026-01-10T12:00:00')).toBe(Date.UTC(2026, 0, 10, 12)) + const pacific = timestampResolver(policy({ timezone: 'America/Los_Angeles' })) + expect(pacific('2026-10-10T16:00:00')).toBe(T) + expect(pacific('2026-10-10T16:00:00-07:00')).toBe(T) + expect(code(() => pacific('2026-10-10T16:00:00-08:00'))).toBe('TIMEZONE_OFFSET_MISMATCH') + // 2026-03-08 02:30 does not exist; 2026-11-01 01:30 happens twice. + expect(code(() => pacific('2026-03-08T02:30:00'))).toBe('NONEXISTENT_LOCAL_TIME') + expect(code(() => pacific('2026-11-01T01:30:00'))).toBe('AMBIGUOUS_LOCAL_TIME') + expect(timestampResolver(policy({ timezone: 'America/Los_Angeles', disambiguation: 'earlier' }))('2026-11-01T01:30:00')).toBe(Date.UTC(2026, 10, 1, 8, 30)) + expect(timestampResolver(policy({ timezone: 'America/Los_Angeles', disambiguation: 'later' }))('2026-11-01T01:30:00')).toBe(Date.UTC(2026, 10, 1, 9, 30)) + expect(code(() => timestampResolver(policy({ timezone: 'America/Los_Angeles', disambiguation: 'reject' }))('2026-11-01T01:30:00'))).toBe('AMBIGUOUS_LOCAL_TIME') + const kolkata = timestampResolver(policy({ timezone: 'Asia/Kolkata' })) + expect(kolkata('2026-10-11T04:30:00')).toBe(T) + }) +}) diff --git a/src/biometrics/references/time.ts b/src/biometrics/references/time.ts new file mode 100644 index 000000000..0a15648fd --- /dev/null +++ b/src/biometrics/references/time.ts @@ -0,0 +1,75 @@ +import { timestampMsSchema, type timePolicySchema } from './contracts' +import type { z } from 'zod' + +type TimePolicy = z.infer +const localPattern = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.(\d{1,3}))?$/ +const offsetPattern = /(Z|([+-])(\d{2}):(\d{2}))$/ + +export class ReferenceInputError extends Error { + constructor(readonly code: string, message: string) { super(message) } +} + +function localMilliseconds(text: string): number { + const parts = localPattern.exec(text) + if (!parts) throw new ReferenceInputError('INVALID_TIMESTAMP', 'Use ISO date and time with seconds and an explicit offset or time policy') + const fields = parts.slice(1, 7).map(Number) + const [year, month, day, hour, minute, second] = fields + const ms = Date.UTC(year, month - 1, day, hour, minute, second, Number((parts[7] ?? '').padEnd(3, '0'))) + const date = new Date(ms) + if (date.getUTCFullYear() !== year || date.getUTCMonth() !== month - 1 || date.getUTCDate() !== day + || date.getUTCHours() !== hour || date.getUTCMinutes() !== minute || date.getUTCSeconds() !== second) { + throw new ReferenceInputError('INVALID_TIMESTAMP', 'Invalid calendar date or time') + } + return ms +} + +/** Per-preview resolver; cache is bounded by the bounded import, not global. */ +export function timestampResolver(policy: TimePolicy) { + let formatter: Intl.DateTimeFormat | undefined + if (policy.timezone) { + try { + formatter = new Intl.DateTimeFormat('en-US', { timeZone: policy.timezone, timeZoneName: 'longOffset' }) + } + catch { throw new ReferenceInputError('INVALID_TIMEZONE', 'Use a valid IANA timezone') } + } + const offsetAt = (ms: number) => { + const name = formatter?.formatToParts(ms).find(part => part.type === 'timeZoneName')?.value + if (name === 'GMT') return 0 + const match = name?.match(/^GMT([+-])(\d{2}):(\d{2})$/) + if (!match) throw new ReferenceInputError('INVALID_TIMEZONE', 'Timezone offset cannot be resolved') + return (match[1] === '-' ? -1 : 1) * (Number(match[2]) * 60 + Number(match[3])) + } + const offsetsByDay = new Map() + return (value: string | number): number => { + if (typeof value === 'number') return timestampMsSchema.parse(value) + const offset = offsetPattern.exec(value) + const local = localMilliseconds(offset ? value.slice(0, -offset[0].length) : value) + let explicitOffset = policy.utcOffsetMinutes + if (offset) { + if (offset[1] === 'Z') explicitOffset = 0 + else { + const hours = Number(offset[3]), minutes = Number(offset[4]) + if (hours > 14 || minutes > 59 || (hours === 14 && minutes > 0)) throw new ReferenceInputError('INVALID_TIMESTAMP', 'Offset is outside the supported range') + explicitOffset = (offset[2] === '-' ? -1 : 1) * (hours * 60 + minutes) + } + } + if (explicitOffset !== undefined) { + const utc = local - explicitOffset * 60000 + if (formatter && offsetAt(utc) !== explicitOffset) throw new ReferenceInputError('TIMEZONE_OFFSET_MISMATCH', 'The supplied offset does not match this local time in the selected timezone') + return timestampMsSchema.parse(utc) + } + if (!formatter) throw new ReferenceInputError('AMBIGUOUS_TIMESTAMP', 'Local timestamps require a timezone or an explicit UTC offset') + const day = Math.floor(local / 86400000) + if (!offsetsByDay.has(day)) { + // Inspect both sides of local DST changes, including non-hour changes. + const offsets = new Set() + for (let hours = -36; hours <= 36; hours += 6) offsets.add(offsetAt(day * 86400000 + hours * 3600000)) + offsetsByDay.set(day, [...offsets]) + } + const candidates = (offsetsByDay.get(day) ?? []).map(minutes => local - minutes * 60000) + .filter(utc => utc + offsetAt(utc) * 60000 === local).sort((a, b) => a - b) + if (!candidates.length) throw new ReferenceInputError('NONEXISTENT_LOCAL_TIME', 'This local time falls in a timezone clock gap; provide the correct timestamp') + if (candidates.length > 1 && policy.disambiguation === 'reject') throw new ReferenceInputError('AMBIGUOUS_LOCAL_TIME', 'This local time occurs twice; provide an offset or choose earlier/later explicitly') + return timestampMsSchema.parse(policy.disambiguation === 'later' ? candidates[candidates.length - 1] : candidates[0]) + } +} diff --git a/src/biometrics/shadow/README.md b/src/biometrics/shadow/README.md new file mode 100644 index 000000000..959f4a93e --- /dev/null +++ b/src/biometrics/shadow/README.md @@ -0,0 +1,1063 @@ +# Prospective shadow monitoring + +`ShadowStore` is the storage component for prospective monitoring. It uses only +`biometric_shadow_runs` and `biometric_shadow_samples`, added by a generated +additive biometrics migration. It does not write canonical vitals, activate a +profile, import algorithm code, or control hardware. The installed application +service connects it to the shipped worker, bounded live sources and public +inspection/submission APIs. Execution requires explicit deployment configuration +and a matching release-installed qualification registry; neither is inferred +from a synthetic benchmark. The sections below describe each layer. + +A frozen plan binds subject and side, active/candidate configuration hashes, reviewed +qualification and engine-build identities, a prospective time window (at most +seven days), selected metric definitions, and explicit point/byte limits. +Identities alone do not establish qualification: the installed controller must +resolve the release-installed build and qualification before admission. +No CPU, RSS, free-space reserve, or production storage defaults are inferred. + +Create is idempotent for an exact plan. New plans cannot start in the past. +Claim invokes a trusted synchronous admission check inside the write transaction +and grants an expiring attempt lease. At most one live worker per side is allowed. +A replaced/expired worker cannot append, renew, or finish over its successor. +Retained samples survive replacement and carry the original attempt identity. +The controller must reset processing state at the new attempt boundary. + +Append validates side, exact active/candidate configuration, input identity, +selected metrics, ordered sequence/time, and the prospective window. Candidate +and contemporaneous active readings are retained separately in each sample; +missing values remain null, never zero. Metrics use canonical units: heart rate +in bpm, breathing rate in breaths/min, and `window_ibi_variability_v1` in ms. +The variability index must not be interpreted as RMSSD or SDNN. Exact retries do +not consume quota; conflicting retries fail. + +Per-run and global point/byte limits are checked in the same immediate SQLite +transaction as insertion. Exhaustion atomically stops that run with +`storage_quota`, preserving prior evidence. A run-count quota also bounds plan +metadata. Byte counts measure serialized sample payloads, not SQLite page/WAL +allocation; filesystem reserve and database maintenance remain host obligations. +There is no automatic deletion of reports, plans, or exposure history. + +`stop` supports host/operator cancellation even after deadline or worker loss, +retaining the original terminal reason on retries. The controller must stop +elapsed windows and enforce execution budgets separately. Heartbeat and sample identity feed the reporting component below. The live +shadow processor, admission controller and public API use the installed service +composition described below. + +## Health and reference reporting + +Plans also freeze `monitoring`: expected sample interval, maximum gap, heartbeat +staleness, minimum candidate coverage, maximum coverage loss, minimum quality, +and the shared reference-alignment policy. There are no implicit quality or +production thresholds. Selecting `window_ibi_variability_v1` requires an explicit +`windowIbiWindowMs` contract; the eventual admitted processor must match it. + +`snapshot(runId, limits)` reads the entire selected run in one SQLite snapshot, +checking row counts, actual payload bytes, per-sample hashes, sequence/time order, +side/configuration/metric identities and original attempt IDs. It refuses an +oversized or corrupted run rather than silently scoring a partial page. Callers +supply limits (at most 100,000 points and 32 MiB); small chart pages must never be +substituted for a report input. + +`reportShadowRun` derives health, gaps, current/candidate coverage, attempt count +and typed reference agreement. Coverage counts distinct usable cadence slots +against slots elapsed in the frozen prospective window: a burst in one slot +cannot hide missing slots. Low-quality and null values remain missing. Reports +identify stale workers, processor failure, quota stops, observation gaps, clock +skew and coverage regressions. A completed/stopped run is not assumed healthy. + +Optional reference imports must match subject and side. The shared normalization +and common-support comparison code handles alignment and metric/window +compatibility. RMSSD and SDNN are not scored against the window variability +index. Out-of-window/future reference rows and incompatibility reasons remain +visible. Reports carry plan/snapshot/report hashes and reference provenance; +they are descriptive prospective evidence, not clinical validation or activation +authorization. Whole-night uncertainty and promotion review remain separate. + +## Shipped-core prospective adapter + +`modules/common/biometrics/shadow.py` provides `ProspectiveShadow`, an in-memory +adapter around the shipped shared processor and routing/pump gates. It accepts +both channels plus complete observed context, keeps isolated candidate state, +and emits only the selected logical side through an injected shadow sink. +It does not open a database or source, load proposed code, activate a profile, +or import hardware control. The controller must still verify release-installed +qualification and connect it to the separate storage path. + +The controller supplies observed arrival timestamps, matching the live core's +clock semantics; recorded replay retains its existing event-time mode. Cold +start, insufficient preroll and missing context remain explicit limitations. +Each output carries sequence, immutable reading/provenance and a rolling input +prefix hash over arrival timestamps, actual channel bytes and recorded context. +Arbitration may delay publication: this hash identifies the prefix observed at +publication, not a claim that a reading used every sample in that prefix. + +Record/output quotas and cumulative CPU checks stop the adapter on excess. +Frame output remains provisional until the cooperative CPU check succeeds. +Sink errors stop further processing; the controller must resolve any uncertain +write through the storage idempotency contract. Cancellation does not flush +pending arbitration; normal finish flushes once. These checks cannot preempt an +individual numerical call. The kernel entrypoint and parent supervisor below +provide CPU/RSS/wall enforcement. The planned source factory and owned attempt host below connect the live +source/controller/storage bridge. + +## Subprocess transport + +`python -m common.biometrics.shadow_worker` exposes the shipped adapter through +bounded JSON lines. Initialization binds run/attempt UUIDs, both configurations +and their canonical hashes, selected side, observation window, processor quotas, +and input/output transport ceilings. Frames have an ordered sequence, explicit +observed-arrival timestamp, record and context. Binary channel values are base64, +validated before conversion and bounded to the core's existing channel limit. + +The worker replies once per frame, with the same run/attempt identity, input +sequence, candidate-only outputs and health. Output readings include metric, +canonical unit and explicit window definition (the shipped variability index's +window comes from the shared processor constant). It accepts explicit finish or +cancel; unexpected EOF, bad identities/order/encoding, or transport overflow +fails instead of publishing a completion acknowledgment. Lines are capped at +128 KiB and cumulative transport is bounded by the initialization request. + +The worker does not receive database handles or hardware access and does not +open the configured biometrics database. The supervisor must supply actual +contemporaneous active readings separately, validate replies, enforce OS limits, +and persist through the fenced shadow store. The components below provide reply +validation, persistence and process supervision; live controller wiring is pending. CPU accounting now includes failed adapter calls; +missing, invalid or regressing CPU clocks terminate the adapter. + +## Supervisor reply validation and fenced sink + +`ShadowWorkerProtocol` validates the parent side of the worker protocol. It +requires the expected command/run/attempt/configuration identity, ordered frame +acknowledgments and candidate output, selected logical side, observation window, +all three metric/unit/window definitions, and consistent monotonic health/quota +receipts. Source side may differ from logical side under single-sleeper routing. +A rejected reply advances no counters; a terminal acknowledgment closes the +protocol. A real locked-Python subprocess test checks this TypeScript/Python +contract across 150 waveform frames. + +`ShadowWorkerSink` maps validated candidate output into the leased shadow store. +Construct one sink per worker attempt. It preserves original source side, +worker output sequence, flags and limitations, including cold start and missing +context. The host supplies actual active readings with exact timestamp, side, +configuration and metric contracts. It must not synthesize active readings by +rerunning a baseline or move a nearby reading onto the candidate timestamp. +Absent active data stays null with `missing_contemporaneous_active_reading`. +Exact uncertain-write retries reuse the same candidate and active snapshot; +changed retries fail. Replaced leases remain unable to write. + +This sink handles candidate output, not a complete live join: the supervisor +must also retain active-only observation times, manage bounded joining and late +arrival, and identify missing candidate output, using `ShadowStreamJoin` below. +Admission, live source capture and public API integration remain unfinished. Reports +must not be used as complete prospective evidence until those paths are wired. + +## Bounded active/candidate stream union + +`ShadowStreamJoin` is the complete timestamp-union sink for a worker attempt. +Plans using it must freeze `joining.maximumPendingPoints`, `maximumLatenessMs`, +and `drainGraceMs`. The grace must exceed lateness; pending points are capped at +10,000, lateness at two minutes and drain grace at five minutes. These are input +ceilings, not recommended pod defaults. Host measurement/admission must choose +appropriate smaller budgets. + +Feed both actual active observations and validated candidate outputs, then call +`advance(nowMs)` after enqueuing arrivals. The join emits timestamps at or before +`nowMs - maximumLatenessMs` in order, retaining points present in only one stream +with nulls on the missing side. Nearby timestamps remain distinct. Exact pending +duplicates are harmless; changed duplicates, arrivals after a timestamp has been +sealed, clock regression and queue exhaustion fail the attempt explicitly. A +lost lease cannot fail or write over its replacement. + +Storage acknowledgment is required before removing a buffered point or advancing +its durable sequence, so an uncertain committed write can be retried exactly. +Storage quota exhaustion closes the join. On restart, a new attempt begins after +the last retained timestamp; discarded in-memory observations are not invented. +The supervisor must collect both streams continuously and surface failures. + +Leases for joined runs may remain valid until the frozen end plus drain grace, +allowing already-buffered samples to be stored after collection ends. Sample +timestamps must still be inside the original window; new worker claims cannot +start after its end. The supervisor stops feeding candidate input at the window +end, drains both sources within grace, and then finishes the run. Grace is not +permission to extend observation or processing indefinitely. + +Joined samples identify whether each stream was present and whether their input +identity is a candidate prefix, active provenance, or unavailable. Active-only +readings with unavailable signal provenance retain a null input hash rather than +a fabricated one. Reports expose active-only/candidate-only counts and unknown +input identity. Live source capture, admission and API/UI wiring +remain required to turn this component into an enabled prospective service. + +## Linux kernel-limit entrypoint + +The production supervisor must launch `common.biometrics.shadow_process` with +explicit `--cpu-seconds` (1–3600) and `--address-space-bytes` budgets. This installs +CPU, core-dump, open-file, low-priority and numerical-thread limits using the +existing replay bootstrap, then a hard address-space limit, before importing the +numerical shadow worker or reading its initialization. Yocto without Python's +`resource` extension uses the fixed `/usr/bin/prlimit` utility; missing or refused +limits fail startup. Unsupported hosts fail rather than claim enforcement. +There are no production budget defaults. The direct `shadow_worker` module +remains a transport test entrypoint and does not install kernel limits. + +The CPU cap includes import/startup and has a one-second hard-limit grace; it is +separate from the adapter's finer cooperative processing budget. Address space +bounds virtual allocations, not resident memory. The parent supervisor below +enforces RSS, wall/idle deadlines, cancellation and process cleanup. The live +controller must signal lease loss and perform admission. This entrypoint runs only +trusted shipped algorithms and is not a sandbox for external package proposals. + +## Parent process supervision + +`runShadowChild` launches that fixed Linux entrypoint in a detached process group, +with trusted host paths, a sanitized environment, and explicit resource budgets. +It validates the frozen configurations and hashes before launch. Input comes from +an async frame iterator, with one frame in flight and one validated acknowledgment +per command. The caller must bound any buffering in its source adapter. Each +candidate batch passes `ShadowWorkerProtocol`; the supervisor waits for each sink +callback before pulling the next frame. The callback must use the leased store +or join and propagate storage/lease errors. + +The supervisor enforces cumulative transport bytes, 128 KiB lines, record counts, +16 KiB diagnostics, source-idle and acknowledgment deadlines, and a separate wall +deadline that also bounds stalled sinks. Unsolicited output, malformed UTF-8/JSON, +missing terminal acknowledgment, nonzero exit and pipe errors fail the run. +Diagnostics are discarded rather than copied into logs. Normal completion +requires both a validated finished receipt and a clean process exit. + +Bounded procfs observations sample the entire detached process group's RSS. +Unavailable, timed-out or excessive observations fail closed. A successful receipt +includes the observed peak, sample count, maximum observation gap, configured +budgets and explicit sampled-memory scope; it does not claim an exact kernel RSS +peak. The address-space kernel cap is a separate limit. There are no pod resource +defaults in this component. + +Cancellation (including a host lease-loss signal), source/sink failures and every +exit path kill the process group and wait for the direct child to close, with a +bounded cleanup deadline. A descendant retaining the worker's pipes is killed as +well. There is no canonical worker state to flush. Iterator cleanup is requested +without allowing a stalled iterator to keep the process alive. A pending host +sink cannot be forcibly cancelled; its writes must remain attempt-fenced. + +This implements process supervision, not service admission or restart recovery. +The live controller must still bind the installed build/qualification and active +profile, manage leases, capture bounded sensor/active streams, renew or cancel on +binding changes, recover after host crashes, and expose the requested API/UI. + +## Fenced attempt lifecycle + +`runShadowAttempt` connects the store, stream join and subprocess supervisor for +one attempt. Its trusted host supplies shipped configurations, synchronous +admission and bounded source adapters; execution budgets come from the saved plan. Admission must resolve +the installed qualification/build, active profile and host resource/source health; +it is checked at claim, on every incoming reading/frame and on each heartbeat. +The heartbeat interval is at most one third of the lease, one second, and the +plan's heartbeat-staleness threshold. An asynchronous admission result is refused. + +The source factory registers both streams and exposes a synchronous unsubscribe. +It must unwind partially registered subscriptions if opening throws, and call +`fail()` when either stream disconnects or overflows. Frames must carry their real +observed arrival times. The controller assigns fresh per-attempt sequence numbers, +refuses future arrivals, and ends the iterator at the frozen window boundary even +if its next read is stalled. A source ending early is a failure. Active readings +may continue arriving during the bounded final drain; timestamps remain in the +original observation window. The controller requires a worker completion receipt, +waits through the allowed lateness interval, flushes the join, closes both sources +and only then records completion. Source cleanup failure prevents completion. + +Admission changes, lease loss, source/worker/sink failures and operator cancellation +abort the worker and stop accepting callbacks. Worker cleanup finishes before the +terminal store transition. Existing operator/quota/join termination is preserved. +If its lease expired during cleanup, `failExpired` can mark that same attempt +failed; it cannot affect a replacement, unexpired lease or already terminal run. + +The owned attempt host and application service below provide the host adapter +and service/API lifecycle. +The controller does not invent qualification, choose production budgets, subscribe +to hardware on its own or recover orphan workers after host crashes. Completion +now persists the validated worker receipt atomically with terminal state. Those integration obligations remain before +prospective runs can be enabled or treated as production-qualified evidence. + +## Live waveform source adapter + +`openShadowFrameSource` subscribes to the shared RAW/NATS dispatch through the +separate `prospectiveFrameTap`. The tap lives on `globalThis`, has at most four +subscribers, excludes ingestion backlogs and unrelated sensor types, and isolates +consumer failures from canonical live ingestion. Current-source NATS termination +and server shutdown notify subscribers of failure. Admission/idle monitoring must +still detect silent sources and temporary disconnects. + +The adapter freezes each frame's actual host arrival timestamp and the trusted +host's contemporaneous occupancy/active-side/pump-availability context. It encodes +signed little-endian int32 channels (at most 5,000 samples each) for the Python +worker, and copies only the bounded pump fields used by the shared gate. It never +retains mutable decoder arrays or invents missing context. Explicit event-age and +future-skew budgets reject delayed live delivery even when it was not labelled as +a backlog. Only the shipped 500 Hz waveform format is accepted when a frequency +is present; hardware compatibility still belongs to admission. + +Frame-count and serialized-byte queue budgets are mandatory. Overflow, invalid +samples/context/clocks, and unread queued frames at collection end fail the +attempt; there is no silent drop or unbounded catch-up. A single consumer reads +in arrival order. Abort, window end and iterator return unsubscribe and settle +pending reads. This adapter supplies candidate input only. The active-reading +adapter, context provider, installed qualification/admission and service wiring +remain required before enabling prospective shadow runs. + +## Read-only active observation capture + +`ShadowActiveReader` polls an existing read-only biometrics SQLite connection. +It starts at the current vitals ID and accepts only subsequently inserted rows +for the selected side and observation window. Each poll reads one consistent +snapshot with an explicit row quota (at most 100); overflow refuses the whole +batch. Bundle and input metadata are bounded in SQL before materialization. +The cursor advances only after the complete batch validates. A deleted or changed +cursor anchor fails rather than silently continuing through a reset or rewrite. +The host must close/reopen its connection on database replacement and treat any +capture failure as an attempt failure. + +Every accepted row must have exactly one matching quality record and a live +processing run with the expected configuration, validated resolved parameters, +shipped manifest, component and source identities, side and time boundaries. +Legacy/unknown, replay, corrupt, mismatched, missing-quality and future observations +are refused. Null metric values remain null, quality is copied from the committed +quality row, and original second-resolution timestamps remain unchanged. Physical +source side can differ from the logical side under single-sleeper routing. + +The existing daemon explicitly records `rawIdentity: not_retained`, so this reader +returns a null input hash. It preserves processing-run ID, physical source side, +generation and manifest identity as `activeProvenance`; the join and candidate +sink retain these fields in the separate shadow sample payload. This is an +additive JSON field, requiring no schema migration or canonical-vitals write. +Polling cadence, final-drain polling, runtime admission/context, connection +replacement detection and receipt persistence are provided by the composed +components below; production service/API wiring remains host integration work; this reader alone does not enable a prospective run. + +## Composed live sources and drain polling + +`openShadowLiveSources` establishes the active-reader watermark before subscribing +to prospective waveform input. It polls the read-only reader on an explicit +10–1000 ms cadence, strictly shorter than the frozen join lateness, and exposes +`flush()` for synchronous polling before the controller seals join timestamps. +`runShadowAttempt` invokes that flush on every heartbeat and at final drain; +commits arriving between the last timer poll and completion are included. Flush, +active delivery and database-identity checks must be synchronous. Their failures +terminate the attempt rather than acknowledge an incomplete read. + +Waveform collection ends at the observation boundary while active polling remains +available during drain. Abort, source disconnect, poll failure and explicit close +remove both subscriptions/timers. Source teardown does not own the SQLite handle. +The host closes that handle afterward. + +`openShadowActiveDatabase` opens an existing regular file read-only with an +explicit bounded busy timeout. It checks the path's device/inode identity before +and after opening and supplies `assertCurrent` to the composed source. Replacement, +removal, a symlink or a closed connection stops capture instead of reading the old +file indefinitely. Paths remain trusted host configuration, never caller input. + +These pieces use the context provider, frozen budgets and receipt persistence +described below. Production installed-build/qualification wiring, orphan recovery +and service/API registration remain. No production defaults or qualified registry entries +are inferred by composing the sources. + +## Committed context snapshots + +`ShadowContextReader` supplies the frame adapter's `(frame, observedAtMs)` context +callback. It reads sleeper configuration through a read-only connection using the +shared `activeSleeperSides` logic. Both side rows and a valid bed mode are required; +missing or corrupt settings produce unknown mode instead of invented defaults. +The host's database identity guard is checked before and after capture. + +Occupancy comes from the sleep detector's version-1 committed presence file, +matching the evidence fields used by Python `BedPresence`: `vitals_presence` and +`vitals_evidence_ts`. File and per-side evidence freshness are both checked against +the observed arrival time with an explicit limit no greater than the daemon's +five-minute ceiling. Missing, stale, future, malformed, oversized, symlinked or +changing files produce unknown presence. Reads are bounded, no-follow, regular-file +only; unknown on one side does not erase valid evidence on the other. The virtual +web/HomeKit occupancy estimate is not substituted for this committed evidence. + +Pump availability starts unknown for each attempt. Only observed `frzHealth` +records with usable pump fields on both sides establish it; incomplete updates +invalidate availability, and an explicit age limit expires it. This does not +invent pre-attempt pump state or remove the processor's pump gates. + +These are current host observations, not proof of the active daemon's exact +cached view: Python independently reloads sleeper settings and committed presence. +Reports explicitly identify that context evidence limitation. Production admission +and qualification must account for those timing differences. The provider does +not itself qualify a run, persist execution policy, or register a live service. + +## Frozen execution policy + +Executable plans now require `execution` (`shadow-execution-policy-v1`). It binds +both physical-channel configuration hashes, record/output/transport/CPU quotas, +process CPU/address-space/RSS/deadline limits, lease and heartbeat cadence, active +polling/row/busy-timeout limits, waveform queue/freshness limits and committed +context freshness/file limits. Schemas are shared with the worker, waveform and +context components; there are no implicit production defaults. + +The complete policy participates in the plan hash and create-idempotency contract. +The store validates heartbeat/lease and polling/lateness compatibility. Reports +expose `executionPolicyHash`. Old plans lacking execution policy remain inspectable +and reportable, but `runShadowAttempt` refuses to execute them. No migration or +backfilled claim of past resource enforcement is needed. + +The attempt reads execution limits only from a private copy of the stored plan; +the host no longer supplies alternative worker limits. Both channel configs must +match before claim, since the opposite channel influences routing and rejection. +Source factories receive their own policy copy for configuring capture, and +cannot change the worker's copy. The production factory must apply those source +limits and the installed qualification must match the complete policy. Selecting +this policy does not establish qualification or record observed worker resources; +admission/service integration remains, while completion records receipts below. + +## Planned source factory + +`plannedShadowSourceFactory` is the concrete `openSources` implementation for an +attempt. It snapshots and validates the saved plan and trusted installation paths, +then refuses a different execution policy or collection end at invocation. It +opens both databases read-only with the saved busy timeout, creates a fresh +committed-context reader, and applies the saved active polling, waveform queue, +freshness and context limits to the composed sources. There are no alternate +numeric limits at this layer. + +The factory owns both connections. It unsubscribes before closing handles on +normal close, abort, source failure, direct flush failure and partial startup. +Configuration and biometrics path identities are both checked during capture. +Tests exercise real SQLite connections and the sensor tap, including frozen +policy substitution, saved queue overflow, cancellation and failed second-database +startup. Installed qualification/build/current-binding admission, orphan recovery +and public service registration remain separate requirements; constructing sources does not authorize a candidate to run. + +## Installed shadow admission + +`installedShadowAdmitter` checks a separate, release-installed +`shadow-qualifications.json` beside the trusted worker. Replay promotion evidence +does not authorize prospective execution. The bounded, regular, no-follow file is +read on every check, so removing an entry revokes subsequent admission. Exactly +one entry must match the saved qualification hash, current manifest, installed +engine/Python/hardware identity, sample rate, active configuration and complete +execution policy. Evidence includes reviewed fixture, pod and held-out artifacts +and explicitly acknowledges the host-context timing limitation. + +Admission also requires an applied, fresh selected-side runtime receipt matching +the generation captured before the attempt, plus fresh memory, disk, CPU load, +event-loop, ingestion and temperature observations under the entry's health +policy. A same-hash reset still changes generation and stops admission. Missing, +ambiguous, stale or asynchronous observations refuse execution. + +The callbacks are trusted host collectors, not request fields. Concrete collectors +must verify installed artifacts and aggregate all relevant daemon receipts; the +current aggregate receipt contract is not itself proof that those components ran. +Production integration must also guard relevant opposite-side binding changes. +No production entry or collector is installed here, and the admission module +does not authorize activation or provide an API for writing qualification files. + +`shadowRuntimeBindingReader` supplies the aggregate binding observation from +read-only configuration and biometrics handles. It freezes expected generations +and hashes for both physical channels, validates desired activation hashes and +bounded runtime projections against the shipped manifest and resolved config, +then requires the latest heart-rate, breathing-rate and HRV receipts to be applied +at those exact generations. It never falls back to an older applied generation. +Missing or pending receipts, future boundaries/heartbeats, and changed opposite-side +bindings refuse observation. The minimum of all six heartbeats lets the admission +policy enforce freshness for every component. Sleep-stage receipts are on-demand, +so they are not evidence of continuous numerical health. + +The collector uses nested read snapshots and checks caller-owned database identity +before and after capture. These are observations, not a cross-database lock or a +promise that activation cannot occur after a read; the attempt must keep checking +admission. It opens no files and writes no runtime state. Host construction still +needs to capture both expected baselines, provide file-identity guards, and connect +this collector with installed build identity and resource-health observations. + + +## Durable completion receipt + +Migration 0030 adds nullable receipt and receipt-hash fields without inventing +measurements for old runs. Completing an executable plan requires a validated +worker receipt in the same fenced transaction as terminal state. The bounded +16 KiB envelope binds run, attempt, plan hash and completion time, and retains +worker health/counters, CPU time, transport bytes, sampled process-group RSS, +sampling count/gap and frozen child budgets. Its storage is bounded per run in +addition to sample-byte quotas and the existing run-count limit. + +Store reads verify the envelope hash and identities and recheck measurements +against the saved execution policy. Snapshots and reports retain this evidence. +RSS remains a sampled sum across the process group, not an exact kernel peak; +recording a receipt does not establish pod qualification or activation authority. +Failed/cancelled attempts currently have no successful worker receipt, and crash +recovery/partial resource evidence remains separate work. Receipt-validation +failure after cleanup marks the same attempt failed, including the expired-lease +fallback, without overwriting a replacement attempt. + +## Host health sampling + +`openShadowHealthMonitor` connects the existing Linux memory, filesystem space, +load, event-loop delay and thermal collector to a trusted live piezo receive-clock +callback supplied by the application. +The host supplies that callback, the database directory and installed qualification policy. +The monitor waits for an initial collection, then exposes a synchronous `read` +for admission. Every read re-evaluates health and freshness against that policy; +missing, stale, future or unhealthy evidence returns null. + +Refresh cadence and collection deadline are each half the allowed sample age, +capped at one second. Only one collection can be in flight. A rejection or timeout +closes the monitor; uncancellable filesystem I/O is observed but never retried or +allowed to publish a late result. Abort/close cancels timers and invalidates cached +evidence. Successful null/unhealthy samples can be refreshed, but never admitted. +The attempt's next admission check stops it when the monitor cannot provide health. +Host service ownership, installed build identity and qualification selection still +need to wire this monitor into the complete attempt lifecycle. + +## Shadow Python build identity + +`probeShadowBuild` invokes the fixed `common.biometrics.shadow_build` probe using +the same interpreter and module directory as the worker, with sanitized thread +settings, a ten-second deadline and a 64 KiB output bound. It checks the canonical +identity hash and requires the shadow-specific source entries. Replay fingerprints +alone cannot qualify a shadow worker. + +The Python fingerprint uses `shadow-python-build-v1` and covers all shared replay +sources and locked dependency metadata plus `shadow.py`, `shadow_worker.py`, +`shadow_process.py` and the probe itself. Thus prospective processing, JSONL +transport and kernel-limit changes invalidate qualification. The probe neither +imports the numerical processor nor opens live sources. Dependency RECORD/WHEEL +hashes identify installation metadata, not every installed binary's contents. +Production composition must still bind the shipped Node host bundle, capture +hardware identity, and revalidate installation identity during execution. + +## Owned attempt host + +`openPlannedShadowHost` composes the installed qualification, both-side runtime +binding collector, live health monitor and saved-policy source factory. It owns +separate read-only admission connections with file-identity guards and the saved +busy timeout. Qualification lookup precedes health sampling, and initial admission +must pass before the host is returned. Subsequent admission also requires the +entire saved plan to match. Installation paths, candidate configs and expected +baseline identities are privately copied; they are trusted host inputs, never +arbitrary RPC executable paths or identity assertions. + +`runPlannedShadowAttempt` passes this host to the leased controller and closes its +resources on success or failure. Closing aborts the attempt first, then closes +health and admission connections. Partial startup and cancellation while waiting +for initial health also unwind ownership, including a late returned health handle. +The source factory separately owns its capture connections and subscriptions. + +This composition still requires production code to resolve candidate/baseline +configs, capture hardware and shipped Node/Python identity, and supply a synchronous +identity collector that detects changes. It does not recover orphan workers, +register APIs/UI, install qualification entries or create production budgets. + +## Resolving candidate profiles and active baselines + +`resolveShadowProfiles` reads the saved candidate revision through the existing +immutable profile validator, including side, pinned manifest, override rebuild and +dependency checks. A null revision explicitly requests the shipped default. The +resolved candidate must match the saved candidate hash. + +Both active baselines come from desired activation generations and their bounded +runtime projections in nested read snapshots. The full runtime binding collector +then verifies all numerical receipts and projection identities; no unacknowledged +fallback is allowed. The selected active hash and both candidate execution hashes +must match the plan. The opposite channel retains its acknowledged configuration. +This operation requires read-only handles and pre/post file-identity guards and +writes no activation, receipt, repair or canonical measurement. + +`runProfiledShadowAttempt` owns temporary profile-resolution handles and closes +them before constructing the admission host. Admission checks the captured +identities again, so changes between resolution and execution are rejected. This +entry point accepts a saved revision ID rather than caller-asserted configs or +active generations. Trusted installed build/hardware identity, service ownership, +crash recovery, qualification and public API/UI integration remain required. + + +## Packaged host identity + +The standard biometrics build also produces `shadow-host.cjs` and its dependency +manifest. This CommonJS module exports the profiled attempt runner, store/plan +schemas, installed qualification reader and build probe; importing it starts no +service. The build rejects hardware, automation, scheduler, HomeKit, live database +singletons and streaming entrypoints. Only the passive `prospectiveFrameTap` is +allowed from streaming. The application supplies the actual receive-clock callback, +so the bundle does not initialize another stream or hardware controller. + +`shadowEngineBuildIdentity` hashes the exact host bundle bytes, prospective Python +fingerprint and Node runtime identity. The real-package test loads the exported +API, inspects its dependency graph, compares packaged/source probe identities and +proves changing the host artifact changes the engine fingerprint. Production must +load and execute this same bundle and guard installation changes during a run; +fingerprinting an unused file would not establish execution identity. Hardware +capture, recovery, service/API/UI registration and current-build qualification +remain required before production runs. + +## Loading the verified package + +`loadInstalledShadowHost` reads a trusted regular-file artifact with no-follow +semantics and a 16 MiB bound, hashes its bytes, and checks its device/inode, size, +mtime and ctime before and after loading. It rejects an unsupported API version, +unverified existing Node cache entry, changed installation or different bundle +path after the first load. The verified package is held on `globalThis`, avoiding +multiple independent loader instances under the application bundler. Parent paths +are canonicalized for Node's cache key while leaf symlinks remain forbidden. + +The returned `assertCurrent` checks both canonical and requested paths; replacement +with identical bytes still requires restart. `inspectInstalledShadowHost` probes +through that loaded API, rechecks the artifact after the asynchronous Python probe, +and verifies the reported host hash matches the loaded bytes. These guards cover +the Node artifact. Python source/dependency change monitoring and hardware capture +remain separate requirements, and the application must actually use this loader +and its guard when registering production execution. + +## Python installation change guard + +The shadow probe now supplies absolute RECORD/WHEEL locations for the installed +NumPy and SciPy distributions; ambiguous or missing metadata paths refuse the +probe. Locations remain outside the portable build hash. The identity still +contains source and metadata content hashes. + +`guardShadowPythonInstallation` validates every listed source and dependency +metadata file against the probe with bounded reads (16 MiB per artifact), rejects +traversing source paths and relative dependency paths, and records requested and +canonical file identities. Metadata text normalization matches Python's text +hashing. Subsequent checks detect device/inode, mode, size, mtime/ctime or link-target +changes without rehashing all content each heartbeat. The interpreter link and +resolved target are watched too. `inspectInstalledShadowHost` now returns a +combined Node/Python guard after verifying the actual loaded package. + +This covers shipped source, installed wheel metadata, and interpreter replacement; +it is not a full native-library binary attestation. The guard must still be called +by production admission alongside actual hardware identity. Service registration, +crash recovery and current-build qualification remain outstanding. + +## Observed hardware identity + +`observedShadowHardware` reads the existing DAC monitor without starting it or +sending commands. It requires running status, a successful poll timestamp within +the qualification's explicit `maximumHardwareObservationAgeMs`, and a recognized +H/I/J sensor revision consistent with the reported pod version. Unknown labels do +not inherit the general status parser's Pod 3 fallback. The hardware ID contains +pod version and exact sensor revision, never a serial number. The 500 Hz value is +the shipped waveform protocol rate; frame admission separately rejects an explicit +incompatible frequency. + +`installedShadowIdentity` combines that observation with the inspected Node/Python +fingerprints and invokes the combined artifact guard on every call. Build or +hardware mismatch rejects admission. This adapter lives on the application side +of the package boundary, so hardware services remain excluded from the passive +host bundle. Production service execution must still supply this collector and +register cancellation/recovery and user-facing operations. + +## Installed application runner + +`runInstalledShadowAttempt` connects the application to the verified package. It +inspects the installed host, opens an owned writable state connection to the +existing biometrics database, and constructs the package's own `ShadowStore`. +The state connection uses an explicit timeout no greater than the saved execution +policy and the same file-replacement guard as read-only capture connections. + +The runner selects the exact installed qualification, builds the observed +hardware/artifact identity collector, supplies the actual shared piezo receive +clock, and invokes the loaded package's `runProfiledShadowAttempt`. Every identity +check also verifies the writable state database still refers to the owned file. +The state handle closes after completion, cancellation or any startup/execution +failure. Capture handles remain read-only; only the store owns shadow writes. + +Service code supplies trusted installation paths, storage policy and timeout; +operations identify an existing run and saved candidate revision. This entry point +does not register background scheduling or public routes, and does not establish +cross-process orphan recovery or concurrent resource admission. Those lifecycle +requirements and current-build qualification remain before production enablement. + +## Host-crash worker termination + +The Node supervisor now passes its own PID to the fixed Python bootstrap. Before +resource-limit setup or numerical imports, the bootstrap requires Linux and arms +`PR_SET_PDEATHSIG` with `SIGKILL`. It checks the expected parent before and after +arming, so a host that dies during startup cannot leave an unprotected worker +processing frames. Missing ctypes/libc support or kernel refusal stops startup. +The expected PID is launch metadata, not a request field. + +A real Linux test launches a detached worker, waits for it to arm protection, +kills its launcher and verifies that the worker terminates. Existing CPU and +address-space kernel tests also pass. This is process-local protection, not a +process-group inheritance guarantee: Linux clears it in forked children. The +shipped numerical worker does not launch candidate subprocess trees. Parent-side +group cleanup remains required, along with durable attempt recovery and resource +admission across concurrent service processes. + +## Kernel-held replay/shadow execution slot + +The shadow bootstrap and waveform replay CLI take a nonblocking exclusive flock on the fixed +`/run/sleepypod-shadow.lock` before numerical imports and holds its descriptor for +the worker lifetime. A second numerical worker fails startup even if its +SQLite lease was granted after a paused worker's lease expired. File creation is +private, no-follow and non-truncating; nonregular, linked, nonempty, wrong-owner or +publicly accessible files are refused. No request/CLI argument selects the path. + +The runtime file is never unlinked during operation. Descriptor close or process +death releases kernel ownership without stale-file cleanup. A real Linux test +verifies exclusion while the owner is stopped with SIGSTOP and acquisition after +SIGKILL/reaping, including the actual replay CLI blocking a shadow contender. +Both entrypoints use the same shared implementation, included in their build +identities. The legacy runtime path is retained for shadow upgrade compatibility. +Build-description probes do not take the slot because they do not run numerical +processing. Lock contention fails startup rather than waiting with an active lease. + +This serializes shipped waveform replay and shadow workers sharing the runtime +path. It does not serialize Node stage replay or archive collection, replace +qualified aggregate host admission, or prove expired-attempt process cleanup. +Before enabling a new installation, stop/reap old replay workers that predate +this lock; they cannot retroactively acquire it. Never delete the lock file to +force admission while any worker may still own its inode. + +## Worker-local wall deadline + +The supervisor passes the saved `wallMs` budget to the Python bootstrap, which +arms `ITIMER_REAL` before numerical imports. It installs the default terminating +SIGALRM action and explicitly unblocks that signal. This supplements the parent's +wall timer, so an idle worker does not depend on Node's event loop to enforce its +elapsed-time budget. No alternate production timeout is selected by the worker. + +The real Linux test starts with SIGALRM blocked, arms a 100 ms deadline, sleeps, +verifies signal termination and then reacquires the execution lock. Parent-side +process-group cleanup remains necessary; a process explicitly stopped by SIGSTOP +cannot handle ordinary pending signals until resumed, and the lock continues to +exclude competing shadow workers in that condition. + +## Bounded expired-run reconciliation + +`ShadowStore.reconcileExpired` scans at most 100 runs per immediate transaction +using a primary-key cursor, including future and terminal rows so maintenance +cannot get stuck on the first unexpired window. It validates candidate plans and +compares state, attempt and lease when changing authority. A corrupt plan rolls +back the batch. Unexpired replacement attempts and terminal evidence are preserved; +final-drain authority remains valid until its lease/deadline expires. + +Expired running attempts become failed with +`shadow_lease_expired_cleanup_unconfirmed`. This revokes append/renew authority +without claiming that an OS process was reaped. Retained samples and attempt IDs +remain available, reports flag unconfirmed cleanup, and the kernel slot continues +to exclude overlapping shadow workers. Pending runs whose observation window was +missed become failed with `shadow_observation_window_missed`; reports do not call +that a processor failure or fabricate observations. + +`reconcileInstalledShadowRuns` exposes this maintenance through the loaded package's +store with owned state-handle cleanup and artifact/database guards. It starts no +Python probe, worker or hardware operation and needs no admission qualification. +Periodic service scheduling must retain the returned cursor until a sweep ends, +then start another sweep. Recovery revokes authority; process cleanup confirmation +and shared replay/shadow resource ownership are separate lifecycle requirements. + +## Application service ownership + +`startShadowService` creates one `globalThis` service using explicit installation, +storage and maintenance policy. Repeated starts reuse the same configuration; +configuration changes require stopping/restarting it. The service snapshots its +configuration, runs bounded recovery pages at the configured cadence, retains its +cursor after a failed transaction, and restarts the scan after a completed sweep. +Recovery continues during a prospective attempt. + +Only one local attempt may execute. Cancellation and shutdown abort that attempt +and await the installed runner's cleanup before releasing its slot. Application +shutdown calls `stopShadowService` before database closure. The database lease and +Linux worker lock still enforce authority across processes; this local owner does +not replace them or serialize replay workers. + +Application instrumentation starts this service after migrations and hardware +initialization when explicit deployment configuration is present. CI skips it +along with hardware initialization. No production resource thresholds or +qualification evidence are inferred by this service. The inspection, creation +and cancellation APIs and browser workflow use this service as described below. + +### Deployment configuration + +Set `SLEEPYPOD_BIOMETRICS_SHADOW_SERVICE` to a JSON object conforming to +`shadowServiceConfigurationSchema` in `startup.ts`. Only `planTemplates` is optional; +unknown fields and input larger than 16 KiB are rejected. + +| Field | Meaning | +| --- | --- | +| `version` | Exactly `shadow-service-configuration-v1` | +| `pythonPath` | Absolute path to the installed locked Python interpreter | +| `modulesPath` | Absolute path to the installed shared Python modules | +| `presenceFile` | Absolute path to the producer's committed presence file | +| `storagePolicy` | Positive integer `maximumRuns`, `maximumPoints`, `maximumBytes` quotas | +| `busyTimeoutMs` | SQLite wait limit, 1–1000 ms, also bounded by each saved execution policy | +| `recoveryIntervalMs` | Recovery and scheduling cadence, 10–300000 ms | +| `recoveryMaximumRuns` | Maximum rows per recovery/scheduling page, 1–100 | +| `planTemplates` | Optional array of up to 8 uniquely named preparation templates; no templates are inferred | + +These are schema bounds, not recommended pod budgets. Choose values from measured +qualification evidence. The two database paths come from the application's open +SQLite databases, and the Node host comes from the fixed packaged build location; +neither can be overridden in this configuration. Missing configuration does not +start the service. Invalid configuration, nonpersistent databases or a missing +package leave it unavailable with a generic diagnostic, without exposing paths or +environment values. Setting valid JSON never creates qualification evidence: +every attempt must still match the installed registry and pass fresh admission. + +### Durable prospective scheduling + +Plans now include `candidateRevisionId`: a saved revision UUID or explicit `null` +for the shipped default. This identity is covered by the plan hash and idempotency +check. Legacy plans remain readable, but the installed execution boundary rejects +missing or mismatched revision identities. Profile resolution still validates the +saved configuration hashes and both acknowledged physical-channel baselines. + +The service scans bounded primary-key pages through the installed package's store +at its maintenance cadence, with a separate scheduling cursor. It selects only +pending executable plans with a saved revision whose observation window is open. +Future plans are revisited on later sweeps; terminal, running and legacy plans are +skipped. Each selected run goes through ordinary qualification, admission and lease +claim checks. A selection is not a reservation and cannot bypass cross-process +authority. Scheduled attempts are serialized locally while recovery continues. +Errors are observed and reported without disabling subsequent sweeps. + +After restart, pending plans remain schedulable from their durable identity. Running +attempts are never resumed by this queue; expired ones retain their samples and are +failed by recovery. Scan cadence and queue size may delay a start, so coverage is +measured from actual samples and missed windows are recorded explicitly. + +## Inspection and cancellation API + +The `biometricShadow` tRPC router and corresponding LAN REST endpoints expose: + +- `GET /api/biometric-tuning/shadow-runs`: required side, optional `afterId` + primary-key cursor and `maximumRuns` (1–50, default 20). +- `POST /api/biometric-tuning/shadow-runs`: a complete proposed `plan` and + `idempotencyKey`. The plan must include candidate revision (or explicit default), + both active generations/hashes, execution policy and prospective window. +- `GET /api/biometric-tuning/shadow-runs/{id}`: integrity-checked metadata. +- `POST /api/biometric-tuning/shadow-runs/{id}/stop`: durable stop followed by + awaited local attempt cancellation. Repeated calls preserve terminal evidence. +- `GET /api/biometric-tuning/shadow-runs/{id}/report`: aggregate health, coverage, + pairing, resource receipt and evidence hashes. +- `POST /api/biometric-tuning/shadow-runs/{id}/compare`: the same report with an + explicit `reference` import matching the saved sleeper and side. Submitted + reference measurements are compared without persisting an import. + +Inspection and stop use the application's database and remain available without +execution configuration, an installed worker, current health or qualification. +A store opened without storage policy cannot create, claim or append observations. +Metadata includes hashes, window, state, counts and saved candidate identity, but +not waveforms or deployment paths. Legacy missing revision identity is explicitly +distinguished from the shipped default through `candidateIdentitySaved`. + +Stopping clears database authority before waiting for local worker cleanup. The +response's `localAttemptSettled` indicates whether a locally owned attempt was +awaited; it does not claim that another process's worker was reaped. Retained +samples and completed/failed results are preserved. + +Reports execute in the shipped coordinator process's bounded read lane, leaving +the application's event loop free. They require a complete integrity-checked +snapshot of at most 100000 points and 32 MiB; larger evidence is refused rather +than sampled. The coordinator also enforces its 64 KiB request, 17 MiB response, +384 MiB Node heap and wall-clock limits. These bounds are not claims of measured +pod resource qualification. Reports require the installed coordinator package but +do not launch a waveform processor or require execution qualification. + +The public response has an explicit schema. Reference summaries include the +shared normalized import content hash; the report hash covers that summary, +snapshot identity and comparison results. No raw samples or submitted reference +annotations are returned. Agreement remains descriptive, never activation +authorization or an independent holdout claim. + +MCP uses these same router operations: `inspect_biometric_calibration` actions +`shadow_runs`, `shadow_run`, `shadow_report` and `shadow_compare` are read-only; +`stop_biometric_shadow` is an explicit idempotent mutation. +`schedule_biometric_shadow` submits a proposed plan through the same creation +operation. + +The calibration screen's Observations tab lists runs for the selected side, +shows UTC windows and saved identities, and can record a stop request. Detail +polling ends for terminal runs; list polling continues to discover new work. +Reports are opened on request and refreshed explicitly rather than recomputed +every status poll. They distinguish unavailable coverage/resource receipts from +zero values and show operational issues without implying validated improvement. +The report UI also compares pasted CSV/JSON reference records for the run's saved +side and sleeper identity. It checks the serialized 64 KiB coordinator request +bound, invalidates old results on edits, and prevents overlapping comparisons. +Results show accepted/rejected counts, outside-window exclusions, alignment +offset/tolerance, shared reference pairs, shared-support MAE, each output's +reference coverage and lost/added observation timestamps. Missing scores remain +unavailable; coverage loss and incompatible definitions remain visible. The +comparison is a fresh snapshot, identified separately from the earlier health +report, and does not persist a reference import. Browser verification and +translation-catalog completion remain release work. + +### Creation admission + +Submitted hashes, generations and policy are assertions, not authority. Creation +loads the verified installed package, checks its Python identity and unique +installed qualification, resolves the stored candidate and acknowledged active +profiles, and observes current hardware and health. It opens admission resources +without claiming a lease, launching a numerical worker or subscribing to sensor +frames. Final admission runs synchronously inside the creation transaction; +failure leaves no queued run. Handles and health sampling close on every exit. +Execution repeats these checks at the actual observation window. + +Both active generations are frozen in the plan. A same-hash reset/rollback on +either side invalidates subsequent profile resolution. Legacy plans lacking +candidate or generation identity remain inspectable but are not scheduled by the +installed execution path. Exact idempotent retries return existing evidence before +new Python/health probes; changed requests with the same key conflict. Creation +still requires explicit server deployment configuration and the installed package. +There are no caller-supplied executable paths, numerical configs or health flags. + +### Server-side plan preparation + +Deployment `planTemplates` supply a bounded ID/label, qualification hash, maximum +window, frozen execution and monitoring policy, joining limits, selected metrics, +optional IBI window and storage ceilings. Their maximum window plus final drain +must fit the worker wall budget; their point/byte ceilings must fit deployment +storage policy. These are explicit operator-installed values, not production +defaults or proof of qualification. No template ships enabled by default. + +`GET /api/biometric-tuning/shadow-templates` returns only template IDs, labels and +maximum windows. `POST /api/biometric-tuning/shadow-plans/prepare` accepts template +ID, side, sleeper identity, saved candidate revision (or explicit default) and +start/end times. The verified package checks the template against the installed +qualification registry and manifest, then captures acknowledged active generations +and candidate configuration from owned read-only database handles. Both physical +channel config hashes and the active baseline must match the template's evidence. + +The response contains the full frozen plan and hash with status +`prepared_not_admitted`. Preparation opens no waveform source, runs no Python +worker and queues nothing. Fresh installed-build, hardware and health admission +remain mandatory at creation and execution. Changed profiles or generations make +the prepared plan stale rather than silently replacing its inputs. + +MCP exposes `shadow_templates` and `shadow_prepare` through the read-only +`inspect_biometric_calibration` tool. Scheduling remains the separate explicit +`schedule_biometric_shadow` operation with its immutable retry key. + +The Observations UI now provides template selection, explicit default or saved +candidate selection with profile pagination, sleeper identity and UTC window +inputs. It preserves exact sleeper IDs and subsecond timestamps. Preparation is +a separate action from scheduling, and edits invalidate the prepared review. +The review shows the server-captured window, generations, candidate and plan hash. +No template is invented when deployment configuration is missing. + +Scheduling freezes the prepared plan and a random request key. After an uncertain +response, the open form keeps those inputs locked and retries the same request +without re-preparing it. Explicit bad-request or quota rejection unlocks editing; +confirmed saves expose a separate new-plan action and refresh the run list. A +saved request is never presented as proof that execution started. Retry state is +saved before sending in same-origin, per-side session storage, bounded to 64 KiB. +Reloading this tab restores the exact plan and key without preparing or sending +automatically, even if template/profile discovery is unavailable. Confirmed saves +and definite rejections clear the record. A failed storage write prevents sending; +invalid or inaccessible recovery blocks new scheduling while preserving the record. +Closing the tab, clearing browser data, or moving to another origin does not carry +recovery forward. The record contains plan metadata, not reference rows or waveforms. + + +### Unconfirmed local cleanup + +A process-group kill refusal or process cleanup confirmation deadline raises +`SHADOW_CLEANUP_UNCONFIRMED`, preserving the original worker failure as its cause. +This result takes precedence over an earlier cancellation or processing error; +the attempt writes `shadow_worker_cleanup_unconfirmed` when its fenced state +still permits a failure update. Existing terminal operator-stop evidence is not +rewritten. Reports label this failure as unconfirmed cleanup. + +The local service quarantines numerical scheduling after that result, while +continuing bounded database recovery. Local cancellation returns false instead +of claiming settlement; ordinary task failures with settled cleanup still release +the slot. The marker is recognized across the separately packaged host boundary. +This is local service state, not durable proof of OS cleanup: restarting a host +must not be treated as verification that old processes are gone. + +The supervisor now requires both the direct-child close event and `ESRCH` from +a signal-0 probe of the detached process group within a five-second deadline. +Surviving members, zombies, or persistent permission/observation failures leave +cleanup unconfirmed. Transient unknown observations are retried, never counted +as absence. No probe signals PID/group zero. These checks cover the original +detached group; they cannot prove cleanup of descendants that deliberately escape +into another group. Shipped numerical code must not create such descendants. +The shared kernel lock remains required across hosts. + + +### Browser demo + +The demo implements all eight observation procedures in browser memory. Its +explicitly labeled synthetic heart-rate template supports preparation, scheduling, +exact idempotent retries, side-specific paginated inspection, stopping, reporting +and reference comparison. Plans are accepted only after preparation in this demo +session, and changed profile generations invalidate an unsubmitted plan. Bounds +are 100 prepared plans, 20 runs and 120 samples per run over at most ten minutes. + +Synthetic active readings repeat 60/61/62 bpm; candidate readings are one bpm +lower and omit every fourth value. They do not execute the selected algorithm. +The demo shares production report calculations, normalization and reference +alignment, so missing coverage is penalized and incompatible records remain +unscored. Reports include a visible simulation issue, no worker resource receipt, +and no qualification or activation authorization. Synthetic plan identities are +not installed in a production registry. No waveform, hardware, DB or network +adapter is imported into the browser demo graph. + +A stop freezes saved synthetic samples. Repeated stop preserves terminal state; +no local process settlement is claimed because the demo has no process. Reloading +resets demo state. A scheduling retry from a previous browser session receives an +explicit bad-request result instead of silently creating a replacement run. + + +### Partial resource evidence + +After supervising cleanup, the child host reports a bounded snapshot even when +processing, input, cancellation or cleanup fails. The last protocol-validated +health reply is retained; it may be absent or stale and is not final CPU usage. +Input bytes count attempted serialized commands; output and diagnostic bytes +count observed transport chunks. RSS is sampled process-group RSS, not an exact +kernel peak; no successful sample is represented by null, never zero. Over-budget +observations remain evidence rather than failing completed-receipt validation. +The observation gap includes the unsampled cleanup tail. + +Completed attempts retain the original `biometric-shadow-receipt-v1` format. +Failed/stopped attempts use `biometric-shadow-partial-receipt-v1` in the same +hash-protected receipt columns, tied to run, plan and attempt. Attachment requires +the exact terminal attempt, cannot change its state/reason/end time, and cannot +replace evidence. An identical retry is idempotent. A later receipt timestamp is +allowed because an operator can stop before process cleanup settles. Old attempts +cannot annotate replacement attempts or completed runs. Missing/rejected writes +remain failures; a cleanup-unconfirmed error still quarantines the host even if +storing its receipt also fails. A host crash can leave no receipt; recovery does +not fabricate measurements or proof of cleanup. + +Reports expose partial receipts through the existing bounded report route and +show a partial-evidence notice; missing RSS is unavailable. Unconfirmed cleanup +is an issue even when an operator's stop reason remains authoritative. This does +not qualify a candidate, prove exact resource peaks, or authorize activation. + + +### Shared application background capacity + +`runInstalledShadowAttempt` and the coordinator supervisor reserve the same +`globalThis` work slot before starting probes, processes, scans or a shadow claim. +The coordinator's exhaustive task/lane map is shared with its queue: stage and +waveform execution, dataset construction/release, experiment creation, RAW +inspection and artifact cleanup are work. One separately reserved read slot keeps +reports available. The existing queue still bounds requests and wait times; +capacity conflicts return `RESOURCE_LIMIT` before starting work. A busy shadow run +remains pending and the service retries it on a later scan; no historical samples +are invented if its window is ultimately missed. + +Ownership lasts until the supervisor settles cleanup. Coordinator completion now +confirms disappearance of the original detached process group after child close, +using the same bounded signal-zero check as shadow cleanup. Unconfirmed cleanup +quarantines that lane instead of admitting replacement work. Ordinary errors +release ownership; cancellation alone does not. Read diagnostics remain available +when the work lane is quarantined. There is no API to clear quarantine: verify OS +cleanup before restarting the application. + +This gate bounds the application process across duplicate module bundles; it is +not an OS-wide lock. The existing private kernel flock still excludes numerical +replay and shadow workers across processes. Deployment must retain a single +application owner and retire old worker processes before startup. Independently +launched Node coordinators in another application process do not share this gate. +Production qualification must measure live processing plus one read process and +one admitted work task, including peak RSS, ingestion delay and storage reserves; +this concurrency bound does not replace per-worker or live-health limits. diff --git a/src/biometrics/shadow/active-database.ts b/src/biometrics/shadow/active-database.ts new file mode 100644 index 000000000..496b6d96a --- /dev/null +++ b/src/biometrics/shadow/active-database.ts @@ -0,0 +1,43 @@ +import Database from 'better-sqlite3' +import { lstatSync } from 'node:fs' +import { z } from 'zod' + +/** Trusted host path, never an RPC path. Keep ownership separate from source + * subscriptions so shutdown can unsubscribe before closing the connection. */ +function openOwnedShadowDatabase(path: string, options: { busyTimeoutMs: number }, readonly: boolean) { + const timeout = z.number().int().min(1).max(1000).parse(options.busyTimeoutMs) + const identity = () => { + const stat = lstatSync(path, { bigint: true }) + if (!stat.isFile() || stat.isSymbolicLink()) throw new Error('Shadow active database must be a regular file') + return `${stat.dev}:${stat.ino}` + } + const expected = identity() + const db = new Database(path, { readonly, fileMustExist: true, timeout }) + let closed = false + const close = () => { + if (closed) return + closed = true + db.close() + } + const assertCurrent = () => { + if (closed || !db.open || identity() !== expected) throw new Error('Shadow active database was closed or replaced') + } + try { + assertCurrent() + } + catch (error) { + close() + throw error + } + return { db, assertCurrent, close } +} + +export function openShadowActiveDatabase(path: string, options: { busyTimeoutMs: number }) { + return openOwnedShadowDatabase(path, options, true) +} + +/** The installed runner owns this writable handle for separate shadow tables. + * The store, not this connection wrapper, enforces the canonical-write boundary. */ +export function openShadowStateDatabase(path: string, options: { busyTimeoutMs: number }) { + return openOwnedShadowDatabase(path, options, false) +} diff --git a/src/biometrics/shadow/active-reader.ts b/src/biometrics/shadow/active-reader.ts new file mode 100644 index 000000000..c8084afcc --- /dev/null +++ b/src/biometrics/shadow/active-reader.ts @@ -0,0 +1,95 @@ +import type Database from 'better-sqlite3' +import { z } from 'zod' +import { biometricManifest, manifestHash, validateConfig } from '../algorithms/registry' +import { configHash, dataHash } from '../algorithms/canonical' +import { shadowActiveReadingSchema } from './worker-sink' + +const integer = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) +const optionsSchema = z.object({ side: z.enum(['left', 'right']), startMs: integer, endMs: integer, + activeConfigHash: z.string().regex(/^[a-f0-9]{64}$/), maximumRows: integer.min(1).max(100) }).strict() +const rowSchema = z.object({ id: integer.positive(), side: z.enum(['left', 'right']), timestamp: integer, + heart_rate: z.number().finite().nullable(), hrv: z.number().finite().nullable(), breathing_rate: z.number().finite().nullable(), + processing_run_id: z.string().uuid(), run_side: z.enum(['left', 'right']), kind: z.literal('live'), + config_hash: z.string(), bundle: z.string(), input_identity: z.string(), started_at_ms: integer, ended_at_ms: integer.nullable(), + quality_count: z.literal(1), quality_score: z.number().min(0).max(1), quality_side: z.enum(['left', 'right']), quality_timestamp: integer, +}) +const bundleSchema = z.object({ schemaVersion: z.literal(1), manifestHash: z.string(), resolved: z.unknown(), + generation: integer, revisionId: z.string().uuid().nullable(), components: z.unknown(), sourceHashes: z.unknown() }).strict() +const inputSchema = z.object({ sourceSide: z.enum(['left', 'right']), timestampBasis: z.literal('processing_clock'), + rawIdentity: z.literal('not_retained'), sensorCalibration: z.literal('not_consumed') }).strict() + +/** Poll an already-open read-only connection. Only observations inserted after + * construction are eligible; historical rows are never replayed as live output. + * The caller controls cadence and must fail its attempt on any reader error. */ +export class ShadowActiveReader { + private readonly options: z.infer + private cursor: number + private anchorHash: string | null + + constructor(private readonly db: Database.Database, options: unknown, private readonly now = Date.now) { + this.options = optionsSchema.parse(options) + if (this.options.startMs >= this.options.endMs || this.options.endMs - this.options.startMs > 7 * 86400000) throw new Error('Invalid active observation window') + if (!db.readonly && db.pragma('query_only', { simple: true }) !== 1) throw new Error('Active capture requires a read-only connection') + const anchor = this.latest() + this.cursor = anchor?.id ?? 0 + this.anchorHash = anchor ? dataHash(anchor) : null + } + + private latest() { + const row = this.db.prepare('SELECT id, side, timestamp, heart_rate, hrv, breathing_rate, processing_run_id FROM vitals ORDER BY id DESC LIMIT 1') + .get() as { id: number } | undefined + if (row) integer.positive().parse(row.id) + return row + } + + poll() { + return this.db.transaction(() => { + if (this.cursor) { + const anchor = this.db.prepare('SELECT id, side, timestamp, heart_rate, hrv, breathing_rate, processing_run_id FROM vitals WHERE id = ?').get(this.cursor) + if (!anchor || dataHash(anchor) !== this.anchorHash) throw new Error('Active observation cursor changed or was removed') + } + const latest = this.latest() + if (!latest || latest.id < this.cursor) { + if (this.cursor) throw new Error('Active observations reset') + return [] + } + const { side, startMs, endMs, maximumRows, activeConfigHash } = this.options + const rows = this.db.prepare(`SELECT v.*, r.side AS run_side, r.kind, r.config_hash, + CASE WHEN length(CAST(r.bundle AS BLOB)) <= 65536 THEN r.bundle END AS bundle, + CASE WHEN length(CAST(r.input_identity AS BLOB)) <= 4096 THEN r.input_identity END AS input_identity, + r.started_at_ms, r.ended_at_ms, + (SELECT COUNT(*) FROM vitals_quality q WHERE q.vitals_id = v.id) AS quality_count, + (SELECT quality_score FROM vitals_quality q WHERE q.vitals_id = v.id LIMIT 1) AS quality_score, + (SELECT side FROM vitals_quality q WHERE q.vitals_id = v.id LIMIT 1) AS quality_side, + (SELECT timestamp FROM vitals_quality q WHERE q.vitals_id = v.id LIMIT 1) AS quality_timestamp + FROM vitals v LEFT JOIN biometric_processing_runs r ON r.id = v.processing_run_id + WHERE v.id > ? AND v.id <= ? AND v.side = ? AND v.timestamp >= ? AND v.timestamp < ? + ORDER BY v.id LIMIT ?`).all(this.cursor, latest.id, side, Math.ceil(startMs / 1000), Math.ceil(endMs / 1000), maximumRows + 1) + if (rows.length > maximumRows) throw new Error('Active observation poll quota exceeded') + const at = this.now() + const readings = rows.map((raw) => { + const row = rowSchema.parse(raw), timestampMs = row.timestamp * 1000 + const bundle = bundleSchema.parse(JSON.parse(row.bundle)), input = inputSchema.parse(JSON.parse(row.input_identity)) + validateConfig(bundle.resolved) + if (!Number.isSafeInteger(timestampMs) || timestampMs > at || timestampMs + 999 < row.started_at_ms + || (row.ended_at_ms !== null && timestampMs >= row.ended_at_ms) + || row.side !== row.run_side || row.quality_side !== row.side || row.quality_timestamp !== row.timestamp + || row.config_hash !== activeConfigHash || configHash(bundle.resolved) !== activeConfigHash + || bundle.manifestHash !== manifestHash || configHash(bundle.components) !== configHash(biometricManifest.components) + || configHash(bundle.sourceHashes) !== configHash(biometricManifest.sourceHashes)) throw new Error('Active observation provenance mismatch') + return shadowActiveReadingSchema.parse({ side, timestampMs, configHash: activeConfigHash, inputHash: null, + provenance: { processingRunId: row.processing_run_id, sourceSide: input.sourceSide, generation: bundle.generation, manifestHash: bundle.manifestHash }, + readings: [ + { metric: 'heart_rate', unit: 'bpm', windowMs: null, value: row.heart_rate, quality: row.quality_score }, + { metric: 'breathing_rate', unit: 'breaths/min', windowMs: null, value: row.breathing_rate, quality: row.quality_score }, + { metric: 'window_ibi_variability_v1', unit: 'ms', windowMs: 300000, value: row.hrv, quality: row.quality_score }, + ] }) + }) + // Advance only after the entire bounded snapshot validates. Unknown, + // malformed or substituted provenance can never be skipped silently. + this.cursor = latest.id + this.anchorHash = dataHash(latest) + return readings + })() + } +} diff --git a/src/biometrics/shadow/admission.ts b/src/biometrics/shadow/admission.ts new file mode 100644 index 000000000..f7bd1f887 --- /dev/null +++ b/src/biometrics/shadow/admission.ts @@ -0,0 +1,91 @@ +import { constants, openSync, closeSync, fstatSync, readSync } from 'node:fs' +import { dirname, join } from 'node:path' +import { z } from 'zod' +import { dataHash } from '../algorithms/canonical' +import { manifestHash } from '../algorithms/registry' +import { admissionPolicySchema, evaluateAdmission } from '../backtest/admission' +import { shadowPlanSchema } from './store' + +const hash = z.string().regex(/^[a-f0-9]{64}$/) +const integer = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) +export const shadowQualificationSchema = z.object({ qualificationId: z.string().min(1).max(200), + manifestHash: hash, engineBuildHash: hash, pythonBuildHash: hash, hardwareId: z.string().min(1).max(200), + sampleRateHz: z.literal(500), activeConfigHash: hash, executionPolicyHash: hash, + fixtureEvidenceHash: hash, podEvidenceHash: hash, heldOutEvidenceHash: hash, + contextEvidence: z.literal('host_observations_not_live_processor_cache_attestation'), + maximumHardwareObservationAgeMs: integer.positive().max(300000), + maximumRuntimeHeartbeatAgeMs: integer.positive().max(300000), healthPolicy: admissionPolicySchema, +}).strict().refine(entry => entry.healthPolicy.qualificationId === entry.qualificationId, 'Qualification policy identity mismatch') +export const shadowQualificationRegistrySchema = z.object({ version: z.literal('biometric-shadow-qualifications-v1'), + entries: z.array(shadowQualificationSchema).max(32) }).strict() +const identitySchema = z.object({ engineBuildHash: hash, pythonBuildHash: hash, hardwareId: z.string().min(1).max(200), sampleRateHz: z.literal(500) }).strict() +const bindingSchema = z.object({ side: z.enum(['left', 'right']), generation: integer, resolvedHash: hash, + appliedHash: hash, appliedGeneration: integer, state: z.literal('applied'), heartbeatAtMs: integer }).strict() + +/** Read release-installed authority beside the trusted worker entrypoint. The + * file has no write API. Missing, ambiguous or changed evidence denies admission. + * Evidence hashes refer to reviewed artifacts; arbitrary supplied JSON is not + * a qualification and callers must never choose this filesystem location. */ +export function readInstalledShadowQualifications(workerPath: string) { + let fd: number | undefined + try { + fd = openSync(join(dirname(workerPath), 'shadow-qualifications.json'), constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK) + const before = fstatSync(fd) + if (!before.isFile() || before.size > 256 * 1024) throw new Error('Invalid shadow qualification file') + const bytes = Buffer.alloc(256 * 1024 + 1) + let length = 0 + while (length < bytes.length) { + const count = readSync(fd, bytes, length, bytes.length - length, null) + if (!count) break + length += count + } + const after = fstatSync(fd) + if (length > 256 * 1024 || before.size !== after.size || before.mtimeMs !== after.mtimeMs) throw new Error('Shadow qualification file changed or exceeded its bound') + return shadowQualificationRegistrySchema.parse(JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes.subarray(0, length)))) + } + catch { throw new Error('Installed shadow qualification unavailable') } + finally { if (fd !== undefined) closeSync(fd) } +} + +/** All observations come from trusted host collectors, never RPC fields. + * Snapshot generation at factory creation prevents a same-hash reset/rollback + * from silently continuing an attempt across a changed active binding. */ +export function installedShadowAdmitter(options: { + workerPath: string + expectedBinding: { side: 'left' | 'right', generation: number, resolvedHash: string } + identity: () => unknown + binding: () => unknown + health: () => unknown + now?: () => number +}) { + const expected = z.object({ side: bindingSchema.shape.side, generation: integer, resolvedHash: hash }).strict().parse(structuredClone(options.expectedBinding)) + const now = options.now ?? Date.now, workerPath = options.workerPath + const identityReader = options.identity, bindingReader = options.binding, healthReader = options.health + const snapshot = (read: () => unknown) => { + const value = read() + if (value && typeof value === 'object' && 'then' in value && typeof value.then === 'function') { + void Promise.resolve(value).catch(() => {}) + throw new Error('Shadow admission observations must be synchronous') + } + return value + } + return (rawPlan: unknown): void => { + const plan = shadowPlanSchema.parse(rawPlan) + if (!plan.execution) throw new Error('Shadow execution policy is missing') + const registry = readInstalledShadowQualifications(workerPath) + const entries = registry.entries.filter(entry => dataHash(entry) === plan.qualificationHash) + if (entries.length !== 1) throw new Error('Shadow qualification missing or ambiguous') + const entry = entries[0], identity = identitySchema.parse(snapshot(identityReader)), binding = bindingSchema.parse(snapshot(bindingReader)), at = now() + if (!Number.isSafeInteger(at) || at < 0) throw new Error('Invalid shadow admission clock') + if (entry.manifestHash !== manifestHash || entry.engineBuildHash !== plan.engineBuildHash + || entry.engineBuildHash !== identity.engineBuildHash || entry.pythonBuildHash !== identity.pythonBuildHash + || entry.hardwareId !== identity.hardwareId || entry.sampleRateHz !== identity.sampleRateHz + || entry.executionPolicyHash !== dataHash(plan.execution) || entry.activeConfigHash !== plan.activeConfigHash) throw new Error('Shadow qualification does not match installed execution') + if (binding.side !== plan.side || binding.side !== expected.side || binding.generation !== expected.generation + || binding.resolvedHash !== expected.resolvedHash || binding.resolvedHash !== plan.activeConfigHash + || binding.appliedHash !== binding.resolvedHash || binding.appliedGeneration !== binding.generation + || at < binding.heartbeatAtMs || at - binding.heartbeatAtMs > entry.maximumRuntimeHeartbeatAgeMs) throw new Error('Shadow active binding changed, unapplied or stale') + const decision = evaluateAdmission(snapshot(healthReader), entry.healthPolicy, at) + if (!decision.admitted) throw new Error(`Shadow host admission refused: ${decision.reasons.join(',')}`) + } +} diff --git a/src/biometrics/shadow/attempt.ts b/src/biometrics/shadow/attempt.ts new file mode 100644 index 000000000..ae04fcb09 --- /dev/null +++ b/src/biometrics/shadow/attempt.ts @@ -0,0 +1,234 @@ +import type { ShadowPartialReceipt } from './receipt' +import type { ShadowExecutionPolicy } from './execution-policy' +import { isShadowCleanupError, ShadowCleanupError } from './cleanup-error' +import { z } from 'zod' +import { configHash, dataHash } from '../algorithms/canonical' +import { validateConfig } from '../algorithms/registry' +import type { ShadowStore } from './store' +import { type ShadowLease } from './store' +import { ShadowStreamJoin } from './stream-join' +import { shadowActiveReadingSchema } from './worker-sink' +import { runShadowChild } from './worker-child' + +type Plan = ReturnType['plan'] +export interface ShadowSources { + frames: AsyncIterable + /** Poll committed active observations before sealing join timestamps. */ + flush?(): void + /** Unsubscribe both streams synchronously, including on failed startup. */ + close(): void +} +export interface ShadowAttemptHost { + location: { pythonPath: string, modulesPath: string } + configs: { left: unknown, right: unknown } + /** Trusted synchronous check of installed qualification/build, active binding, + * resource admission and source health. Rechecked throughout the attempt. */ + admit(plan: Plan): void + /** Must register bounded source adapters; the frame iterator is stopped at + * collection end while active readings may arrive during the drain window. */ + openSources(input: { execution: ShadowExecutionPolicy, signal: AbortSignal, collectionEndsAtMs: number, fail: () => void, active: (reading: unknown) => void }): ShadowSources +} + +/** Execute one fenced attempt. A restart is a new claim and fresh processor; + * historical pending frames are never synthesized. The host owns crash recovery + * and must terminate any orphan process before admitting a replacement. */ +export async function runShadowAttempt(store: ShadowStore, runId: string, host: ShadowAttemptHost, signal?: AbortSignal) { + const plan = store.get(runId).plan + if (!plan.joining || !plan.execution) throw new Error('Shadow attempt requires frozen join and execution policies') + const execution = structuredClone(plan.execution) + const joining = plan.joining + z.number().int().min(1000).max(3600000).parse(execution.leaseMs) + z.number().int().positive().max(Math.min(1000, Math.floor(execution.leaseMs / 3), plan.monitoring.heartbeatStaleMs)).parse(execution.heartbeatMs) + const configs = structuredClone(host.configs) + const configHashes = { left: configHash(validateConfig(configs.left)), right: configHash(validateConfig(configs.right)) } + if (configHashes[plan.side] !== plan.candidateConfigHash || dataHash(configHashes) !== dataHash(execution.configHashes)) throw new Error('Shadow candidate differs from frozen plan') + if (signal?.aborted) throw new Error('Shadow attempt cancelled before claim') + const admit = (frozen: Plan) => { + const result: unknown = host.admit(structuredClone(frozen)) + if (result !== undefined) { + void Promise.resolve(result).catch(() => {}) + throw new Error('Shadow admission must complete synchronously') + } + } + const lease: ShadowLease = store.claim(runId, execution.leaseMs, admit) + const controller = new AbortController() + let sources: ShadowSources | undefined, worker: ReturnType | undefined + let failure: Error | undefined, terminalState: 'failed' | 'stopped' = 'failed', accepting = true + let rejectFailure!: (error: Error) => void + const failed = new Promise((_resolve, reject) => { + rejectFailure = reject + }) + void failed.catch(() => {}) + const stop = (reason: string, state: 'failed' | 'stopped' = 'failed') => { + if (failure) return + failure = new Error(reason) + terminalState = state + accepting = false + controller.abort() + rejectFailure(failure) + } + const abort = () => stop('shadow_cancelled', 'stopped') + signal?.addEventListener('abort', abort, { once: true }) + if (signal?.aborted) abort() + let endCollection!: () => void, endDrain!: () => void + const collectionEnded = new Promise((resolve) => { + endCollection = resolve + }) + const drained = new Promise((resolve) => { + endDrain = resolve + }) + const collectionTimer = setTimeout(() => endCollection(), Math.max(0, plan.endMs - Date.now())) + const drainTimer = setTimeout(() => endDrain(), Math.max(0, plan.endMs + joining.maximumLatenessMs - Date.now())) + // Stop before the store's exclusive lease deadline so failure can be fenced. + const deadline = setTimeout(() => stop('shadow_drain_deadline'), Math.max(0, plan.endMs + joining.drainGraceMs - Date.now() - 1)) + let heartbeat: ReturnType | undefined + let resources: ShadowPartialReceipt | undefined + const recordResources = (cause: unknown) => { + try { + if (resources) store.recordPartialReceipt(lease, resources) + } + catch (error) { + const combined = new AggregateError([cause, error], 'Shadow partial resource evidence could not be stored') + if (isShadowCleanupError(cause)) throw new ShadowCleanupError(combined) + throw combined + } + } + let receipt: Awaited> | undefined + try { + const join = new ShadowStreamJoin(store, lease) + const check = () => { + if (failure) throw failure + admit(plan) + store.renew(lease, execution.leaseMs) + } + const advance = () => { + const flushed: unknown = sources?.flush?.() + if (flushed !== undefined) { + void Promise.resolve(flushed).catch(() => {}) + throw new Error('Shadow source flush must complete synchronously') + } + if (failure) throw failure + if (join.advance(Date.now()).stopped) stop('storage_quota', 'stopped') + } + heartbeat = setInterval(() => { + try { + check() + advance() + } + catch { stop('shadow_admission_or_lease_lost') } + }, execution.heartbeatMs) + sources = host.openSources({ execution: structuredClone(execution), signal: controller.signal, collectionEndsAtMs: plan.endMs, + fail: () => { + if (accepting) stop('shadow_source_failed') + }, + active: (reading) => { + if (!accepting) return + try { + check() + const active = shadowActiveReadingSchema.parse(reading) + if (active.timestampMs > Date.now()) throw new Error('shadow_future_active_reading') + join.active(active) + } + catch { stop('shadow_active_stream_failed') } + } }) + const source = sources.frames[Symbol.asyncIterator]() + async function* frames() { + let sequence = 0 + try { + while (Date.now() < plan.endMs) { + const next = await Promise.race([Promise.resolve().then(() => source.next()), collectionEnded.then(() => ({ done: true as const, value: undefined })), failed]) + if (next.done) { + if (Date.now() < plan.endMs) throw new Error('shadow_source_ended_early') + break + } + check() + const frame = z.object({ type: z.literal('frame'), observedAtMs: z.number().int().min(plan.startMs).max(plan.endMs - 1) }).passthrough().parse(next.value) + if (frame.observedAtMs > Date.now()) throw new Error('shadow_future_arrival') + yield { ...frame, sequence: sequence++ } + } + } + finally { + if (source.return) void Promise.resolve().then(() => source.return?.()).catch(() => {}) + } + } + worker = runShadowChild(host.location, { schema: 'biometric-shadow-worker-v1', ...lease, runId, + configs, configHashes, side: plan.side, startMs: plan.startMs, endMs: plan.endMs, + maximumRecords: execution.maximumRecords, maximumOutputs: execution.maximumOutputs, maximumCpuMs: execution.maximumCpuMs, + maximumInputBytes: execution.maximumInputBytes, maximumOutputBytes: execution.maximumOutputBytes }, frames(), (output) => { + check() + join.candidate(output) + }, { budgets: structuredClone(execution.budgets), signal: controller.signal, onResources: (value) => { resources = value } }) + receipt = await Promise.race([worker, failed]) + await Promise.race([drained, failed]) + check() + advance() + if (failure) throw failure + accepting = false + } + catch (error) { + stop(error instanceof Error ? error.message.slice(0, 500) : 'shadow_attempt_failed') + } + finally { + accepting = false + controller.abort() + clearInterval(heartbeat) + clearTimeout(collectionTimer) + clearTimeout(drainTimer) + clearTimeout(deadline) + signal?.removeEventListener('abort', abort) + try { + const result: unknown = sources?.close() + if (result !== undefined) { + void Promise.resolve(result).catch(() => {}) + stop('shadow_source_cleanup_must_be_synchronous') + } + } + catch { stop('shadow_source_cleanup_failed') } + // The subprocess supervisor bounds its own cleanup, including on abortion. + if (worker) await worker.catch((error: unknown) => { + if (isShadowCleanupError(error)) { + failure = error + terminalState = 'failed' + } + }) + } + if (!failure) { + try { + admit(plan) + } + catch { stop('shadow_admission_changed_before_completion') } + } + if (failure) { + try { + store.finish(lease, terminalState, failure.message) + } + catch { + // The supervisor has settled; cleanup may be unconfirmed. An expired attempt can be marked failed, + // but this compare-and-set cannot touch a replacement or terminal run. + try { + store.failExpired(lease, failure.message) + } + catch { /* Storage failure remains observable through the rejected run. */ } + } + recordResources(failure) + throw failure + } + try { + if (!receipt) throw new Error('Shadow worker receipt missing') + store.finish(lease, 'completed', 'observation_window_complete', receipt) + } + catch (error) { + // A synchronous final read/cleanup can outlast the remaining lease even + // when its admission check succeeds. Cleanup is complete; fence the failure. + try { + try { + store.finish(lease, 'failed', 'shadow_completion_failed') + } + catch { store.failExpired(lease, 'shadow_completion_lease_lost') } + } + catch { /* Preserve the original completion error. */ } + recordResources(error) + throw error + } + return { lease, receipt, run: store.get(runId) } +} diff --git a/src/biometrics/shadow/build.ts b/src/biometrics/shadow/build.ts new file mode 100644 index 000000000..351a86bfc --- /dev/null +++ b/src/biometrics/shadow/build.ts @@ -0,0 +1,36 @@ +import { execFile } from 'node:child_process' +import { tmpdir } from 'node:os' +import { promisify } from 'node:util' +import { z } from 'zod' +import { workerBuildHash } from '../backtest/child' +import { configHash } from '../algorithms/canonical' +import { waveformPythonBuildSchema } from '../backtest/waveform-build' + +const dependencyPaths = z.object({ record: z.string().min(1).max(4096), wheel: z.string().min(1).max(4096) }).strict() +export const shadowPythonBuildSchema = waveformPythonBuildSchema.extend({ + dependencyFiles: z.object({ numpy: dependencyPaths, scipy: dependencyPaths }).strict(), identity: waveformPythonBuildSchema.shape.identity.extend({ + version: z.literal('shadow-python-build-v1'), + }).refine(identity => ['shadow_build.py', 'shadow.py', 'shadow_worker.py', 'shadow_process.py'].every(name => + identity.sources[`common/biometrics/${name}`] !== undefined), 'Shadow source identity incomplete') }) + +/** Fixed shipped probe, exact worker interpreter, bounded output/deadline. Never + * accepts a module name, shell command or source code from a request. */ +export async function probeShadowBuild(location: { pythonPath: string, modulesPath: string }, signal?: AbortSignal) { + const { stdout } = await promisify(execFile)(location.pythonPath, ['-B', '-m', 'common.biometrics.shadow_build'], { + cwd: tmpdir(), env: { NODE_ENV: 'production', PYTHONPATH: location.modulesPath, LANG: 'C.UTF-8', TZ: 'UTC', + OPENBLAS_NUM_THREADS: '1', OMP_NUM_THREADS: '1', MKL_NUM_THREADS: '1', NUMEXPR_NUM_THREADS: '1' }, + encoding: 'utf8', maxBuffer: 65536, timeout: 10000, killSignal: 'SIGKILL', signal, + }).catch(() => { throw new Error('Shadow build identity is unavailable') }) + const result = shadowPythonBuildSchema.parse(JSON.parse(stdout)) + if (configHash(result.identity) !== result.hash) throw new Error('Shadow build identity integrity mismatch') + return result +} + +/** Bind the complete packaged host graph and prospective Python fingerprint. + * The service must execute this bundle, not hash an unrelated unused artifact. */ +export async function shadowEngineBuildIdentity(hostPath: string, location: { pythonPath: string, modulesPath: string }, signal?: AbortSignal) { + const [hostHash, python] = await Promise.all([workerBuildHash(hostPath), probeShadowBuild(location, signal)]) + const identity = { version: 'shadow-engine-build-v1', hostHash, pythonHash: python.hash, + node: { version: process.version, platform: process.platform, arch: process.arch, icu: process.versions.icu ?? null } } + return { hash: configHash(identity), identity, python } +} diff --git a/src/biometrics/shadow/cleanup-error.ts b/src/biometrics/shadow/cleanup-error.ts new file mode 100644 index 000000000..7031654e3 --- /dev/null +++ b/src/biometrics/shadow/cleanup-error.ts @@ -0,0 +1,11 @@ +/** Structural code survives the separately bundled installed host boundary. */ +export class ShadowCleanupError extends Error { + readonly code = 'SHADOW_CLEANUP_UNCONFIRMED' + constructor(cause?: unknown) { + super('shadow_worker_cleanup_unconfirmed', { cause }) + this.name = 'ShadowCleanupError' + } +} +export function isShadowCleanupError(error: unknown): error is Error & { code: 'SHADOW_CLEANUP_UNCONFIRMED' } { + return error instanceof Error && 'code' in error && error.code === 'SHADOW_CLEANUP_UNCONFIRMED' +} diff --git a/src/biometrics/shadow/context.ts b/src/biometrics/shadow/context.ts new file mode 100644 index 000000000..30b04658c --- /dev/null +++ b/src/biometrics/shadow/context.ts @@ -0,0 +1,96 @@ +import { shadowContextPolicySchema as policySchema } from './execution-policy' +import type Database from 'better-sqlite3' +import { constants, openSync, closeSync, fstatSync, readSync } from 'node:fs' +import { z } from 'zod' +import { activeSleeperSides } from '../../lib/singleSleeper' + +const sideSchema = z.object({ side: z.enum(['left', 'right']), away_mode: z.union([z.literal(0), z.literal(1)]) }) +const bedSchema = z.object({ bed_mode: z.enum(['two', 'solo-left', 'solo-right']) }) +const object = (value: unknown): Record | null => value !== null && typeof value === 'object' && !Array.isArray(value) ? value as Record : null +function presence(path: string, at: number, policy: z.infer) { + let fd: number | undefined + try { + fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK) + const before = fstatSync(fd) + if (!before.isFile() || before.size > policy.maximumPresenceBytes || at < before.mtimeMs || at - before.mtimeMs > policy.presenceStaleMs) return null + const bytes = Buffer.alloc(policy.maximumPresenceBytes + 1) + let length = 0 + while (length < bytes.length) { + const count = readSync(fd, bytes, length, bytes.length - length, null) + if (!count) break + length += count + } + const after = fstatSync(fd) + if (length > policy.maximumPresenceBytes || after.size !== before.size || after.mtimeMs !== before.mtimeMs) return null + const state = object(JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes.subarray(0, length)))) + return state?.version === 1 ? state : null + } + catch { return null } + finally { if (fd !== undefined) closeSync(fd) } +} +function occupied(state: Record | null, side: string, at: number, stale: number) { + const row = object(state?.[side]) + if (typeof row?.vitals_presence !== 'boolean' || typeof row.vitals_evidence_ts !== 'number') return null + const age = at - row.vitals_evidence_ts * 1000 + return Number.isFinite(age) && age >= 0 && age <= stale ? row.vitals_presence : null +} +function pumpEvidence(raw: unknown) { + const side = object(raw), nested = object(side?.pump) + if (!side) return false + const values = [...['pumpRpm', 'pump_rpm', 'pumpRPM', 'rpm', 'pumpDuty', 'pump_duty', 'duty'].map(key => side[key]), + ...['rpm', 'duty', 'power'].map(key => nested?.[key])] + return values.some(value => (typeof value === 'number' && Number.isFinite(value)) + || (typeof value === 'string' && value.trim() !== '' && value.length <= 32 && Number.isFinite(Number(value)))) +} + +/** Current committed evidence for this shadow attempt, not an attestation of + * the active Python process's independently cached view. Host qualification + * must account for that timing difference. Missing context stays unknown. */ +export class ShadowContextReader { + private readonly policy: z.infer + private pumpAt: { left: number | null, right: number | null } = { left: null, right: null } + private lastArrival = -1 + constructor(private readonly db: Database.Database, private readonly presencePath: string, + policy: unknown, private readonly assertDatabaseCurrent: () => void) { + this.policy = policySchema.parse(policy) + if (!db.readonly && db.pragma('query_only', { simple: true }) !== 1) throw new Error('Shadow context requires read-only configuration') + } + + private checkIdentity() { + const checked: unknown = this.assertDatabaseCurrent() + if (checked !== undefined) { + void Promise.resolve(checked).catch(() => {}) + throw new Error('Shadow context identity check must be synchronous') + } + } + + snapshot(frame: Record, observedAtMs: number) { + if (!Number.isSafeInteger(observedAtMs) || observedAtMs < 0 || observedAtMs < this.lastArrival) throw new Error('Shadow context arrival clock regressed') + this.lastArrival = observedAtMs + // Replacement is an attempt failure, not merely missing settings. + this.checkIdentity() + let activeSides: ('left' | 'right')[] | null = null + try { + activeSides = this.db.transaction(() => { + const sides = z.array(sideSchema).length(2).parse(this.db.prepare('SELECT side, away_mode FROM side_settings LIMIT 3').all()) + if (new Set(sides.map(row => row.side)).size !== 2) throw new Error('Ambiguous side settings') + const bed = bedSchema.parse(this.db.prepare('SELECT bed_mode FROM device_settings WHERE id = 1').get()) + return activeSleeperSides({ bedMode: bed.bed_mode, + left: { awayMode: sides.find(row => row.side === 'left')?.away_mode }, + right: { awayMode: sides.find(row => row.side === 'right')?.away_mode } }) + })() + } + catch { /* Missing/corrupt settings are unknown, never an invented two-sleeper default. */ } + const state = presence(this.presencePath, observedAtMs, this.policy) + this.checkIdentity() + if (frame.type === 'frzHealth') { + for (const side of ['left', 'right'] as const) { + // An incomplete update invalidates prior knowledge for that side. + this.pumpAt[side] = pumpEvidence(frame[side]) ? observedAtMs : null + } + } + return { activeSides, leftOccupied: occupied(state, 'left', observedAtMs, this.policy.presenceStaleMs), + rightOccupied: occupied(state, 'right', observedAtMs, this.policy.presenceStaleMs), + pumpContextKnown: Object.values(this.pumpAt).every(at => at !== null && observedAtMs - at <= this.policy.pumpStaleMs) } + } +} diff --git a/src/biometrics/shadow/descriptors.ts b/src/biometrics/shadow/descriptors.ts new file mode 100644 index 000000000..61bea5319 --- /dev/null +++ b/src/biometrics/shadow/descriptors.ts @@ -0,0 +1,25 @@ +import { z } from 'zod' +import type { ShadowStore } from './store' + +const integer = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) +const hash = z.string().regex(/^[a-f0-9]{64}$/) +export const shadowRunDescriptorSchema = z.object({ + id: z.string().uuid(), side: z.enum(['left', 'right']), subjectId: z.string(), + state: z.enum(['pending', 'running', 'completed', 'stopped', 'failed']), + planHash: hash, candidateConfigHash: hash, activeConfigHash: hash, + candidateRevisionId: z.string().uuid().nullable(), candidateIdentitySaved: z.boolean(), + startMs: integer, endMs: integer, createdAtMs: integer, endedAtMs: integer.nullable(), + heartbeatAtMs: integer.nullable(), reason: z.string().nullable(), pointCount: integer, byteCount: integer, + workerReceiptAvailable: z.boolean(), +}).strict() +export const shadowRunPageSchema = z.object({ items: z.array(shadowRunDescriptorSchema).max(50), nextCursor: z.string().uuid().nullable() }).strict() + +/** Public metadata excludes storage paths, waveforms and worker transport data. */ +export function describeShadowRun(run: ReturnType) { + return shadowRunDescriptorSchema.parse({ id: run.id, side: run.side, subjectId: run.plan.subjectId, state: run.state, + planHash: run.plan_hash, candidateConfigHash: run.plan.candidateConfigHash, activeConfigHash: run.plan.activeConfigHash, + candidateRevisionId: run.plan.candidateRevisionId ?? null, candidateIdentitySaved: run.plan.candidateRevisionId !== undefined, + startMs: run.plan.startMs, endMs: run.plan.endMs, createdAtMs: run.created_at_ms, endedAtMs: run.ended_at_ms, + heartbeatAtMs: run.heartbeat_at_ms, reason: run.reason, pointCount: run.point_count, byteCount: run.byte_count, + workerReceiptAvailable: run.workerReceipt !== null }) +} diff --git a/src/biometrics/shadow/device-identity.ts b/src/biometrics/shadow/device-identity.ts new file mode 100644 index 000000000..6c0b1338b --- /dev/null +++ b/src/biometrics/shadow/device-identity.ts @@ -0,0 +1,35 @@ +import { z } from 'zod' +import { getDacMonitorIfRunning } from '@/src/hardware/dacMonitor.instance' +import { shadowQualificationSchema } from './admission' +import type { inspectInstalledShadowHost } from './installed-host' + +/** Observe the existing successful status poll only; never start polling or send + * a DAC command. Preserve exact hardware revision without retaining serials. + * The general UI parser's unknown-label fallback is not qualification evidence. */ +export function observedShadowHardware(maximumAgeMs: number, now = Date.now()) { + z.number().int().positive().max(300000).parse(maximumAgeMs) + z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER).parse(now) + const monitor = getDacMonitorIfRunning() + if (!monitor || monitor.getStatus() !== 'running') throw new Error('Shadow hardware observation unavailable') + const observedAtMs = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER).parse(monitor.getLastPollAt()) + if (observedAtMs > now || now - observedAtMs > maximumAgeMs) throw new Error('Shadow hardware observation stale or future') + const status = z.object({ podVersion: z.enum(['H00', 'I00', 'J00']), sensorLabel: z.string().min(1).max(1000) }).parse(monitor.getLastStatus()) + const revision = status.sensorLabel.split('-').reverse().find(segment => /^[A-Z]\d{2}$/.test(segment)) + if (!revision || !/^[HIJ]\d{2}$/.test(revision) || status.podVersion !== `${revision[0]}00`) throw new Error('Shadow sensor hardware revision unavailable or inconsistent') + return { hardwareId: `pod-sensor-v1/${status.podVersion}/${revision}`, sampleRateHz: 500 as const } +} + +/** Called by the application, outside the passive packaged host. Every admission + * observes current hardware and rechecks the loaded Node/Python artifacts. */ +export function installedShadowIdentity(inspection: Awaited>, rawQualification: unknown) { + const qualification = shadowQualificationSchema.parse(rawQualification) + const engineBuildHash = inspection.build.hash, pythonBuildHash = inspection.build.python.hash + const assertCurrent = inspection.assertCurrent + if (engineBuildHash !== qualification.engineBuildHash || pythonBuildHash !== qualification.pythonBuildHash) throw new Error('Shadow installed build differs from qualification') + return () => { + assertCurrent() + const hardware = observedShadowHardware(qualification.maximumHardwareObservationAgeMs) + if (hardware.hardwareId !== qualification.hardwareId || hardware.sampleRateHz !== qualification.sampleRateHz) throw new Error('Shadow hardware differs from qualification') + return { engineBuildHash, pythonBuildHash, ...hardware } + } +} diff --git a/src/biometrics/shadow/execution-policy.ts b/src/biometrics/shadow/execution-policy.ts new file mode 100644 index 000000000..24e43e763 --- /dev/null +++ b/src/biometrics/shadow/execution-policy.ts @@ -0,0 +1,23 @@ +import { z } from 'zod' +const positive = z.number().int().positive().max(Number.MAX_SAFE_INTEGER) +const hash = z.string().regex(/^[a-f0-9]{64}$/) +export const shadowChildBudgetsSchema = z.object({ cpuSeconds: positive.max(3600), addressSpaceBytes: positive, + maximumRssBytes: positive, wallMs: positive.max(7 * 86400000 + 300000), + acknowledgmentMs: positive.max(300000), idleMs: positive.max(300000), + rssIntervalMs: positive.min(50).max(1000), rssTimeoutMs: positive.max(10000), +}).strict() +export const shadowFrameBudgetsSchema = z.object({ maximumFrames: positive.max(1000), maximumBytes: positive.max(32 * 1024 * 1024), + maximumEventAgeMs: positive.max(120000), maximumFutureSkewMs: z.number().int().min(0).max(5000) }).strict() +export const shadowContextPolicySchema = z.object({ presenceStaleMs: positive.max(300000), + pumpStaleMs: positive.max(300000), maximumPresenceBytes: positive.max(256 * 1024) }).strict() + +/** Explicit ceilings, never production defaults. The reviewed qualification + * must match the complete policy, including both physical channel configs. */ +export const shadowExecutionPolicySchema = z.object({ version: z.literal('shadow-execution-policy-v1'), + configHashes: z.object({ left: hash, right: hash }).strict(), + maximumRecords: positive, maximumOutputs: positive, maximumCpuMs: positive, maximumInputBytes: positive, maximumOutputBytes: positive, + budgets: shadowChildBudgetsSchema, leaseMs: positive.min(1000).max(3600000), heartbeatMs: positive.max(1000), + sources: z.object({ pollMs: positive.min(10).max(1000), maximumActiveRows: positive.max(100), + databaseBusyTimeoutMs: positive.max(1000), frames: shadowFrameBudgetsSchema, context: shadowContextPolicySchema }).strict(), +}).strict().refine(policy => policy.heartbeatMs * 3 <= policy.leaseMs, 'Heartbeat must fit the lease') +export type ShadowExecutionPolicy = z.infer diff --git a/src/biometrics/shadow/frame-source.ts b/src/biometrics/shadow/frame-source.ts new file mode 100644 index 000000000..c33a65c07 --- /dev/null +++ b/src/biometrics/shadow/frame-source.ts @@ -0,0 +1,149 @@ +import { shadowFrameBudgetsSchema as budgetsSchema } from './execution-policy' +import { z } from 'zod' +import { subscribeProspectiveFrames } from '../../streaming/prospectiveFrameTap' + +const contextSchema = z.object({ activeSides: z.array(z.enum(['left', 'right'])).max(2).nullable(), + leftOccupied: z.boolean().nullable(), rightOccupied: z.boolean().nullable(), pumpContextKnown: z.boolean(), +}).strict().refine(value => value.activeSides === null || new Set(value.activeSides).size === value.activeSides.length, 'Duplicate active side') +const pumpKeys = ['pumpRpm', 'pump_rpm', 'pumpRPM', 'rpm', 'pumpDuty', 'pump_duty', 'duty'] as const +function pumpValue(value: unknown) { + if (value === null) return null + if (typeof value === 'number' && Number.isFinite(value)) return value + if (typeof value === 'string' && value.length <= 32 && value.trim() !== '' && Number.isFinite(Number(value))) return value + throw new Error('Invalid prospective pump context') +} +function pumpSide(raw: unknown) { + if (raw === undefined) return undefined + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) throw new Error('Invalid prospective pump side') + const source = raw as Record, result: Record = {} + for (const key of pumpKeys) if (source[key] !== undefined) result[key] = pumpValue(source[key]) + if (source.pump !== undefined) { + if (!source.pump || typeof source.pump !== 'object' || Array.isArray(source.pump)) throw new Error('Invalid prospective nested pump') + const pump = source.pump as Record, selected: Record = {} + for (const key of ['rpm', 'duty', 'power']) if (pump[key] !== undefined) selected[key] = pumpValue(pump[key]) + result.pump = selected + } + return result +} +function channel(raw: unknown) { + if (!Array.isArray(raw) || raw.length > 5000) throw new Error('Invalid prospective waveform channel') + const bytes = Buffer.alloc(raw.length * 4) + for (let index = 0; index < raw.length; index++) { + const value = raw[index] + if (!Number.isInteger(value) || value < -2147483648 || value > 2147483647) throw new Error('Invalid prospective waveform sample') + bytes.writeInt32LE(value, index * 4) + } + return bytes.toString('base64') +} +function record(frame: Record) { + const result: Record = { type: frame.type } + if (frame.ts !== undefined) result.ts = z.number().finite().nonnegative().parse(frame.ts) + if (frame.type === 'piezo-dual') { + if (frame.freq !== undefined) result.freq = z.literal(500).parse(frame.freq) + result.left1 = channel(frame.left1) + result.right1 = channel(frame.right1) + } + else { + const left = pumpSide(frame.left), right = pumpSide(frame.right) + if (left !== undefined) result.left = left + if (right !== undefined) result.right = right + } + return result +} + +/** Subscribes to actual live decoded frames. Overflow fails the attempt rather + * than silently dropping evidence or allowing the worker to lag without bound. + * Returned JSON is a private snapshot; raw stream objects are never retained. */ +export function openShadowFrameSource(options: { + startMs: number + endMs: number + signal: AbortSignal + budgets: z.infer + context: (frame: Record, observedAtMs: number) => unknown + fail: () => void +}) { + const budgets = budgetsSchema.parse(options.budgets) + if (!Number.isSafeInteger(options.startMs) || !Number.isSafeInteger(options.endMs) || options.startMs < 0 || options.endMs <= options.startMs || options.endMs - options.startMs > 7 * 86400000 || options.endMs - Date.now() > 7 * 86400000) throw new Error('Invalid prospective source window') + let closed = false, used = false, bytes = 0, previousArrival = options.startMs + const queue: { text: string, bytes: number }[] = [] + let waiter: ((value: IteratorResult) => void) | undefined + let unsubscribe: (() => void) | undefined + const close = () => { + if (closed) return + closed = true + unsubscribe?.() + options.signal.removeEventListener('abort', close) + clearTimeout(endTimer) + queue.length = 0 + bytes = 0 + waiter?.({ done: true, value: undefined }) + waiter = undefined + } + const notifyFailure = () => { + try { + options.fail() + } + catch { /* Isolate consumer errors from the live server's timers. */ } + } + const finishCollection = () => { + const lost = queue.length > 0 + close() + if (lost) notifyFailure() + } + const endTimer = setTimeout(finishCollection, Math.max(0, options.endMs - Date.now())) + if (options.signal.aborted) close() + else { + options.signal.addEventListener('abort', close, { once: true }) + try { + unsubscribe = subscribeProspectiveFrames({ failed: () => { + close() + notifyFailure() + }, frame: (frame, observedAtMs) => { + if (closed) return + if (!Number.isSafeInteger(observedAtMs) || observedAtMs > Date.now()) throw new Error('Invalid prospective arrival clock') + if (observedAtMs < options.startMs || observedAtMs >= options.endMs) return + if (observedAtMs < previousArrival) throw new Error('Prospective arrival clock regressed') + const eventMs = z.number().finite().nonnegative().parse(frame.ts) * 1000 + if (!Number.isFinite(eventMs) || eventMs > Number.MAX_SAFE_INTEGER || observedAtMs - eventMs > budgets.maximumEventAgeMs || eventMs - observedAtMs > budgets.maximumFutureSkewMs) throw new Error('Prospective event time outside freshness policy') + if (queue.length >= budgets.maximumFrames) throw new Error('Prospective frame queue full') + const value = { type: 'frame', observedAtMs, record: record(frame), context: contextSchema.parse(options.context(frame, observedAtMs)) } + const text = JSON.stringify(value), size = Buffer.byteLength(text) + if (size > 128 * 1024 || bytes + size > budgets.maximumBytes) throw new Error('Prospective frame byte quota exceeded') + previousArrival = observedAtMs + if (waiter) { + const resolve = waiter + waiter = undefined + resolve({ done: false, value: JSON.parse(text) }) + } + else { + queue.push({ text, bytes: size }) + bytes += size + } + } }) + } + catch (error) { + close() + throw error + } + } + return { close, frames: { [Symbol.asyncIterator]() { + if (used) throw new Error('Prospective frame source has one consumer') + used = true + return { next: (): Promise> => { + if (closed) return Promise.resolve({ done: true, value: undefined }) + if (waiter) return Promise.reject(new Error('Concurrent prospective frame reads')) + const row = queue.shift() + if (row) { + bytes -= row.bytes + return Promise.resolve({ done: false, value: JSON.parse(row.text) }) + } + return new Promise((resolve) => { + waiter = resolve + }) + }, return: async (): Promise> => { + if (Date.now() >= options.endMs) finishCollection() + else close() + return { done: true, value: undefined } + } } + } } } +} diff --git a/src/biometrics/shadow/health-monitor.ts b/src/biometrics/shadow/health-monitor.ts new file mode 100644 index 000000000..1595779f9 --- /dev/null +++ b/src/biometrics/shadow/health-monitor.ts @@ -0,0 +1,64 @@ +import { admissionPolicySchema, collectAdmissionSample, evaluateAdmission } from '../backtest/admission' +import type { AdmissionSample } from '../backtest/admission' + +/** One collection in flight, with a deadline derived from the installed policy. + * Timeout is terminal: uncancellable filesystem I/O must not accumulate behind + * retries. The attempt observes null on its next synchronous admission check. */ +export async function startShadowHealthMonitor(collect: () => Promise, rawPolicy: unknown, signal?: AbortSignal) { + const policy = admissionPolicySchema.parse(rawPolicy) + const cadenceMs = Math.min(1000, Math.floor(policy.maximumSampleAgeMs / 2)) + if (cadenceMs < 1) throw new Error('Shadow health freshness policy is too short') + let closed = false, sample: AdmissionSample | null = null + let refreshTimer: ReturnType | undefined, deadline: ReturnType | undefined + let rejectPending: ((error: Error) => void) | undefined + const close = () => { + closed = true + sample = null + clearTimeout(refreshTimer) + clearTimeout(deadline) + signal?.removeEventListener('abort', close) + rejectPending?.(new Error('Shadow health monitor closed')) + } + const refresh = async () => { + if (closed) throw new Error('Shadow health monitor closed') + try { + const collected = await Promise.race([Promise.resolve().then(() => { + if (closed) throw new Error('Shadow health monitor closed') + return collect() + }), new Promise((_resolve, reject) => { + rejectPending = reject + deadline = setTimeout(() => reject(new Error('Shadow health collection deadline exceeded')), cadenceMs) + })]) + if (closed) throw new Error('Shadow health monitor closed') + sample = collected ? structuredClone(collected) : null + } + catch (error) { + close() + throw error + } + finally { + clearTimeout(deadline) + rejectPending = undefined + } + } + const schedule = () => { + if (closed) return + refreshTimer = setTimeout(() => { + void refresh().then(schedule).catch(() => {}) + }, cadenceMs) + } + signal?.addEventListener('abort', close, { once: true }) + if (signal?.aborted) close() + await refresh() + schedule() + return { + read: () => !closed && evaluateAdmission(sample, policy).admitted ? structuredClone(sample) : null, + close, + } +} + +/** Production collector uses host filesystem health and the actual live piezo + * receive clock. Paths and qualification policy are supplied by the host only. */ +export async function openShadowHealthMonitor(databaseDirectory: string, policy: unknown, piezoReceivedAt: () => number | undefined, signal?: AbortSignal) { + return startShadowHealthMonitor(() => collectAdmissionSample(databaseDirectory, piezoReceivedAt), policy, signal) +} diff --git a/src/biometrics/shadow/host-entry.ts b/src/biometrics/shadow/host-entry.ts new file mode 100644 index 000000000..7094f4d51 --- /dev/null +++ b/src/biometrics/shadow/host-entry.ts @@ -0,0 +1,8 @@ +// Shipped, fingerprinted host API. Importing this module opens no database, +// starts no service, subscribes to no source and performs no hardware command. +export { runProfiledShadowAttempt, openProfiledShadowHost } from './host' +export { prepareShadowPlan } from './preparation' +export { ShadowStore, shadowPlanSchema, shadowStoragePolicySchema } from './store' +export { shadowEngineBuildIdentity } from './build' +export { readInstalledShadowQualifications } from './admission' +export const shadowHostApiVersion = 'shadow-host-api-v1' as const diff --git a/src/biometrics/shadow/host.ts b/src/biometrics/shadow/host.ts new file mode 100644 index 000000000..e91b6f5e1 --- /dev/null +++ b/src/biometrics/shadow/host.ts @@ -0,0 +1,139 @@ +import { resolveShadowProfiles } from './profiles' +import { dirname } from 'node:path' +import { dataHash } from '../algorithms/canonical' +import { shadowPlanSchema, type ShadowStore } from './store' +import { installedShadowAdmitter, readInstalledShadowQualifications } from './admission' +import { openShadowActiveDatabase } from './active-database' +import { openShadowHealthMonitor } from './health-monitor' +import { shadowRuntimeBindingReader } from './runtime-binding' +import { plannedShadowSourceFactory } from './planned-sources' +import { runShadowAttempt, type ShadowAttemptHost } from './attempt' + +export interface ShadowHostInstallation { + workerPath: string + pythonPath: string + modulesPath: string + configurationDatabase: string + biometricsDatabase: string + presenceFile: string + configs: { left: unknown, right: unknown } + expectedBindings: Record<'left' | 'right', { generation: number, resolvedHash: string }> + /** Trusted installed-build/hardware collector; must detect changes, never RPC data. */ + identity: () => unknown + piezoReceivedAt: () => number | undefined +} + +/** Own admission connections and health sampling for exactly one saved plan. + * Qualification lookup precedes I/O sampling; no source subscription or worker + * launch occurs until the attempt controller has claimed its fenced lease. */ +export async function openPlannedShadowHost(rawPlan: unknown, installation: ShadowHostInstallation, signal?: AbortSignal) { + const plan = shadowPlanSchema.parse(rawPlan), planHash = dataHash(plan) + if (!plan.execution || !plan.joining) throw new Error('Shadow host requires a saved execution policy') + const { identity, piezoReceivedAt, ...rawLocations } = installation + const locations = structuredClone(rawLocations) + const controller = new AbortController() + let closed = false + let config: ReturnType | undefined, bio: ReturnType | undefined + let health: Awaited> | undefined + const close = () => { + if (closed) return + closed = true + signal?.removeEventListener('abort', close) + controller.abort() + try { + health?.close() + } + finally { + try { + config?.close() + } + finally { bio?.close() } + } + } + signal?.addEventListener('abort', close, { once: true }) + if (signal?.aborted) close() + try { + if (closed) throw new Error('Shadow host cancelled before opening') + const entries = readInstalledShadowQualifications(locations.workerPath).entries.filter(entry => dataHash(entry) === plan.qualificationHash) + if (entries.length !== 1) throw new Error('Shadow qualification missing or ambiguous') + const databaseOptions = { busyTimeoutMs: plan.execution.sources.databaseBusyTimeoutMs } + config = openShadowActiveDatabase(locations.configurationDatabase, databaseOptions) + bio = openShadowActiveDatabase(locations.biometricsDatabase, databaseOptions) + const configuration = config, biometrics = bio + const binding = shadowRuntimeBindingReader(configuration.db, biometrics.db, { side: plan.side, expected: locations.expectedBindings, + assertCurrent: () => { + configuration.assertCurrent() + biometrics.assertCurrent() + } }) + health = await openShadowHealthMonitor(dirname(locations.biometricsDatabase), entries[0].healthPolicy, piezoReceivedAt, controller.signal) + if (closed) { + health.close() + throw new Error('Shadow host cancelled while opening') + } + const admission = installedShadowAdmitter({ workerPath: locations.workerPath, + expectedBinding: { side: plan.side, ...locations.expectedBindings[plan.side] }, identity, binding, health: health.read }) + const host: ShadowAttemptHost = { + location: { pythonPath: locations.pythonPath, modulesPath: locations.modulesPath }, configs: locations.configs, + admit: (candidate) => { + if (closed || dataHash(candidate) !== planHash) throw new Error('Shadow host closed or saved plan changed') + admission(candidate) + }, + openSources: plannedShadowSourceFactory(plan, locations), + } + host.admit(plan) + return { host, signal: controller.signal, close } + } + catch (error) { + close() + throw error + } +} + +/** Own cleanup even when claim fails, the worker fails, or a caller cancels. */ +export async function runPlannedShadowAttempt(store: ShadowStore, runId: string, installation: ShadowHostInstallation, signal?: AbortSignal) { + const owned = await openPlannedShadowHost(store.get(runId).plan, installation, signal) + try { + return await runShadowAttempt(store, runId, owned.host, owned.signal) + } + finally { owned.close() } +} + +/** Public service integration supplies a saved revision ID, never resolved + * numerical configs or asserted active generations. Admission rechecks the + * captured generations after these snapshot handles have been closed. */ +export async function runProfiledShadowAttempt(store: ShadowStore, runId: string, + installation: Omit, revisionId: string | null, signal?: AbortSignal) { + const owned = await openProfiledShadowHost(store.get(runId).plan, installation, revisionId, signal) + try { + return await runShadowAttempt(store, runId, owned.host, owned.signal) + } + finally { owned.close() } +} + +/** Admission-only preparation for a proposed plan. Does not launch a worker, + * claim a lease or open sensor sources. Caller owns the returned resources. */ +export async function openProfiledShadowHost(rawPlan: unknown, + installation: Omit, revisionId: string | null, signal?: AbortSignal) { + const plan = shadowPlanSchema.parse(rawPlan) + if (!plan.execution) throw new Error('Shadow profile resolution requires execution policy') + if (signal?.aborted) throw new Error('Shadow attempt cancelled before profile resolution') + const options = { busyTimeoutMs: plan.execution.sources.databaseBusyTimeoutMs } + const configuration = openShadowActiveDatabase(installation.configurationDatabase, options) + let biometrics: ReturnType | undefined + let resolved: ReturnType + try { + biometrics = openShadowActiveDatabase(installation.biometricsDatabase, options) + const bio = biometrics + resolved = resolveShadowProfiles(configuration.db, bio.db, plan, revisionId, () => { + configuration.assertCurrent() + bio.assertCurrent() + }) + } + finally { + try { + configuration.close() + } + finally { biometrics?.close() } + } + return openPlannedShadowHost(plan, { ...installation, configs: resolved.configs, expectedBindings: resolved.expectedBindings }, signal) +} diff --git a/src/biometrics/shadow/installed-attempt.ts b/src/biometrics/shadow/installed-attempt.ts new file mode 100644 index 000000000..61c4311fd --- /dev/null +++ b/src/biometrics/shadow/installed-attempt.ts @@ -0,0 +1,87 @@ +import { withBiometricBackgroundSlot } from '../background-admission' +import { z } from 'zod' +import { dataHash } from '../algorithms/canonical' +import { getSensorFrameTimes } from '@/src/streaming/piezoStream' +import type { ShadowStore } from './store' +import { inspectInstalledShadowHost, loadInstalledShadowHost } from './installed-host' +import { installedShadowIdentity } from './device-identity' +import { openShadowStateDatabase } from './active-database' +import type { ShadowHostInstallation } from './host' + +type Installation = Pick + +/** Application-side execution boundary. Paths/storage policy are trusted service + * configuration. User operations identify only an existing run and revision. + * Executes the inspected package, never the source-copy attempt controller. */ +async function runAdmittedShadowAttempt(runId: string, revisionId: string | null, installation: Installation, + storagePolicy: unknown, busyTimeoutMs: number, signal?: AbortSignal) { + z.string().uuid().parse(runId) + z.string().uuid().nullable().parse(revisionId) + z.number().int().min(1).max(1000).parse(busyTimeoutMs) + const paths = structuredClone(installation), policy = structuredClone(storagePolicy) + if (signal?.aborted) throw new Error('Installed shadow attempt cancelled') + const inspected = await inspectInstalledShadowHost(paths.workerPath, paths, signal) + if (signal?.aborted) throw new Error('Installed shadow attempt cancelled') + inspected.assertCurrent() + const database = openShadowStateDatabase(paths.biometricsDatabase, { busyTimeoutMs }) + try { + const store = new inspected.api.ShadowStore(database.db, policy) + const plan = store.get(runId).plan + if (!plan.expectedBindings) throw new Error('Shadow active generations missing from saved plan') + if (plan.candidateRevisionId === undefined || plan.candidateRevisionId !== revisionId) throw new Error('Shadow candidate revision differs from saved plan') + if (!plan.execution || busyTimeoutMs > plan.execution.sources.databaseBusyTimeoutMs) throw new Error('Shadow store timeout exceeds saved execution policy') + const entries = inspected.api.readInstalledShadowQualifications(paths.workerPath).entries.filter(entry => dataHash(entry) === plan.qualificationHash) + if (entries.length !== 1) throw new Error('Shadow qualification missing or ambiguous') + const observedIdentity = installedShadowIdentity(inspected, entries[0]) + const identity = () => { + database.assertCurrent() + return observedIdentity() + } + identity() + return await inspected.api.runProfiledShadowAttempt(store, runId, { ...paths, identity, + piezoReceivedAt: () => getSensorFrameTimes()['piezo-dual'] }, revisionId, signal) + } + finally { database.close() } +} + +/** Reserve before build probes, database ownership, source subscription or claim. + * Busy pending runs remain pending for the service's next scheduling pass. */ +export function runInstalledShadowAttempt(...args: Parameters) { + return withBiometricBackgroundSlot('work', () => runAdmittedShadowAttempt(...args)) +} + +/** Queue inspection runs the verified package with an owned state connection. */ +export function pendingInstalledShadowRun(installation: Pick, + storagePolicy: unknown, busyTimeoutMs: number, options: Parameters[0]) { + z.number().int().min(1).max(1000).parse(busyTimeoutMs) + const loaded = loadInstalledShadowHost(installation.workerPath) + loaded.assertCurrent() + const database = openShadowStateDatabase(installation.biometricsDatabase, { busyTimeoutMs }) + try { + database.assertCurrent() + const result = new loaded.api.ShadowStore(database.db, storagePolicy).pending(options) + database.assertCurrent() + loaded.assertCurrent() + return result + } + finally { database.close() } +} + +/** Recovery needs the installed store implementation but never starts a Python + * probe/worker, subscribes to sensors or requires an admission qualification. */ +export function reconcileInstalledShadowRuns(installation: Pick, + storagePolicy: unknown, busyTimeoutMs: number, options: Parameters[0]) { + z.number().int().min(1).max(1000).parse(busyTimeoutMs) + const loaded = loadInstalledShadowHost(installation.workerPath) + loaded.assertCurrent() + const database = openShadowStateDatabase(installation.biometricsDatabase, { busyTimeoutMs }) + try { + database.assertCurrent() + const store = new loaded.api.ShadowStore(database.db, storagePolicy) + const result = store.reconcileExpired(options) + database.assertCurrent() + loaded.assertCurrent() + return result + } + finally { database.close() } +} diff --git a/src/biometrics/shadow/installed-host.ts b/src/biometrics/shadow/installed-host.ts new file mode 100644 index 000000000..e2d86baba --- /dev/null +++ b/src/biometrics/shadow/installed-host.ts @@ -0,0 +1,81 @@ +import { guardShadowPythonInstallation } from './python-installation' +import { createHash } from 'node:crypto' +import { constants, openSync, closeSync, fstatSync, lstatSync, readSync, realpathSync } from 'node:fs' +import { createRequire } from 'node:module' +import { resolve } from 'node:path' +import type * as HostAPI from './host-entry' + +type API = typeof HostAPI +const maximumBytes = 16 * 1024 * 1024 +function signature(path: string) { + const stat = lstatSync(path, { bigint: true }) + if (!stat.isFile() || stat.isSymbolicLink() || stat.size > BigInt(maximumBytes)) throw new Error('Invalid installed shadow host artifact') + return `${stat.dev}:${stat.ino}:${stat.size}:${stat.mtimeNs}:${stat.ctimeNs}` +} +interface InstalledHost { path: string, signature: string, hostHash: string, api: API } +const state = globalThis as typeof globalThis & { __sleepypodInstalledShadowHostV1?: InstalledHost } + +/** Trusted deployment path only. Load one exact release artifact per process; + * changed installations require restart rather than reusing Node's stale cache. + * This guards the Node bundle, not Python binaries or dependency wheel contents. */ +export function loadInstalledShadowHost(hostPath: string) { + const requestedPath = resolve(hostPath) + signature(requestedPath) // reject a leaf symlink before canonicalizing parent paths + const path = realpathSync(requestedPath) + let cached = state.__sleepypodInstalledShadowHostV1 + if (cached) { + if (cached.path !== path || signature(path) !== cached.signature) throw new Error('Installed shadow host changed; restart required') + } + else { + const before = signature(path) + const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK) + let hostHash: string + try { + const opened = fstatSync(fd, { bigint: true }) + if (`${opened.dev}:${opened.ino}:${opened.size}:${opened.mtimeNs}:${opened.ctimeNs}` !== before) throw new Error('Shadow host changed before hashing') + const digest = createHash('sha256'), buffer = Buffer.alloc(65536) + let length = 0 + while (true) { + const count = readSync(fd, buffer, 0, buffer.length, null) + if (!count) break + length += count + if (length > maximumBytes) throw new Error('Shadow host artifact exceeds its bound') + digest.update(buffer.subarray(0, count)) + } + hostHash = digest.digest('hex') + } + finally { closeSync(fd) } + if (signature(path) !== before) throw new Error('Shadow host changed during hashing') + const require = createRequire(path) + if (require.cache[path]) throw new Error('Shadow host was loaded without installed identity verification') + const api = require(path) as Partial + if (signature(path) !== before) throw new Error('Shadow host changed while loading') + if (api.shadowHostApiVersion !== 'shadow-host-api-v1' || ['runProfiledShadowAttempt', 'openProfiledShadowHost', 'prepareShadowPlan', 'ShadowStore', 'shadowEngineBuildIdentity', 'readInstalledShadowQualifications'] + .some(name => typeof api[name as keyof API] !== 'function') || !api.shadowPlanSchema || !api.shadowStoragePolicySchema) throw new Error('Invalid installed shadow host API') + cached = { path, signature: before, hostHash, api: api as API } + state.__sleepypodInstalledShadowHostV1 = cached + } + const installed = cached + return { api: installed.api, hostHash: installed.hostHash, + assertCurrent: () => { + if (signature(path) !== installed.signature || signature(requestedPath) !== installed.signature) throw new Error('Installed shadow host changed; restart required') + } } +} + +/** Probe through the loaded package and prove the fingerprint describes the + * artifact actually loaded, including rechecking it after the async Python probe. */ +export async function inspectInstalledShadowHost(hostPath: string, location: { pythonPath: string, modulesPath: string }, signal?: AbortSignal) { + if (signal?.aborted) throw new Error('Shadow build inspection cancelled') + const target = { ...location } + const loaded = loadInstalledShadowHost(hostPath) + const build = await loaded.api.shadowEngineBuildIdentity(resolve(hostPath), target, signal) + loaded.assertCurrent() + if (build.identity.hostHash !== loaded.hostHash) throw new Error('Loaded shadow host differs from inspected artifact') + const pythonCurrent = guardShadowPythonInstallation(target, build.python) + const assertCurrent = () => { + loaded.assertCurrent() + pythonCurrent() + } + assertCurrent() + return { ...loaded, build, assertCurrent } +} diff --git a/src/biometrics/shadow/installed-preparation.ts b/src/biometrics/shadow/installed-preparation.ts new file mode 100644 index 000000000..8bd49e4e9 --- /dev/null +++ b/src/biometrics/shadow/installed-preparation.ts @@ -0,0 +1,44 @@ +import type { z } from 'zod' +import type { runInstalledShadowAttempt } from './installed-attempt' +import { openShadowActiveDatabase } from './active-database' +import { loadInstalledShadowHost } from './installed-host' +import { prepareShadowRequestSchema, shadowPlanTemplateSchema, shadowTemplatePageSchema } from './preparation-contracts' +import { configuredShadowService } from './startup' + +export function configuredShadowTemplates() { + return shadowTemplatePageSchema.parse({ items: (configuredShadowService()?.planTemplates ?? []).map(({ id, label, maximumWindowMs }) => ({ id, label, maximumWindowMs })) }) +} + +/** Fixed server paths and package implementation; no Python probe or sources. + * A prepared response cannot be submitted without ordinary creation admission. */ +export function prepareInstalledShadow(raw: z.input, + installation: Parameters[2], busyTimeoutMs: number) { + const request = prepareShadowRequestSchema.parse(raw) + const templates = configuredShadowService()?.planTemplates ?? [] + const matches = templates.filter(template => template.id === request.templateId) + if (matches.length !== 1) throw new Error('Shadow template unavailable') + const template = shadowPlanTemplateSchema.parse(matches[0]) + if (busyTimeoutMs > template.execution.sources.databaseBusyTimeoutMs) throw new Error('Shadow store timeout exceeds template policy') + const loaded = loadInstalledShadowHost(installation.workerPath) + loaded.assertCurrent() + const configuration = openShadowActiveDatabase(installation.configurationDatabase, { busyTimeoutMs }) + let biometrics: ReturnType | undefined + try { + biometrics = openShadowActiveDatabase(installation.biometricsDatabase, { busyTimeoutMs }) + const bio = biometrics + const guard = () => { + loaded.assertCurrent() + configuration.assertCurrent() + bio.assertCurrent() + } + const prepared = loaded.api.prepareShadowPlan(configuration.db, bio.db, request, template, installation.workerPath, guard) + guard() + return prepared + } + finally { + try { + configuration.close() + } + finally { biometrics?.close() } + } +} diff --git a/src/biometrics/shadow/live-sources.ts b/src/biometrics/shadow/live-sources.ts new file mode 100644 index 000000000..66c78e60b --- /dev/null +++ b/src/biometrics/shadow/live-sources.ts @@ -0,0 +1,86 @@ +import type Database from 'better-sqlite3' +import { z } from 'zod' +import { ShadowActiveReader } from './active-reader' +import { openShadowFrameSource } from './frame-source' +import type { ShadowSources } from './attempt' + +/** Compose the real read-only active reader with the live waveform tap. The + * owner retains the database connection and must detect path/inode replacement. + * Polling and final-drain flush use the same bounded, synchronous snapshot. */ +export function openShadowLiveSources(options: { + db: Database.Database + side: 'left' | 'right' + startMs: number + endMs: number + activeConfigHash: string + maximumActiveRows: number + maximumLatenessMs: number + pollMs: number + frameBudgets: Parameters[0]['budgets'] + context: Parameters[0]['context'] + assertDatabaseCurrent: () => void + signal: AbortSignal + active: (reading: unknown) => void + fail: () => void +}): ShadowSources { + z.number().int().min(1).max(120000).parse(options.maximumLatenessMs) + z.number().int().min(10).max(Math.min(1000, options.maximumLatenessMs - 1)).parse(options.pollMs) + function synchronous(action: () => void) { + const result: unknown = action() + if (result !== undefined) { + void Promise.resolve(result).catch(() => {}) + throw new Error('Shadow live source callbacks must complete synchronously') + } + } + synchronous(options.assertDatabaseCurrent) + // Establish the active watermark first; never miss a commit while enabling + // candidate input, and never substitute historical active rows. + const reader = new ShadowActiveReader(options.db, { side: options.side, startMs: options.startMs, endMs: options.endMs, + activeConfigHash: options.activeConfigHash, maximumRows: options.maximumActiveRows }) + let closed = false, waveform: ReturnType | undefined + let timer: ReturnType | undefined + const close = () => { + if (closed) return + closed = true + clearInterval(timer) + waveform?.close() + options.signal.removeEventListener('abort', close) + } + const fail = () => { + if (closed) return + close() + try { + options.fail() + } + catch { /* Isolate controller errors from the live server's timer. */ } + } + const flush = () => { + if (closed) throw new Error('Shadow live sources are closed') + synchronous(options.assertDatabaseCurrent) + const readings = reader.poll() + synchronous(options.assertDatabaseCurrent) + for (const reading of readings) { + if (closed || options.signal.aborted) throw new Error('Shadow active delivery cancelled') + synchronous(() => options.active(reading)) + } + } + try { + waveform = openShadowFrameSource({ startMs: options.startMs, endMs: options.endMs, budgets: options.frameBudgets, + signal: options.signal, context: options.context, fail }) + if (options.signal.aborted) close() + else { + options.signal.addEventListener('abort', close, { once: true }) + timer = setInterval(() => { + try { + flush() + } + catch { fail() } + }, options.pollMs) + } + return { frames: waveform.frames, flush, close } + } + catch (error) { + close() + throw error + } +} diff --git a/src/biometrics/shadow/plan-contracts.ts b/src/biometrics/shadow/plan-contracts.ts new file mode 100644 index 000000000..c39c7ef84 --- /dev/null +++ b/src/biometrics/shadow/plan-contracts.ts @@ -0,0 +1,34 @@ +import { z } from 'zod' +import { shadowExecutionPolicySchema } from './execution-policy' + +const integer = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) +const positive = integer.positive() +const hash = z.string().regex(/^[a-f0-9]{64}$/) +const metric = z.enum(['heart_rate', 'breathing_rate', 'window_ibi_variability_v1']) +const binding = z.object({ generation: positive, resolvedHash: hash }).strict() +export const shadowExpectedBindingsSchema = z.object({ left: binding, right: binding }).strict() +export const shadowMonitoringSchema = z.object({ + expectedIntervalMs: positive, maximumGapMs: positive, heartbeatStaleMs: positive, + minimumCandidateCoverage: z.number().positive().max(1), maximumCoverageLoss: z.number().min(0).max(1), minimumQuality: z.number().min(0).max(1), + alignment: z.object({ version: z.literal('reference-alignment-v1'), pointToleranceMs: integer.max(300000), + clockOffsetMs: z.number().int().min(-3600000).max(3600000), offsetSource: z.enum(['fixed', 'training']) }).strict(), +}).strict() +export const shadowPlanSchema = z.object({ schema: z.literal('biometric-shadow-plan-v1'), side: z.enum(['left', 'right']), + execution: shadowExecutionPolicySchema.optional(), + // Absent on legacy evidence; null explicitly selects the shipped default. + candidateRevisionId: z.string().uuid().nullable().optional(), + expectedBindings: shadowExpectedBindingsSchema.optional(), + subjectId: z.string().min(1).max(200), monitoring: shadowMonitoringSchema, + windowIbiWindowMs: positive.max(86400000).optional(), + joining: z.object({ maximumPendingPoints: positive.max(10000), maximumLatenessMs: positive.max(120000), + drainGraceMs: positive.max(300000) }).strict().refine(value => value.drainGraceMs > value.maximumLatenessMs, 'Drain grace must exceed lateness').optional(), + candidateConfigHash: hash, activeConfigHash: hash, qualificationHash: hash, engineBuildHash: hash, + startMs: integer, endMs: integer, maximumPoints: positive, maximumBytes: positive, + metrics: z.array(metric).min(1).max(3), +}).strict().refine(plan => plan.endMs > plan.startMs && plan.endMs - plan.startMs <= 7 * 86400000 + && Number.isSafeInteger(plan.endMs + (plan.joining?.drainGraceMs ?? 0)) + && (!plan.execution || (plan.joining !== undefined && plan.execution.sources.pollMs < plan.joining.maximumLatenessMs + && plan.execution.heartbeatMs <= plan.monitoring.heartbeatStaleMs + && plan.execution.configHashes[plan.side] === plan.candidateConfigHash)) + && new Set(plan.metrics).size === plan.metrics.length + && (!plan.metrics.includes('window_ibi_variability_v1') || plan.windowIbiWindowMs !== undefined), 'Invalid shadow window or metrics') diff --git a/src/biometrics/shadow/planned-sources.ts b/src/biometrics/shadow/planned-sources.ts new file mode 100644 index 000000000..1f7083186 --- /dev/null +++ b/src/biometrics/shadow/planned-sources.ts @@ -0,0 +1,83 @@ +import { dataHash } from '../algorithms/canonical' +import { shadowPlanSchema } from './store' +import { openShadowActiveDatabase } from './active-database' +import { ShadowContextReader } from './context' +import { openShadowLiveSources } from './live-sources' +import type { ShadowAttemptHost, ShadowSources } from './attempt' + +/** Trusted installation paths only. This factory owns its read-only connections + * and closes them after unsubscribing on every failure/cancellation/finish path. + * Admission and worker launch remain the attempt controller's responsibility. */ +export function plannedShadowSourceFactory(rawPlan: unknown, locations: { + biometricsDatabase: string + configurationDatabase: string + presenceFile: string +}): ShadowAttemptHost['openSources'] { + const plan = shadowPlanSchema.parse(structuredClone(rawPlan)) + if (!plan.execution || !plan.joining) throw new Error('Planned shadow sources require execution and joining policies') + const execution = structuredClone(plan.execution), joining = { ...plan.joining }, paths = { ...locations } + const executionHash = dataHash(execution) + return (input) => { + if (dataHash(input.execution) !== executionHash || input.collectionEndsAtMs !== plan.endMs) throw new Error('Shadow source policy differs from saved plan') + if (input.signal.aborted) throw new Error('Shadow sources cancelled before opening') + let biometrics: ReturnType | undefined + let configuration: ReturnType | undefined + let live: ShadowSources | undefined, closed = false + const close = () => { + if (closed) return + closed = true + input.signal.removeEventListener('abort', close) + try { + live?.close() + } + finally { + try { + configuration?.close() + } + finally { biometrics?.close() } + } + } + try { + const databaseOptions = { busyTimeoutMs: execution.sources.databaseBusyTimeoutMs } + biometrics = openShadowActiveDatabase(paths.biometricsDatabase, databaseOptions) + configuration = openShadowActiveDatabase(paths.configurationDatabase, databaseOptions) + const bio = biometrics, config = configuration + const assertCurrent = () => { + bio.assertCurrent() + config.assertCurrent() + } + const context = new ShadowContextReader(config.db, paths.presenceFile, execution.sources.context, assertCurrent) + live = openShadowLiveSources({ db: bio.db, side: plan.side, startMs: plan.startMs, endMs: plan.endMs, + activeConfigHash: plan.activeConfigHash, maximumActiveRows: execution.sources.maximumActiveRows, + maximumLatenessMs: joining.maximumLatenessMs, pollMs: execution.sources.pollMs, + frameBudgets: execution.sources.frames, context: (frame, at) => context.snapshot(frame, at), + assertDatabaseCurrent: assertCurrent, signal: input.signal, active: input.active, + fail: () => { + try { + close() + } + finally { input.fail() } + } }) + input.signal.addEventListener('abort', close, { once: true }) + if (input.signal.aborted) { + close() + throw new Error('Shadow sources cancelled while opening') + } + const source = live + return { frames: source.frames, flush: () => { + if (closed) throw new Error('Planned shadow sources are closed') + try { + source.flush?.() + } + catch (error) { + close() + throw error + } + }, close } + } + catch (error) { + close() + throw error + } + } +} diff --git a/src/biometrics/shadow/preparation-contracts.ts b/src/biometrics/shadow/preparation-contracts.ts new file mode 100644 index 000000000..38f54fe5a --- /dev/null +++ b/src/biometrics/shadow/preparation-contracts.ts @@ -0,0 +1,22 @@ +import { z } from 'zod' +import { shadowPlanSchema } from './plan-contracts' +import { shadowExecutionPolicySchema } from './execution-policy' +import { submitShadowSchema } from './submission-contracts' + +const time = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) +export const shadowPlanTemplateSchema = z.object({ + id: z.string().min(1).max(80), label: z.string().min(1).max(200), + qualificationHash: z.string().regex(/^[a-f0-9]{64}$/), + maximumWindowMs: time.positive().max(7 * 86400000), + execution: shadowExecutionPolicySchema, monitoring: shadowPlanSchema.shape.monitoring, + joining: shadowPlanSchema.shape.joining.unwrap(), metrics: shadowPlanSchema.shape.metrics, + windowIbiWindowMs: shadowPlanSchema.shape.windowIbiWindowMs, + maximumPoints: shadowPlanSchema.shape.maximumPoints, maximumBytes: shadowPlanSchema.shape.maximumBytes, +}).strict().refine(template => template.maximumWindowMs + template.joining.drainGraceMs <= template.execution.budgets.wallMs, + 'Shadow template window and drain exceed worker wall budget') +export const prepareShadowRequestSchema = z.object({ templateId: z.string().min(1).max(80), side: z.enum(['left', 'right']), + subjectId: z.string().min(1).max(200), candidateRevisionId: z.string().uuid().nullable(), startMs: time, endMs: time }).strict() +export const preparedShadowPlanSchema = z.object({ status: z.literal('prepared_not_admitted'), plan: submitShadowSchema.shape.plan, + planHash: z.string().regex(/^[a-f0-9]{64}$/) }).strict() +export const shadowTemplatePageSchema = z.object({ items: z.array(z.object({ id: shadowPlanTemplateSchema.shape.id, + label: shadowPlanTemplateSchema.shape.label, maximumWindowMs: shadowPlanTemplateSchema.shape.maximumWindowMs }).strict()).max(8) }).strict() diff --git a/src/biometrics/shadow/preparation.ts b/src/biometrics/shadow/preparation.ts new file mode 100644 index 000000000..52c745d2a --- /dev/null +++ b/src/biometrics/shadow/preparation.ts @@ -0,0 +1,35 @@ +import type Database from 'better-sqlite3' +import type { z } from 'zod' +import { configHash, dataHash } from '../algorithms/canonical' +import { manifestHash } from '../algorithms/registry' +import { captureShadowProfiles } from './profiles' +import { readInstalledShadowQualifications } from './admission' +import { prepareShadowRequestSchema, preparedShadowPlanSchema, shadowPlanTemplateSchema } from './preparation-contracts' +import { submitShadowSchema } from './submission-contracts' + +/** A server-owned template provides reviewed ceilings, never caller defaults. + * Preparation proves saved input consistency, not current hardware admission. */ +export function prepareShadowPlan(configuration: Database.Database, biometrics: Database.Database, + rawRequest: z.input, rawTemplate: unknown, workerPath: string, + assertCurrent: () => void, now = Date.now()) { + const request = prepareShadowRequestSchema.parse(rawRequest), template = shadowPlanTemplateSchema.parse(rawTemplate) + if (template.id !== request.templateId) throw new Error('Shadow template changed') + const duration = request.endMs - request.startMs + if (request.startMs < now || duration <= 0 || duration > template.maximumWindowMs + || duration + template.joining.drainGraceMs > template.execution.budgets.wallMs) throw new Error('Shadow window exceeds template limits or is not prospective') + const entries = readInstalledShadowQualifications(workerPath).entries.filter(entry => dataHash(entry) === template.qualificationHash) + if (entries.length !== 1) throw new Error('Shadow template qualification missing or ambiguous') + const entry = entries[0] + if (entry.manifestHash !== manifestHash || entry.executionPolicyHash !== dataHash(template.execution)) throw new Error('Shadow template differs from installed qualification') + const captured = captureShadowProfiles(configuration, biometrics, request.side, request.candidateRevisionId, assertCurrent, now) + if (captured.activeConfigHash !== entry.activeConfigHash + || dataHash({ left: configHash(captured.configs.left), right: configHash(captured.configs.right) }) !== dataHash(template.execution.configHashes)) throw new Error('Shadow template does not match candidate and active profiles') + const policy = { qualificationHash: template.qualificationHash, execution: template.execution, monitoring: template.monitoring, + joining: template.joining, metrics: template.metrics, maximumPoints: template.maximumPoints, maximumBytes: template.maximumBytes, + ...(template.windowIbiWindowMs === undefined ? {} : { windowIbiWindowMs: template.windowIbiWindowMs }) } + const plan = submitShadowSchema.shape.plan.parse({ ...policy, schema: 'biometric-shadow-plan-v1', + side: request.side, subjectId: request.subjectId, candidateRevisionId: request.candidateRevisionId, + startMs: request.startMs, endMs: request.endMs, expectedBindings: captured.expectedBindings, + candidateConfigHash: captured.candidateConfigHash, activeConfigHash: captured.activeConfigHash, engineBuildHash: entry.engineBuildHash }) + return preparedShadowPlanSchema.parse({ status: 'prepared_not_admitted', plan, planHash: dataHash(plan) }) +} diff --git a/src/biometrics/shadow/process-cleanup.ts b/src/biometrics/shadow/process-cleanup.ts new file mode 100644 index 000000000..167481e19 --- /dev/null +++ b/src/biometrics/shadow/process-cleanup.ts @@ -0,0 +1,28 @@ +import { setTimeout as delay } from 'node:timers/promises' + +/** Confirm both the owned child's close event and disappearance of its detached + * process group. Signal 0 observes existence; only ESRCH proves absence. + * Zombies and permission failures remain unconfirmed. Never signal PID/group 0. + * This cannot account for descendants that escape into a different group. */ +export async function confirmShadowProcessCleanup(pid: number | undefined, isClosed: () => boolean, timeoutMs = 5000) { + if (pid !== undefined && (!Number.isSafeInteger(pid) || pid < 1)) return false + if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 5000) return false + const deadline = performance.now() + timeoutMs + while (true) { + let absent = pid === undefined + if (pid !== undefined) { + try { + process.kill(-pid, 0) + } + catch (error) { + // Some kernels briefly return EPERM while killed children are reaped. + // Retry unknown observations within the deadline; never call them absent. + absent = (error as NodeJS.ErrnoException).code === 'ESRCH' + } + } + if (absent && isClosed()) return true + const remaining = deadline - performance.now() + if (remaining <= 0) return false + await delay(Math.min(25, remaining)) + } +} diff --git a/src/biometrics/shadow/profiles.ts b/src/biometrics/shadow/profiles.ts new file mode 100644 index 000000000..e045bbd3c --- /dev/null +++ b/src/biometrics/shadow/profiles.ts @@ -0,0 +1,63 @@ +import type Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { z } from 'zod' +import * as schema from '@/src/db/schema' +import { configHash, dataHash } from '../algorithms/canonical' +import { defaultConfig, validateConfig } from '../algorithms/registry' +import { ProfileStore } from '../profiles/store' +import { runtimePayloadSchema } from '../profiles/runtime' +import { shadowRuntimeBindingReader } from './runtime-binding' +import { shadowPlanSchema } from './store' + +/** Capture saved candidate and acknowledged baselines without activation or + * repair. Both physical channels must have adopted their desired generation. */ +export function captureShadowProfiles(configuration: Database.Database, biometrics: Database.Database, + rawSide: 'left' | 'right', rawRevisionId: string | null, assertCurrent: () => void, now = Date.now()) { + const side = z.enum(['left', 'right']).parse(rawSide), revisionId = z.string().uuid().nullable().parse(rawRevisionId) + for (const db of [configuration, biometrics]) { + if (!db.open || (!db.readonly && db.pragma('query_only', { simple: true }) !== 1)) throw new Error('Shadow profile resolution requires read-only databases') + } + const check = () => { + const value: unknown = assertCurrent() + if (value !== undefined) { + void Promise.resolve(value).catch(() => {}) + throw new Error('Shadow profile identity guard must be synchronous') + } + } + check() + const result = configuration.transaction(() => biometrics.transaction(() => { + const profiles = new ProfileStore(drizzle(configuration, { schema })) + const candidate = revisionId === null ? structuredClone(defaultConfig) : profiles.validateRevision(revisionId, side).resolved + const capture = (side: 'left' | 'right') => { + const desired = z.object({ generation: z.number().int().positive().max(Number.MAX_SAFE_INTEGER), resolvedHash: z.string().regex(/^[a-f0-9]{64}$/) }) + .parse(configuration.prepare(`SELECT b.generation, e.next_hash AS resolvedHash FROM biometric_profile_bindings b + JOIN biometric_activation_events e ON e.side = b.side AND e.generation = b.generation WHERE b.side = ?`).get(side)) + const row = biometrics.prepare(`SELECT CASE WHEN length(CAST(payload AS BLOB)) <= 65536 THEN payload ELSE NULL END AS payload + FROM biometric_runtime_configs WHERE side = ? AND generation = ?`).get(side, desired.generation) as { payload: string | null } | undefined + if (!row?.payload) throw new Error('Shadow baseline projection missing or oversized') + const payload = runtimePayloadSchema.parse(JSON.parse(row.payload)) + return { expected: desired, resolved: validateConfig(payload.resolved) } + } + const left = capture('left'), right = capture('right') + const expectedBindings = { left: left.expected, right: right.expected } + const binding = shadowRuntimeBindingReader(configuration, biometrics, { side, expected: expectedBindings, assertCurrent: check, now: () => now })() + const configs = { left: left.resolved, right: right.resolved, [side]: candidate } + return { revisionId, configs, expectedBindings, heartbeatAtMs: binding.heartbeatAtMs, candidateConfigHash: configHash(candidate), activeConfigHash: binding.resolvedHash } + })())() + check() + return structuredClone(result) +} + +/** Validate a frozen plan against a current capture. */ +export function resolveShadowProfiles(configuration: Database.Database, biometrics: Database.Database, rawPlan: unknown, + rawRevisionId: string | null, assertCurrent: () => void, now = Date.now()) { + const plan = shadowPlanSchema.parse(rawPlan), revisionId = z.string().uuid().nullable().parse(rawRevisionId) + if (!plan.execution) throw new Error('Shadow profile resolution requires execution policy') + if (plan.candidateRevisionId !== undefined && plan.candidateRevisionId !== revisionId) throw new Error('Shadow candidate revision differs from saved plan') + const result = captureShadowProfiles(configuration, biometrics, plan.side, revisionId, assertCurrent, now) + if (result.candidateConfigHash !== plan.candidateConfigHash) throw new Error('Shadow candidate differs from saved plan') + if (plan.expectedBindings && dataHash(plan.expectedBindings) !== dataHash(result.expectedBindings)) throw new Error('Shadow active generations differ from saved plan') + if (result.activeConfigHash !== plan.activeConfigHash) throw new Error('Shadow active baseline differs from saved plan') + if (dataHash({ left: configHash(result.configs.left), right: configHash(result.configs.right) }) !== dataHash(plan.execution.configHashes)) throw new Error('Shadow physical-channel configs differ from saved plan') + return { revisionId: result.revisionId, configs: result.configs, expectedBindings: result.expectedBindings, heartbeatAtMs: result.heartbeatAtMs } +} diff --git a/src/biometrics/shadow/python-installation.ts b/src/biometrics/shadow/python-installation.ts new file mode 100644 index 000000000..159da29cb --- /dev/null +++ b/src/biometrics/shadow/python-installation.ts @@ -0,0 +1,66 @@ +import { createHash } from 'node:crypto' +import { constants, openSync, closeSync, fstatSync, lstatSync, readSync, realpathSync } from 'node:fs' +import { isAbsolute, resolve } from 'node:path' +import { configHash } from '../algorithms/canonical' +import { shadowPythonBuildSchema } from './build' + +const maximumBytes = 16 * 1024 * 1024 +function signature(path: string) { + const stat = lstatSync(path, { bigint: true }) + return `${stat.dev}:${stat.ino}:${stat.mode}:${stat.size}:${stat.mtimeNs}:${stat.ctimeNs}` +} + +/** Capture known source and dependency metadata artifacts after the Python + * probe, validating their content against that probe before retaining cheap + * synchronous file-identity checks. Does not attest every installed binary. */ +export function guardShadowPythonInstallation(location: { pythonPath: string, modulesPath: string }, rawBuild: unknown) { + const build = shadowPythonBuildSchema.parse(rawBuild) + if (configHash(build.identity) !== build.hash) throw new Error('Shadow Python build integrity mismatch') + const files: { path: string, hash: string, metadata: boolean }[] = [] + for (const [relative, hash] of Object.entries(build.identity.sources)) { + if (relative.split('/').some(part => !part || part === '.' || part === '..')) throw new Error('Invalid Python source identity path') + files.push({ path: resolve(location.modulesPath, relative), hash, metadata: false }) + } + for (const name of ['numpy', 'scipy'] as const) { + for (const kind of ['record', 'wheel'] as const) { + const path = build.dependencyFiles[name][kind] + if (!isAbsolute(path)) throw new Error('Dependency metadata path must be absolute') + files.push({ path, hash: build.identity.dependencies[name][`${kind}_sha256`], metadata: true }) + } + } + const watched: { path: string, canonical: string, requestedSignature: string, canonicalSignature: string }[] = [] + const watch = (path: string) => { + const requestedSignature = signature(path), canonical = realpathSync(path), canonicalSignature = signature(canonical) + watched.push({ path, canonical, requestedSignature, canonicalSignature }) + return canonical + } + const assertCurrent = () => { + for (const file of watched) { + if (signature(file.path) !== file.requestedSignature || realpathSync(file.path) !== file.canonical + || signature(file.canonical) !== file.canonicalSignature) throw new Error('Shadow Python installation changed') + } + } + for (const file of files) { + const canonical = watch(file.path), fd = openSync(canonical, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK) + try { + const stat = fstatSync(fd) + if (!stat.isFile() || stat.size > maximumBytes) throw new Error('Invalid Python identity artifact') + const buffer = Buffer.alloc(65536), chunks: Buffer[] = [] + let length = 0 + while (true) { + const count = readSync(fd, buffer, 0, buffer.length, null) + if (!count) break + length += count + if (length > maximumBytes) throw new Error('Python identity artifact exceeds its bound') + chunks.push(Buffer.from(buffer.subarray(0, count))) + } + const bytes = Buffer.concat(chunks, length) + const content = file.metadata ? new TextDecoder('utf-8', { fatal: true }).decode(bytes).replace(/\r\n?/g, '\n') : bytes + if (createHash('sha256').update(content).digest('hex') !== file.hash) throw new Error('Python identity artifact differs from probe') + } + finally { closeSync(fd) } + } + watch(resolve(location.pythonPath)) // interpreter links and target replacement + assertCurrent() + return assertCurrent +} diff --git a/src/biometrics/shadow/receipt.ts b/src/biometrics/shadow/receipt.ts new file mode 100644 index 000000000..b4a0e2f5d --- /dev/null +++ b/src/biometrics/shadow/receipt.ts @@ -0,0 +1,46 @@ +import { z } from 'zod' +import { dataHash } from '../algorithms/canonical' +import { shadowChildBudgetsSchema } from './execution-policy' +import { shadowWorkerHealthSchema } from './worker-protocol' +import type { ShadowExecutionPolicy } from './execution-policy' +const integer = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) +export const shadowWorkerReceiptSchema = z.object({ health: shadowWorkerHealthSchema, + inputBytes: integer, outputBytes: integer, peakObservedRssBytes: integer.positive(), rssSamples: integer.positive(), + maximumObservationGapMs: z.number().finite().nonnegative(), budgets: shadowChildBudgetsSchema, + memoryScope: z.literal('sum_rss_of_shadow_process_group'), exactKernelPeak: z.literal(false), +}).strict() +export function validateShadowWorkerReceipt(raw: unknown, execution: ShadowExecutionPolicy) { + const receipt = shadowWorkerReceiptSchema.parse(raw) + const h = receipt.health + if (h.state !== 'completed' || h.reason !== 'window_complete' || h.records !== h.counts.records || h.outputs > h.counts.scored_outputs + || h.counts.accepted_frames + h.counts.empty_channel_frames + h.counts.pump_gated_frames > h.records + || h.counts.scored_outputs + h.counts.preroll_outputs > 2 * h.counts.accepted_frames + || h.records > execution.maximumRecords || h.outputs > execution.maximumOutputs || h.cpu_ms > execution.maximumCpuMs + || receipt.inputBytes > execution.maximumInputBytes || receipt.outputBytes > execution.maximumOutputBytes + || receipt.peakObservedRssBytes > execution.budgets.maximumRssBytes + || dataHash(receipt.budgets) !== dataHash(execution.budgets)) throw new Error('Shadow worker receipt does not match execution policy') + return receipt +} +const completedEnvelopeSchema = z.object({ schema: z.literal('biometric-shadow-receipt-v1'), + runId: z.string().uuid(), attemptId: z.string().uuid(), planHash: z.string().regex(/^[a-f0-9]{64}$/), + recordedAtMs: integer, receipt: shadowWorkerReceiptSchema }).strict() + +/** Supervisor observations from an incomplete attempt. Quota excess is evidence, + * not a reason to discard the receipt. CPU/health is only the last valid reply. */ +export const shadowPartialReceiptSchema = z.object({ + kind: z.literal('partial_supervisor_observation'), health: shadowWorkerHealthSchema.nullable(), + inputBytes: integer, outputBytes: integer, diagnosticBytes: integer, + peakObservedRssBytes: integer.positive().nullable(), rssSamples: integer, + maximumObservationGapMs: z.number().finite().nonnegative(), budgets: shadowChildBudgetsSchema, + cleanupConfirmed: z.boolean(), memoryScope: z.literal('sum_rss_of_shadow_process_group'), exactKernelPeak: z.literal(false), +}).strict().refine(value => (value.rssSamples === 0) === (value.peakObservedRssBytes === null), 'RSS availability mismatch') +export type ShadowPartialReceipt = z.infer +export function validateShadowPartialReceipt(raw: unknown, execution: ShadowExecutionPolicy) { + const receipt = shadowPartialReceiptSchema.parse(raw) + if (dataHash(receipt.budgets) !== dataHash(execution.budgets)) throw new Error('Shadow partial receipt policy mismatch') + return receipt +} +export const shadowPartialEnvelopeSchema = z.object({ schema: z.literal('biometric-shadow-partial-receipt-v1'), + runId: z.string().uuid(), attemptId: z.string().uuid(), planHash: z.string().regex(/^[a-f0-9]{64}$/), + recordedAtMs: integer, receipt: shadowPartialReceiptSchema }).strict() +export const shadowReceiptEnvelopeSchema = z.discriminatedUnion('schema', [completedEnvelopeSchema, shadowPartialEnvelopeSchema]) diff --git a/src/biometrics/shadow/report-contracts.ts b/src/biometrics/shadow/report-contracts.ts new file mode 100644 index 000000000..2d452942a --- /dev/null +++ b/src/biometrics/shadow/report-contracts.ts @@ -0,0 +1,40 @@ +import { z } from 'zod' +import { previewSummarySchema, referenceImportSchema, referenceMetricSchema } from '../references/contracts' +import { shadowReceiptEnvelopeSchema } from './receipt' +import { shadowMonitoringSchema } from './plan-contracts' + +const count = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) +const finite = z.number().finite() +const nullable = finite.nullable() +const hash = z.string().regex(/^[a-f0-9]{64}$/) +const score = z.object({ status: z.enum(['no_reference', 'compared', 'no_overlap', 'incompatible']), + metrics: z.object({ mae: finite, medianAbsoluteError: finite, p95AbsoluteError: finite, signedBias: finite }).strict().nullable(), + pairedCount: count, referenceCoverage: nullable, validObservationCount: count, unmatchedReferences: count, unmatchedObservations: count }).strict() +const support = z.object({ baseline: score, candidate: score }).strict() +const coverage = z.object({ usableCount: count, usableSlots: count, coverage: nullable }).strict() +const comparison = z.object({ version: z.literal('paired-numeric-support-v1'), metric: referenceMetricSchema, + baseline: z.enum(['default', 'current', 'candidate']), candidate: z.enum(['default', 'current', 'candidate']), baselineHash: hash, candidateHash: hash, + commonSupportScope: z.literal('all_configurations'), supportDefinition: z.literal('identical_unambiguous_observation_timestamps'), + commonObservationCount: count, ownSupport: support, commonSupport: support, + differences: z.object({ commonMae: nullable, commonMedianAbsoluteError: nullable, commonP95AbsoluteError: nullable, + commonSignedBias: nullable, ownMae: nullable, referenceCoverage: nullable }).strict(), + lostObservationTimestamps: count, addedObservationTimestamps: count, + ambiguousObservationTimestamps: z.object({ baseline: count, candidate: count }).strict(), coverageReduced: z.boolean(), + interpretation: z.literal('descriptive_numeric_agreement_not_validated_improvement'), +}).strict() +export const shadowReportRequestSchema = z.object({ id: z.string().uuid(), reference: referenceImportSchema.optional() }).strict() +export const shadowReportSchema = z.object({ + schema: z.literal('biometric-shadow-report-v1'), runId: z.string().uuid(), planHash: hash, snapshotHash: hash, reportHash: hash, + capturedAtMs: count, state: z.enum(['pending', 'running', 'completed', 'stopped', 'failed']), reason: z.string().nullable(), + health: z.enum(['issues_detected', 'waiting', 'within_observed_limits']), issues: z.array(z.string()), + expectedSlots: count, observedSamples: count, maximumGapMs: count, attemptsObserved: count, + alignment: shadowMonitoringSchema.shape.alignment, + metrics: z.array(z.object({ metric: z.enum(['heart_rate', 'breathing_rate', 'window_ibi_variability_v1']), windowMs: count.nullable(), + current: coverage, candidate: coverage, coverageDifference: nullable, issues: z.array(z.string()), + referenceExclusions: z.record(z.string(), count), comparison }).strict()).max(3), + pairing: z.object({ activeOnly: count, candidateOnly: count, unknownInputIdentity: count }).strict(), + workerReceipt: shadowReceiptEnvelopeSchema.nullable(), executionPolicyHash: hash.nullable(), + contextEvidence: z.literal('host_observations_not_live_processor_cache_attestation'), limitations: z.array(z.string()), + referenceSummary: previewSummarySchema.nullable(), excludedBoundaryReferences: count, + interpretation: z.literal('prospective_observation_and_reference_agreement_not_activation_authorization'), +}).strict() diff --git a/src/biometrics/shadow/report-core.ts b/src/biometrics/shadow/report-core.ts new file mode 100644 index 000000000..a9786b3d7 --- /dev/null +++ b/src/biometrics/shadow/report-core.ts @@ -0,0 +1,87 @@ +import { compareNumericConfigurations } from '../backtest/numeric-comparison' +import { referenceCompatibility } from '../references/alignment' +import type { previewReferenceImport } from '../references/normalize' +import type { ShadowStore } from './store' + +type Snapshot = ReturnType +export type ShadowReportSnapshot = Pick & { + run: Pick +} +/** Pure report calculations shared by persisted production evidence and the + * explicitly simulated browser demo. Storage integrity remains the caller's job. */ +export function summarizeShadowSnapshot(snapshot: ShadowReportSnapshot, reference: ReturnType | null, executionPolicyHash: string | null) { + const { run, samples, capturedAtMs } = snapshot + const { plan } = run, policy = plan.monitoring + const until = Math.max(plan.startMs, Math.min(plan.endMs, capturedAtMs)) + const expectedSlots = capturedAtMs < plan.startMs + ? 0 + : capturedAtMs < plan.endMs + ? Math.floor((capturedAtMs - plan.startMs) / policy.expectedIntervalMs) + 1 + : Math.ceil((plan.endMs - plan.startMs) / policy.expectedIntervalMs) + const windowSamples = samples.filter(sample => sample.timestampMs <= capturedAtMs && sample.timestampMs < plan.endMs) + const clockSkew = samples.length !== windowSamples.length || (run.heartbeat_at_ms !== null && run.heartbeat_at_ms > capturedAtMs) + let maximumGapMs = 0, previous = plan.startMs + for (const sample of windowSamples) { + maximumGapMs = Math.max(maximumGapMs, sample.timestampMs - previous) + previous = sample.timestampMs + } + maximumGapMs = Math.max(maximumGapMs, until - previous) + const measurements = reference?.measurements.filter(row => row.startMs >= plan.startMs && row.startMs < plan.endMs && row.endMs <= plan.endMs && row.endMs <= until) ?? [] + const metrics = plan.metrics.map((metric) => { + const windowMs = metric === 'window_ibi_variability_v1' ? plan.windowIbiWindowMs ?? null : null + const outputs = (['current', 'candidate'] as const).map((label) => { + const slots = new Set() + const observations = windowSamples.map((sample) => { + const row = sample.readings.find(row => row.metric === metric) + if (!row) throw new Error('Missing shadow metric') + const reading = label === 'current' ? row.active : row.candidate + const value = reading.quality >= policy.minimumQuality ? reading.value : null + if (value !== null) slots.add(Math.floor((sample.timestampMs - plan.startMs) / policy.expectedIntervalMs)) + return { timestampMs: sample.timestampMs, value } + }) + return { label, configHash: label === 'current' ? plan.activeConfigHash : plan.candidateConfigHash, observations, + usableCount: observations.filter(row => row.value !== null).length, usableSlots: slots.size, + coverage: expectedSlots ? Math.min(1, slots.size / expectedSlots) : null } + }) + const current = outputs[0], candidate = outputs[1] + const coverageDifference = current.coverage === null || candidate.coverage === null ? null : candidate.coverage - current.coverage + const issues = [] + if (candidate.coverage !== null && candidate.coverage < policy.minimumCandidateCoverage) issues.push('candidate_coverage_below_minimum') + if (coverageDifference !== null && coverageDifference < -policy.maximumCoverageLoss) issues.push('coverage_regression') + const exclusions: Record = {} + for (const row of measurements) { + const reason = referenceCompatibility(metric, windowMs, row) + if (reason) exclusions[reason] = (exclusions[reason] ?? 0) + 1 + } + return { metric, windowMs, current: { usableCount: current.usableCount, usableSlots: current.usableSlots, coverage: current.coverage }, + candidate: { usableCount: candidate.usableCount, usableSlots: candidate.usableSlots, coverage: candidate.coverage }, coverageDifference, issues, + referenceExclusions: exclusions, comparison: compareNumericConfigurations(metric, windowMs, outputs, measurements, policy.alignment)[0] } + }) + const issues: string[] = [] + if (run.workerReceipt?.schema === 'biometric-shadow-partial-receipt-v1' && !run.workerReceipt.receipt.cleanupConfirmed) issues.push('worker_cleanup_unconfirmed') + if (clockSkew) issues.push('clock_skew') + if (run.reason?.startsWith('join_')) issues.push('join_failed') + if (run.reason === 'shadow_observation_window_missed') issues.push('observation_window_missed') + else if (run.state === 'failed') issues.push('processor_failed') + if (!issues.includes('worker_cleanup_unconfirmed') && (run.reason === 'shadow_lease_expired_cleanup_unconfirmed' || run.reason === 'shadow_worker_cleanup_unconfirmed')) issues.push('worker_cleanup_unconfirmed') + if (run.reason === 'storage_quota') issues.push('storage_quota') + if (run.state === 'running' && ((run.lease_until_ms ?? 0) <= capturedAtMs || run.heartbeat_at_ms === null + || capturedAtMs - run.heartbeat_at_ms > policy.heartbeatStaleMs)) issues.push('worker_stale') + if (maximumGapMs > policy.maximumGapMs) issues.push('observation_gap') + if (!windowSamples.length && capturedAtMs >= plan.startMs) issues.push('no_observations') + if (metrics.some(metric => metric.issues.length)) issues.push('coverage_degraded') + const report = { schema: 'biometric-shadow-report-v1' as const, runId: run.id, planHash: run.plan_hash, snapshotHash: snapshot.snapshotHash, + capturedAtMs, state: run.state, reason: run.reason, health: issues.length ? 'issues_detected' as const : capturedAtMs < plan.startMs ? 'waiting' as const : 'within_observed_limits' as const, + issues, expectedSlots, observedSamples: windowSamples.length, maximumGapMs, alignment: policy.alignment, + attemptsObserved: new Set(samples.map(sample => sample.attemptId)).size, metrics, + pairing: { activeOnly: samples.filter(sample => sample.pairing?.activePresent && !sample.pairing.candidatePresent).length, + candidateOnly: samples.filter(sample => sample.pairing?.candidatePresent && !sample.pairing.activePresent).length, + unknownInputIdentity: samples.filter(sample => sample.inputHash === null).length }, + workerReceipt: run.workerReceipt, + executionPolicyHash: executionPolicyHash, + contextEvidence: 'host_observations_not_live_processor_cache_attestation' as const, + limitations: [...new Set(samples.flatMap(sample => sample.candidateProvenance?.limitations ?? []))].sort(), + referenceSummary: reference?.summary ?? null, excludedBoundaryReferences: reference ? reference.measurements.length - measurements.length : 0, + interpretation: 'prospective_observation_and_reference_agreement_not_activation_authorization' as const } + return report +} diff --git a/src/biometrics/shadow/report-operation.ts b/src/biometrics/shadow/report-operation.ts new file mode 100644 index 000000000..187ee839e --- /dev/null +++ b/src/biometrics/shadow/report-operation.ts @@ -0,0 +1,25 @@ +import type Database from 'better-sqlite3' +import type { z } from 'zod' +import { ReferenceInputError } from '../references/time' +import { ShadowStore } from './store' +import { reportShadowRun } from './report' +import { shadowReportRequestSchema, shadowReportSchema } from './report-contracts' + +/** Coordinator-only bounded computation; no execution policy or worker needed. + * Failure on oversized evidence is explicit, never a downsampled report. */ +export function readShadowReport(db: Database.Database, raw: z.input) { + const input = shadowReportRequestSchema.parse(raw) + const store = new ShadowStore(db) + try { + const plan = store.get(input.id).plan + if (input.reference && (input.reference.side !== plan.side || input.reference.subjectId !== plan.subjectId)) { + throw new ReferenceInputError('SUBJECT_MISMATCH', 'Shadow reference must match the saved sleeper and side') + } + return shadowReportSchema.parse(reportShadowRun(store, input.id, { maximumPoints: 100000, maximumBytes: 32 * 1024 * 1024 }, input.reference)) + } + catch (error) { + if (error instanceof Error && error.message === 'Unknown shadow run') throw new ReferenceInputError('NOT_FOUND', 'Shadow run not found') + if (error instanceof Error && error.message === 'Shadow report input limit exceeded') throw new ReferenceInputError('RESOURCE_LIMIT', 'Shadow report exceeds the bounded evidence limit') + throw error + } +} diff --git a/src/biometrics/shadow/report.ts b/src/biometrics/shadow/report.ts new file mode 100644 index 000000000..08c6cc593 --- /dev/null +++ b/src/biometrics/shadow/report.ts @@ -0,0 +1,15 @@ +import { dataHash } from '../algorithms/canonical' +import { referenceImportSchema } from '../references/contracts' +import { previewReferenceImport } from '../references/normalize' +import type { ShadowStore } from './store' +import { summarizeShadowSnapshot } from './report-core' + +/** Complete, integrity-checked storage snapshot; never canonical vitals. */ +export function reportShadowRun(store: ShadowStore, runId: string, limits: Parameters[1], rawReference?: unknown) { + const snapshot = store.snapshot(runId, limits), { plan } = snapshot.run + const referenceInput = rawReference === undefined ? null : referenceImportSchema.parse(rawReference) + if (referenceInput && (referenceInput.subjectId !== plan.subjectId || referenceInput.side !== plan.side)) throw new Error('Shadow reference subject or side mismatch') + const reference = referenceInput ? previewReferenceImport(referenceInput) : null + const report = summarizeShadowSnapshot(snapshot, reference, plan.execution ? dataHash(plan.execution) : null) + return { ...report, reportHash: dataHash(report) } +} diff --git a/src/biometrics/shadow/runtime-binding.ts b/src/biometrics/shadow/runtime-binding.ts new file mode 100644 index 000000000..2dcd0c713 --- /dev/null +++ b/src/biometrics/shadow/runtime-binding.ts @@ -0,0 +1,71 @@ +import type Database from 'better-sqlite3' +import { z } from 'zod' +import { configHash } from '../algorithms/canonical' +import { manifestHash, validateConfig } from '../algorithms/registry' +import { runtimePayloadSchema } from '../profiles/runtime' + +const integer = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) +const expectedSchema = z.object({ generation: integer.positive(), resolvedHash: z.string().regex(/^[a-f0-9]{64}$/) }).strict() +const receiptSchema = z.object({ generation: integer, applied_hash: z.string(), state: z.literal('applied'), + effective_sample_time_ms: integer, heartbeat_at_ms: integer }) +const sides = ['left', 'right'] as const +const components = ['heart_rate', 'breathing_rate', 'hrv_index'] as const + +/** Read-only observation of both physical-channel baselines. No default-config + * fallback: an unacknowledged channel cannot establish prospective admission. + * The aggregate uses the oldest heartbeat so the installed qualification's age + * limit applies to every numerical component on both sides. */ +export function shadowRuntimeBindingReader(configDb: Database.Database, bioDb: Database.Database, options: { + side: 'left' | 'right' + expected: Record<'left' | 'right', { generation: number, resolvedHash: string }> + assertCurrent: () => void + now?: () => number +}) { + const expected = z.object({ left: expectedSchema, right: expectedSchema }).strict().parse(options.expected) + const side = z.enum(sides).parse(options.side), now = options.now ?? Date.now, guard = options.assertCurrent + const check = () => { + for (const db of [configDb, bioDb]) { + if (!db.open || (!db.readonly && db.pragma('query_only', { simple: true }) !== 1)) throw new Error('Shadow runtime observation requires read-only databases') + } + const result: unknown = guard() + if (result !== undefined) { + void Promise.resolve(result).catch(() => {}) + throw new Error('Shadow database identity guard must be synchronous') + } + } + check() + return () => { + check() + const at = integer.parse(now()) + const heartbeats: number[] = [] + configDb.transaction(() => bioDb.transaction(() => { + for (const channel of sides) { + const target = expected[channel] + const binding = configDb.prepare('SELECT generation FROM biometric_profile_bindings WHERE side = ?').get(channel) as { generation: number } | undefined + if (binding?.generation !== target.generation) throw new Error('Shadow runtime desired generation changed') + const event = configDb.prepare('SELECT next_hash FROM biometric_activation_events WHERE side = ? AND generation = ?').get(channel, target.generation) as { next_hash: string } | undefined + if (event?.next_hash !== target.resolvedHash) throw new Error('Shadow runtime desired configuration changed') + const projection = bioDb.prepare(`SELECT resolved_hash, boundary, CASE WHEN length(CAST(payload AS BLOB)) <= 65536 THEN payload ELSE NULL END AS payload + FROM biometric_runtime_configs WHERE side = ? AND generation = ?`).get(channel, target.generation) as { resolved_hash: string, boundary: string, payload: string | null } | undefined + if (!projection?.payload) throw new Error('Shadow runtime projection missing or oversized') + const payload = runtimePayloadSchema.parse(JSON.parse(projection.payload)) + if (payload.side !== channel || payload.generation !== target.generation || payload.manifestHash !== manifestHash + || payload.resolvedHash !== target.resolvedHash || projection.resolved_hash !== target.resolvedHash + || payload.boundary !== projection.boundary || configHash(validateConfig(payload.resolved)) !== target.resolvedHash) throw new Error('Shadow runtime projection mismatch') + for (const component of components) { + // Read latest intent, including pending/failed generations; never fall + // back to an older applied receipt after a replacement is announced. + const receipt = receiptSchema.parse(bioDb.prepare(`SELECT generation, applied_hash, state, effective_sample_time_ms, heartbeat_at_ms + FROM biometric_runtime_receipts WHERE side = ? AND component = ? ORDER BY generation DESC LIMIT 1`).get(channel, component)) + if (receipt.generation !== target.generation || receipt.applied_hash !== target.resolvedHash + || receipt.effective_sample_time_ms > at || receipt.heartbeat_at_ms > at + || receipt.heartbeat_at_ms < receipt.effective_sample_time_ms) throw new Error('Shadow runtime receipt mismatch or clock skew') + heartbeats.push(receipt.heartbeat_at_ms) + } + } + })())() + check() + return { side, ...expected[side], appliedHash: expected[side].resolvedHash, + appliedGeneration: expected[side].generation, state: 'applied' as const, heartbeatAtMs: Math.min(...heartbeats) } + } +} diff --git a/src/biometrics/shadow/service-host.ts b/src/biometrics/shadow/service-host.ts new file mode 100644 index 000000000..cb5dfbc37 --- /dev/null +++ b/src/biometrics/shadow/service-host.ts @@ -0,0 +1,53 @@ +import { dataHash } from '../algorithms/canonical' +import { runInstalledShadowAttempt, reconcileInstalledShadowRuns, pendingInstalledShadowRun } from './installed-attempt' +import { shadowStoragePolicySchema } from './store' +import { ShadowService } from './service' + +interface InstallationOptions { + installation: Parameters[2] + storagePolicy: unknown + busyTimeoutMs: number + recoveryIntervalMs: number + recoveryMaximumRuns: number +} +const shared = globalThis as typeof globalThis & { + __biometricShadowService?: { service: ShadowService, configurationHash: string } +} + +/** Explicit deployment configuration only; no unqualified execution defaults. */ +export function startShadowService(raw: InstallationOptions) { + const options = structuredClone(raw) + options.storagePolicy = shadowStoragePolicySchema.parse(options.storagePolicy) + if (!Number.isSafeInteger(options.busyTimeoutMs) || options.busyTimeoutMs < 1 || options.busyTimeoutMs > 1000) throw new Error('Invalid shadow database timeout') + const configurationHash = dataHash(options) + const existing = shared.__biometricShadowService + if (existing) { + if (existing.configurationHash !== configurationHash) throw new Error('Shadow service configuration changed; restart required') + return existing.service + } + const service = new ShadowService({ + recoveryIntervalMs: options.recoveryIntervalMs, + recoveryMaximumRuns: options.recoveryMaximumRuns, + recover: cursor => reconcileInstalledShadowRuns(options.installation, options.storagePolicy, options.busyTimeoutMs, cursor), + pending: cursor => pendingInstalledShadowRun(options.installation, options.storagePolicy, options.busyTimeoutMs, cursor), + execute: (runId, revisionId, signal) => runInstalledShadowAttempt(runId, revisionId, options.installation, + options.storagePolicy, options.busyTimeoutMs, signal), + onRecoveryError: () => console.warn('[biometrics] Shadow recovery failed; service will retry'), + onExecutionError: () => console.warn('[biometrics] Shadow scheduling or execution failed'), + }) + shared.__biometricShadowService = { service, configurationHash } + service.start() + return service +} + +export async function stopShadowService() { + const current = shared.__biometricShadowService + if (!current) return + await current.service.stop() + if (shared.__biometricShadowService === current) delete shared.__biometricShadowService +} + +/** Durable stop must be recorded before invoking this local cleanup helper. */ +export async function cancelShadowServiceAttempt(runId: string) { + return await shared.__biometricShadowService?.service.cancel(runId) ?? false +} diff --git a/src/biometrics/shadow/service.ts b/src/biometrics/shadow/service.ts new file mode 100644 index 000000000..e3bc4bfb1 --- /dev/null +++ b/src/biometrics/shadow/service.ts @@ -0,0 +1,113 @@ +import { isShadowCleanupError } from './cleanup-error' +import { z } from 'zod' +import type { ShadowStore } from './store' + +interface ShadowServiceOptions { + execute: (runId: string, revisionId: string | null, signal: AbortSignal) => Promise + recover: (options: Parameters[0]) => ReturnType + recoveryIntervalMs: number + recoveryMaximumRuns: number + onRecoveryError?: () => void + pending?: (options: Parameters[0]) => ReturnType + onExecutionError?: () => void +} + +/** Owns local attempts and bounded maintenance. The store lease and worker flock + * remain authoritative across processes. An aborted attempt keeps its slot until + * execution has finished cleanup. Scheduling reads saved pending plans only. */ +export class ShadowService { + private stopped = false + private cleanupUnconfirmed = false + private timer: ReturnType | null = null + private cursor: string | undefined + private pendingCursor: string | undefined + private active: { runId: string, controller: AbortController, done: Promise } | null = null + private readonly options: ShadowServiceOptions + + constructor(options: ShadowServiceOptions) { + z.number().int().min(10).max(300000).parse(options.recoveryIntervalMs) + z.number().int().min(1).max(100).parse(options.recoveryMaximumRuns) + this.options = { ...options } + } + + start() { + if (this.stopped || this.timer) return + this.reconcile() + } + + private reconcile() { + try { + const result = this.options.recover({ maximumRuns: this.options.recoveryMaximumRuns, + ...(this.cursor ? { afterId: this.cursor } : {}) }) + this.cursor = result.nextCursor ?? undefined + } + catch { + try { + this.options.onRecoveryError?.() + } + catch { /* Logging must not disable recovery. */ } + } + if (!this.stopped && !this.cleanupUnconfirmed && !this.active && this.options.pending) { + try { + const result = this.options.pending({ maximumRuns: this.options.recoveryMaximumRuns, + ...(this.pendingCursor ? { afterId: this.pendingCursor } : {}) }) + this.pendingCursor = result.nextCursor ?? undefined + if (result.run) void this.run(result.run.runId, result.run.revisionId).catch(() => this.reportExecutionError()) + } + catch { this.reportExecutionError() } + } + if (!this.stopped) { + this.timer = setTimeout(() => { + this.timer = null + this.reconcile() + }, this.options.recoveryIntervalMs) + this.timer.unref() + } + } + + private reportExecutionError() { + try { + this.options.onExecutionError?.() + } + catch { /* Logging must not interrupt scheduling or cause rejection. */ } + } + + run(runId: string, revisionId: string | null) { + z.string().uuid().parse(runId) + z.string().uuid().nullable().parse(revisionId) + if (this.stopped) throw new Error('Shadow service stopped') + if (this.cleanupUnconfirmed) throw new Error('Shadow service worker cleanup unconfirmed') + if (this.active) throw new Error('Shadow service busy') + const controller = new AbortController() + // Defer execution so even synchronous failure cannot leave a stale slot. + const done = Promise.resolve().then(() => { + if (controller.signal.aborted) throw new Error('Shadow attempt cancelled') + return this.options.execute(runId, revisionId, controller.signal) + }).catch((error: unknown) => { + if (isShadowCleanupError(error)) this.cleanupUnconfirmed = true + throw error + }).finally(() => { + if (this.active?.done === done) this.active = null + }) + this.active = { runId, controller, done } + return done + } + + async cancel(runId: string) { + z.string().uuid().parse(runId) + const active = this.active + if (!active || active.runId !== runId) return false + active.controller.abort() + await active.done.catch(() => {}) + return !this.cleanupUnconfirmed + } + + async stop() { + this.stopped = true + if (this.timer) clearTimeout(this.timer) + this.timer = null + const active = this.active + active?.controller.abort() + await active?.done.catch(() => {}) + } +} diff --git a/src/biometrics/shadow/startup.ts b/src/biometrics/shadow/startup.ts new file mode 100644 index 000000000..d3646c990 --- /dev/null +++ b/src/biometrics/shadow/startup.ts @@ -0,0 +1,65 @@ +import { isAbsolute } from 'node:path' +import { sql } from 'drizzle-orm' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import type * as configSchema from '@/src/db/schema' +import type * as biometricsSchema from '@/src/db/biometrics-schema' +import { z } from 'zod' +import { packagedShadowHostPath } from '../backtest/location' +import { shadowStoragePolicySchema } from './store' +import { startShadowService } from './service-host' +import { shadowPlanTemplateSchema } from './preparation-contracts' + +const absolutePath = z.string().min(1).max(4096).refine(value => isAbsolute(value) && !value.includes('\0'), 'Expected absolute installation path') +export const shadowServiceConfigurationSchema = z.object({ + version: z.literal('shadow-service-configuration-v1'), + pythonPath: absolutePath, + modulesPath: absolutePath, + presenceFile: absolutePath, + storagePolicy: shadowStoragePolicySchema, + busyTimeoutMs: z.number().int().min(1).max(1000), + recoveryIntervalMs: z.number().int().min(10).max(300000), + recoveryMaximumRuns: z.number().int().min(1).max(100), + planTemplates: z.array(shadowPlanTemplateSchema).max(8).refine(items => new Set(items.map(item => item.id)).size === items.length, 'Duplicate shadow template IDs').optional(), +}).strict().refine(value => (value.planTemplates ?? []).every(template => template.maximumPoints <= value.storagePolicy.maximumPoints + && template.maximumBytes <= value.storagePolicy.maximumBytes), 'Shadow templates exceed deployment storage policy') + +/** Environment configuration is an operator-owned resource policy, not evidence + * of qualification. Every execution still verifies the installed qualification + * registry, current build, acknowledged bindings and observed pod health. */ +export function configuredShadowService() { + const raw = process.env.SLEEPYPOD_BIOMETRICS_SHADOW_SERVICE + if (raw === undefined) return null + if (Buffer.byteLength(raw) > 16384) throw new Error('Shadow service configuration too large') + return shadowServiceConfigurationSchema.parse(JSON.parse(raw)) +} + +/** Resolve databases from the running app, never from an API/request or an + * independently configured database URL. Missing configuration leaves no timer, + * database connection, Python probe or sensor subscription behind. */ +export function startConfiguredShadowService(configuration: BetterSQLite3Database, + biometrics: BetterSQLite3Database) { + try { + const options = configuredShadowServiceOptions(configuration, biometrics) + return options ? startShadowService(options) : null + } + catch { + // Avoid exposing deployment paths or arbitrary environment content in logs. + console.warn('[biometrics] Shadow service unavailable; check deployment configuration and installed package') + return null + } +} + +/** Resolve server-owned paths for submission without starting background work. */ +export function configuredShadowServiceOptions(configuration: BetterSQLite3Database, + biometrics: BetterSQLite3Database) { + const options = configuredShadowService() + if (!options) return null + const configurationDatabase = absolutePath.parse(configuration.all<{ name: string, file: string }>(sql`PRAGMA database_list`).find(row => row.name === 'main')?.file) + const biometricsDatabase = absolutePath.parse(biometrics.all<{ name: string, file: string }>(sql`PRAGMA database_list`).find(row => row.name === 'main')?.file) + if (configurationDatabase === biometricsDatabase) throw new Error('Shadow databases must be separate') + return { installation: { workerPath: packagedShadowHostPath(), + pythonPath: options.pythonPath, modulesPath: options.modulesPath, presenceFile: options.presenceFile, + configurationDatabase, biometricsDatabase }, storagePolicy: options.storagePolicy, + busyTimeoutMs: options.busyTimeoutMs, recoveryIntervalMs: options.recoveryIntervalMs, + recoveryMaximumRuns: options.recoveryMaximumRuns } +} diff --git a/src/biometrics/shadow/store.ts b/src/biometrics/shadow/store.ts new file mode 100644 index 000000000..aff965682 --- /dev/null +++ b/src/biometrics/shadow/store.ts @@ -0,0 +1,357 @@ +import { shadowReceiptEnvelopeSchema, shadowPartialEnvelopeSchema, validateShadowPartialReceipt, validateShadowWorkerReceipt } from './receipt' +import { shadowPlanSchema } from './plan-contracts' +export { shadowPlanSchema, shadowMonitoringSchema, shadowExpectedBindingsSchema } from './plan-contracts' +import type Database from 'better-sqlite3' +import { randomUUID } from 'node:crypto' +import { z } from 'zod' +import { canonicalJson, dataHash } from '../algorithms/canonical' + +const integer = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) +const positive = integer.positive() +const hash = z.string().regex(/^[a-f0-9]{64}$/) +const metric = z.enum(['heart_rate', 'breathing_rate', 'window_ibi_variability_v1']) +const reading = z.object({ value: z.number().finite().nullable(), quality: z.number().min(0).max(1) }).strict() +export const shadowActiveProvenanceSchema = z.object({ processingRunId: z.string().uuid(), sourceSide: z.enum(['left', 'right']), + generation: integer, manifestHash: hash }).strict() +export const shadowSampleSchema = z.object({ sequence: integer, side: z.enum(['left', 'right']), timestampMs: integer, inputHash: hash.nullable(), + candidateConfigHash: hash, activeConfigHash: hash, + readings: z.array(z.object({ metric, candidate: reading, active: reading }).strict()).min(1).max(3), + pairing: z.object({ policy: z.literal('exact_timestamp_union_v1'), candidatePresent: z.boolean(), activePresent: z.boolean() }).strict().optional(), + inputIdentityKind: z.enum(['candidate_prefix', 'active_provenance', 'unavailable']).optional(), + activeProvenance: shadowActiveProvenanceSchema.optional(), + candidateProvenance: z.object({ sourceSide: z.enum(['left', 'right']), outputSequence: integer, + flags: z.array(z.string().max(200)).max(32).nullable(), limitations: z.array(z.string().max(200)).max(33) }).strict().optional(), +}).strict() +export const shadowStoragePolicySchema = z.object({ maximumRuns: positive, maximumPoints: positive, maximumBytes: positive }).strict() +type Plan = z.infer +interface RunRow { + worker_receipt: string | null + worker_receipt_hash: string | null + id: string + plan_hash: string + plan: string + side: 'left' | 'right' + state: 'pending' | 'running' | 'completed' | 'stopped' | 'failed' + attempt_id: string | null + lease_until_ms: number | null + heartbeat_at_ms: number | null + created_at_ms: number + ended_at_ms: number | null + reason: string | null + point_count: number + byte_count: number +} +export interface ShadowLease { runId: string, attemptId: string } + +/** Storage only: accepts observations from a separately admitted shipped + * processor. No algorithm imports, canonical writes, activation or hardware. + * The host must verify qualification/build and enforce CPU/RSS before claim. */ +export class ShadowStore { + private readonly policy: z.infer | null + constructor(private readonly db: Database.Database, policy?: unknown, private readonly now = Date.now) { + this.policy = policy === undefined ? null : shadowStoragePolicySchema.parse(policy) + } + + private executionPolicy() { + if (!this.policy) throw new Error('Shadow execution storage policy required') + return this.policy + } + + list(options: { side: 'left' | 'right', maximumRuns: number, afterId?: string }) { + const input = z.object({ side: z.enum(['left', 'right']), maximumRuns: positive.max(50), afterId: z.string().uuid().optional() }).strict().parse(options) + return this.db.transaction(() => { + const rows = this.db.prepare('SELECT id FROM biometric_shadow_runs WHERE side = ? AND id > ? ORDER BY id LIMIT ?') + .all(input.side, input.afterId ?? '', input.maximumRuns + 1) as { id: string }[] + const items = rows.slice(0, input.maximumRuns).map(row => this.get(row.id)) + return { items, nextCursor: rows.length > input.maximumRuns ? items.at(-1)?.id ?? null : null } + }).deferred() + } + + get(id: string) { + const row = this.db.prepare('SELECT * FROM biometric_shadow_runs WHERE id = ?').get(id) as RunRow | undefined + if (!row) throw new Error('Unknown shadow run') + const plan = shadowPlanSchema.parse(JSON.parse(row.plan)) + if (dataHash(plan) !== row.plan_hash || row.side !== plan.side) throw new Error('Shadow plan integrity mismatch') + let workerReceipt = null + if (row.worker_receipt !== null) { + if (Buffer.byteLength(row.worker_receipt) > 16384) throw new Error('Shadow receipt exceeds storage limit') + workerReceipt = shadowReceiptEnvelopeSchema.parse(JSON.parse(row.worker_receipt)) + if (!plan.execution || dataHash(workerReceipt) !== row.worker_receipt_hash || workerReceipt.runId !== row.id + || workerReceipt.attemptId !== row.attempt_id || workerReceipt.planHash !== row.plan_hash) throw new Error('Shadow receipt integrity mismatch') + if (workerReceipt.schema === 'biometric-shadow-receipt-v1') { + if (row.state !== 'completed' || workerReceipt.recordedAtMs !== row.ended_at_ms) throw new Error('Shadow receipt integrity mismatch') + validateShadowWorkerReceipt(workerReceipt.receipt, plan.execution) + } + else { + if (!['failed', 'stopped'].includes(row.state) || row.ended_at_ms === null || workerReceipt.recordedAtMs < row.ended_at_ms) throw new Error('Shadow receipt integrity mismatch') + validateShadowPartialReceipt(workerReceipt.receipt, plan.execution) + } + } + else if (row.worker_receipt_hash !== null) throw new Error('Shadow receipt integrity mismatch') + return { ...row, plan, workerReceipt } + } + + /** Complete bounded snapshot for scoring. Refuse oversize data rather than + * silently scoring a paginated/downsampled subset. */ + snapshot(id: string, limits: { maximumPoints: number, maximumBytes: number }) { + z.object({ maximumPoints: positive.max(100000), maximumBytes: positive.max(32 * 1024 * 1024) }).strict().parse(limits) + return this.db.transaction(() => { + const run = this.get(id) + const totals = this.db.prepare(`SELECT COUNT(*) AS points, COALESCE(SUM(byte_length), 0) AS bytes, + COALESCE(SUM(length(CAST(payload AS BLOB))), 0) AS actual_bytes FROM biometric_shadow_samples WHERE run_id = ?`).get(id) as { points: number, bytes: number, actual_bytes: number } + if (totals.points !== run.point_count || totals.bytes !== run.byte_count || totals.bytes !== totals.actual_bytes) throw new Error('Shadow sample totals integrity mismatch') + if (totals.points > limits.maximumPoints || totals.actual_bytes > limits.maximumBytes) throw new Error('Shadow report input limit exceeded') + const rows = this.db.prepare('SELECT sequence, timestamp_ms, content_hash, payload, byte_length FROM biometric_shadow_samples WHERE run_id = ? ORDER BY sequence').all(id) as { sequence: number, timestamp_ms: number, content_hash: string, payload: string, byte_length: number }[] + const capturedAtMs = this.now() + let previousTime = -1 + const samples = rows.map((row, index) => { + const envelope = shadowSampleSchema.extend({ attemptId: z.string().uuid() }).parse(JSON.parse(row.payload)) + const { attemptId, ...sample } = envelope + if (dataHash(sample) !== row.content_hash || row.byte_length !== Buffer.byteLength(row.payload) + || row.sequence !== index || sample.sequence !== index || sample.timestampMs !== row.timestamp_ms + || sample.timestampMs <= previousTime || sample.timestampMs < run.plan.startMs || sample.timestampMs >= run.plan.endMs + || sample.side !== run.side || sample.candidateConfigHash !== run.plan.candidateConfigHash || sample.activeConfigHash !== run.plan.activeConfigHash + || sample.readings.length !== run.plan.metrics.length || new Set(sample.readings.map(row => row.metric)).size !== run.plan.metrics.length + || sample.readings.some(row => !run.plan.metrics.includes(row.metric))) throw new Error('Shadow sample integrity mismatch') + previousTime = sample.timestampMs + return { ...sample, attemptId } + }) + const snapshot = { schema: 'biometric-shadow-snapshot-v1' as const, run, capturedAtMs, samples } + return { ...snapshot, snapshotHash: dataHash(snapshot) } + }).deferred() + } + + cursor(id: string) { + return this.db.transaction(() => { + const run = this.get(id) + const last = this.db.prepare('SELECT sequence, timestamp_ms FROM biometric_shadow_samples WHERE run_id = ? ORDER BY sequence DESC LIMIT 1').get(id) as { sequence: number, timestamp_ms: number } | undefined + if ((last ? last.sequence + 1 : 0) !== run.point_count) throw new Error('Shadow cursor integrity mismatch') + return { nextSequence: run.point_count, lastTimestampMs: last?.timestamp_ms ?? null } + }).deferred() + } + + findCreated(raw: unknown, idempotencyKey: string) { + const plan = shadowPlanSchema.parse(raw) + z.string().min(1).max(200).parse(idempotencyKey) + const previous = this.db.prepare('SELECT id, plan_hash FROM biometric_shadow_runs WHERE idempotency_key = ?').get(idempotencyKey) as { id: string, plan_hash: string } | undefined + if (!previous) return null + if (previous.plan_hash !== dataHash(plan)) throw new Error('Shadow idempotency conflict') + return this.get(previous.id) + } + + create(raw: unknown, idempotencyKey: string, admit?: (plan: Plan) => void) { + const policy = this.executionPolicy() + const plan = shadowPlanSchema.parse(raw), planHash = dataHash(plan) + z.string().min(1).max(200).parse(idempotencyKey) + return this.db.transaction(() => { + const previous = this.findCreated(plan, idempotencyKey) + if (previous) { + return previous + } + const now = this.now() + if (plan.startMs < now) throw new Error('Shadow window must be prospective') + if (plan.maximumPoints > policy.maximumPoints || plan.maximumBytes > policy.maximumBytes) throw new Error('Shadow plan exceeds storage policy') + const count = (this.db.prepare('SELECT COUNT(*) AS count FROM biometric_shadow_runs').get() as { count: number }).count + if (count >= policy.maximumRuns) throw new Error('Shadow run quota exceeded') + const admitted: unknown = admit?.(structuredClone(plan)) + if (admitted !== undefined) { + void Promise.resolve(admitted).catch(() => {}) + throw new Error('Shadow admission must complete synchronously') + } + if (plan.startMs < this.now()) throw new Error('Shadow window must be prospective') + const id = randomUUID() + this.db.prepare(`INSERT INTO biometric_shadow_runs + (id, idempotency_key, plan_hash, plan, side, state, created_at_ms) VALUES (?, ?, ?, ?, ?, 'pending', ?)`).run(id, idempotencyKey, planHash, canonicalJson(plan), plan.side, now) + return this.get(id) + }).immediate() + } + + /** Read-only bounded queue scan. It never resumes a running attempt after a + * restart: recovery must revoke that authority, retaining incomplete evidence. + * The caller still has to claim and revalidate the selected plan. */ + pending(options: { maximumRuns: number, afterId?: string }) { + const input = z.object({ maximumRuns: positive.max(100), afterId: z.string().uuid().optional() }).strict().parse(options) + return this.db.transaction(() => { + const now = integer.parse(this.now()) + const rows = this.db.prepare('SELECT id, state FROM biometric_shadow_runs WHERE id > ? ORDER BY id LIMIT ?') + .all(input.afterId ?? '', input.maximumRuns + 1) as { id: string, state: RunRow['state'] }[] + const batch = rows.slice(0, input.maximumRuns) + for (const row of batch) { + if (row.state !== 'pending') continue + const run = this.get(row.id) + if (run.attempt_id !== null || run.lease_until_ms !== null) throw new Error('Invalid pending shadow authority') + if (!run.plan.execution || !run.plan.expectedBindings || run.plan.candidateRevisionId === undefined || now < run.plan.startMs || now >= run.plan.endMs) continue + return { run: { runId: run.id, revisionId: run.plan.candidateRevisionId }, nextCursor: row.id } + } + return { run: null, nextCursor: rows.length > input.maximumRuns ? batch.at(-1)?.id ?? null : null } + }).deferred() + } + + /** Admission is a trusted synchronous host check, never a client boolean. */ + claim(id: string, leaseMs: number, admit: (plan: Plan) => void): ShadowLease { + this.executionPolicy() + z.number().int().min(1000).max(3600000).parse(leaseMs) + return this.db.transaction(() => { + const run = this.get(id), now = this.now() + if (!['pending', 'running'].includes(run.state) || (run.state === 'running' && (run.lease_until_ms ?? 0) > now)) throw new Error('Shadow run unavailable') + if (now < run.plan.startMs || now >= run.plan.endMs) throw new Error('Outside shadow window') + if (this.db.prepare(`SELECT id FROM biometric_shadow_runs WHERE side = ? AND state = 'running' AND lease_until_ms > ? AND id != ?`).get(run.side, now, id)) throw new Error('Side already has an active shadow worker') + // Pass a copy: an admission callback cannot mutate the frozen plan. + const admitted: unknown = admit(structuredClone(run.plan)) + if (admitted !== undefined) { + void Promise.resolve(admitted).catch(() => undefined) + throw new Error('Shadow admission must complete synchronously') + } + const attemptId = randomUUID() + this.db.prepare('UPDATE biometric_shadow_runs SET state = \'running\', attempt_id = ?, lease_until_ms = ?, heartbeat_at_ms = ? WHERE id = ?') + .run(attemptId, Math.min(run.plan.endMs + (run.plan.joining?.drainGraceMs ?? 0), now + leaseMs), now, id) + return { runId: id, attemptId } + }).immediate() + } + + private current(lease: ShadowLease) { + const run = this.get(lease.runId), now = this.now() + if (run.state !== 'running' || run.attempt_id !== lease.attemptId || (run.lease_until_ms ?? 0) <= now || now >= run.plan.endMs + (run.plan.joining?.drainGraceMs ?? 0)) throw new Error('Stale shadow lease') + return { run, now } + } + + append(lease: ShadowLease, raw: unknown) { + const policy = this.executionPolicy() + const sample = shadowSampleSchema.parse(raw), contentHash = dataHash(sample) + return this.db.transaction(() => { + const { run, now } = this.current(lease) + if (sample.side !== run.side || sample.candidateConfigHash !== run.plan.candidateConfigHash || sample.activeConfigHash !== run.plan.activeConfigHash + || sample.timestampMs < run.plan.startMs || sample.timestampMs >= run.plan.endMs || sample.timestampMs > now + || sample.readings.length !== run.plan.metrics.length || new Set(sample.readings.map(row => row.metric)).size !== run.plan.metrics.length + || sample.readings.some(row => !run.plan.metrics.includes(row.metric))) throw new Error('Shadow sample differs from frozen plan') + const previous = this.db.prepare('SELECT content_hash FROM biometric_shadow_samples WHERE run_id = ? AND sequence = ?').get(run.id, sample.sequence) as { content_hash: string } | undefined + if (previous) { + if (previous.content_hash !== contentHash) throw new Error('Shadow sample idempotency conflict') + return { state: 'already_recorded' as const } + } + const last = this.db.prepare('SELECT sequence, timestamp_ms FROM biometric_shadow_samples WHERE run_id = ? ORDER BY sequence DESC LIMIT 1').get(run.id) as { sequence: number, timestamp_ms: number } | undefined + if (sample.sequence !== (last ? last.sequence + 1 : 0) || (last && sample.timestampMs <= last.timestamp_ms)) throw new Error('Out-of-order shadow sample') + const payload = canonicalJson({ ...sample, attemptId: lease.attemptId }), bytes = Buffer.byteLength(payload) + const totals = this.db.prepare('SELECT COALESCE(SUM(point_count), 0) AS points, COALESCE(SUM(byte_count), 0) AS bytes FROM biometric_shadow_runs').get() as { points: number, bytes: number } + if (run.point_count + 1 > run.plan.maximumPoints || run.byte_count + bytes > run.plan.maximumBytes + || totals.points + 1 > policy.maximumPoints || totals.bytes + bytes > policy.maximumBytes) { + this.db.prepare('UPDATE biometric_shadow_runs SET state = \'stopped\', reason = \'storage_quota\', ended_at_ms = ?, lease_until_ms = NULL WHERE id = ?').run(now, run.id) + return { state: 'quota_reached' as const } + } + this.db.prepare('INSERT INTO biometric_shadow_samples VALUES (?, ?, ?, ?, ?, ?)').run(run.id, sample.sequence, sample.timestampMs, contentHash, payload, bytes) + this.db.prepare('UPDATE biometric_shadow_runs SET point_count = point_count + 1, byte_count = byte_count + ?, heartbeat_at_ms = ? WHERE id = ?').run(bytes, now, run.id) + return { state: 'recorded' as const } + }).immediate() + } + + /** Host/operator cancellation also works after a worker has disappeared or + * the observation window has ended. Terminal evidence is preserved. */ + stop(id: string, reason: string) { + z.string().min(1).max(500).parse(reason) + return this.db.transaction(() => { + this.get(id) + this.db.prepare('UPDATE biometric_shadow_runs SET state = \'stopped\', reason = ?, ended_at_ms = ?, lease_until_ms = NULL WHERE id = ? AND state IN (\'pending\', \'running\')') + .run(reason, this.now(), id) + return this.get(id) + }).immediate() + } + + renew(lease: ShadowLease, leaseMs: number) { + z.number().int().min(1000).max(3600000).parse(leaseMs) + this.db.transaction(() => { + const { run, now } = this.current(lease) + this.db.prepare('UPDATE biometric_shadow_runs SET lease_until_ms = ?, heartbeat_at_ms = ? WHERE id = ?').run(Math.min(run.plan.endMs + (run.plan.joining?.drainGraceMs ?? 0), now + leaseMs), now, run.id) + }).immediate() + } + + /** Bounded keyset maintenance. Revoke expired authority, not a claim that an + * OS process was reaped. A resumed worker cannot append/renew after recovery; + * the kernel execution slot still excludes overlap until that worker exits. */ + reconcileExpired(options: { maximumRuns: number, afterId?: string }) { + const input = z.object({ maximumRuns: positive.max(100), afterId: z.string().uuid().optional() }).strict().parse(options) + return this.db.transaction(() => { + const now = integer.parse(this.now()) + const rows = this.db.prepare('SELECT id, state FROM biometric_shadow_runs WHERE id > ? ORDER BY id LIMIT ?') + .all(input.afterId ?? '', input.maximumRuns + 1) as { id: string, state: RunRow['state'] }[] + const batch = rows.slice(0, input.maximumRuns) + const recovered: { runId: string, attemptId: string | null, reason: string }[] = [] + for (const row of batch) { + if (row.state !== 'pending' && row.state !== 'running') continue + const run = this.get(row.id) + let reason: string + if (run.state === 'pending') { + if (now < run.plan.endMs) continue + if (run.attempt_id !== null || run.lease_until_ms !== null) throw new Error('Invalid pending shadow authority') + reason = 'shadow_observation_window_missed' + } + else { + z.string().uuid().parse(run.attempt_id) + integer.parse(run.lease_until_ms) + if ((run.lease_until_ms ?? 0) > now && now < run.plan.endMs + (run.plan.joining?.drainGraceMs ?? 0)) continue + reason = 'shadow_lease_expired_cleanup_unconfirmed' + } + const updated = this.db.prepare(`UPDATE biometric_shadow_runs SET state = 'failed', reason = ?, ended_at_ms = ?, lease_until_ms = NULL + WHERE id = ? AND state = ? AND attempt_id IS ? AND lease_until_ms IS ?`) + .run(reason, now, run.id, run.state, run.attempt_id, run.lease_until_ms) + if (updated.changes !== 1) throw new Error('Shadow recovery authority changed') + recovered.push({ runId: run.id, attemptId: run.attempt_id, reason }) + } + return { scanned: batch.length, recovered, nextCursor: rows.length > input.maximumRuns ? batch.at(-1)?.id ?? null : null } + }).immediate() + } + + /** After the host has reaped its worker, record an expired attempt's failure. + * A newer attempt or an already terminal run is never changed. */ + failExpired(lease: ShadowLease, reason: string) { + z.string().min(1).max(500).parse(reason) + return this.db.transaction(() => { + const run = this.get(lease.runId), now = this.now() + if (run.state !== 'running' || run.attempt_id !== lease.attemptId + || ((run.lease_until_ms ?? 0) > now && now < run.plan.endMs + (run.plan.joining?.drainGraceMs ?? 0))) return false + this.db.prepare('UPDATE biometric_shadow_runs SET state = \'failed\', reason = ?, ended_at_ms = ?, lease_until_ms = NULL WHERE id = ?') + .run(reason, now, run.id) + return true + }).immediate() + } + + /** Attach cleanup evidence without changing terminal authority, even when an + * operator or recovery already ended this exact attempt. Never replace evidence. */ + recordPartialReceipt(lease: ShadowLease, raw: unknown) { + return this.db.transaction(() => { + const run = this.get(lease.runId), now = this.now() + if (run.attempt_id !== lease.attemptId || !['failed', 'stopped'].includes(run.state)) return false + if (!run.plan.execution || run.ended_at_ms === null || now < run.ended_at_ms) throw new Error('Shadow partial receipt requires a terminal executable attempt') + const receipt = validateShadowPartialReceipt(raw, run.plan.execution) + if (run.workerReceipt) { + if (run.workerReceipt.schema === 'biometric-shadow-partial-receipt-v1' && dataHash(run.workerReceipt.receipt) === dataHash(receipt)) return true + throw new Error('Shadow receipt already recorded') + } + const envelope = shadowPartialEnvelopeSchema.parse({ schema: 'biometric-shadow-partial-receipt-v1', runId: run.id, + attemptId: lease.attemptId, planHash: run.plan_hash, recordedAtMs: now, receipt }) + const encoded = canonicalJson(envelope) + if (Buffer.byteLength(encoded) > 16384) throw new Error('Shadow receipt exceeds storage limit') + this.db.prepare('UPDATE biometric_shadow_runs SET worker_receipt = ?, worker_receipt_hash = ? WHERE id = ?') + .run(encoded, dataHash(envelope), run.id) + return true + }).immediate() + } + + finish(lease: ShadowLease, state: 'completed' | 'stopped' | 'failed', reason: string, rawReceipt?: unknown) { + z.enum(['completed', 'stopped', 'failed']).parse(state) + z.string().min(1).max(500).parse(reason) + this.db.transaction(() => { + const { run, now } = this.current(lease) + let receipt = null, receiptHash = null + if (state === 'completed' && run.plan.execution) { + receipt = canonicalJson(shadowReceiptEnvelopeSchema.parse({ schema: 'biometric-shadow-receipt-v1', + runId: run.id, attemptId: lease.attemptId, planHash: run.plan_hash, recordedAtMs: now, + receipt: validateShadowWorkerReceipt(rawReceipt, run.plan.execution) })) + if (Buffer.byteLength(receipt) > 16384) throw new Error('Shadow receipt exceeds storage limit') + receiptHash = dataHash(JSON.parse(receipt)) + } + else if (rawReceipt !== undefined) throw new Error('Worker receipt requires a completed executable plan') + this.db.prepare('UPDATE biometric_shadow_runs SET state = ?, reason = ?, ended_at_ms = ?, lease_until_ms = NULL, worker_receipt = ?, worker_receipt_hash = ? WHERE id = ?') + .run(state, reason, now, receipt, receiptHash, run.id) + }).immediate() + } +} diff --git a/src/biometrics/shadow/stream-join.ts b/src/biometrics/shadow/stream-join.ts new file mode 100644 index 000000000..e1534295c --- /dev/null +++ b/src/biometrics/shadow/stream-join.ts @@ -0,0 +1,105 @@ +import { dataHash } from '../algorithms/canonical' +import type { ShadowLease, ShadowStore } from './store' +import { shadowWorkerOutputSchema } from './worker-protocol' +import { shadowActiveReadingSchema } from './worker-sink' +import type { z } from 'zod' + +type Candidate = z.infer +type Active = z.infer +interface Pair { candidate?: Candidate, active?: Active } + +/** Exact timestamp union, with bounded buffering and a frozen lateness policy. + * The supervisor advances the clock and validates worker batches beforehand. + * No nearby timestamp is relabelled to make a match. */ +export class ShadowStreamJoin { + private readonly plan: ReturnType['plan'] + private readonly policy: NonNullable['plan']['joining']> + private readonly pending = new Map() + private sealedThrough: number + private lastClock = -1 + private nextSequence: number + private closed = false + + constructor(private readonly store: ShadowStore, private readonly lease: ShadowLease) { + const run = store.get(lease.runId) + if (run.state !== 'running' || run.attempt_id !== lease.attemptId || !run.plan.joining) throw new Error('Shadow join requires a live attempt and frozen join policy') + this.plan = structuredClone(run.plan) + this.policy = { ...run.plan.joining } + const cursor = store.cursor(lease.runId) + this.nextSequence = cursor.nextSequence + this.sealedThrough = cursor.lastTimestampMs ?? run.plan.startMs - 1 + } + + private fail(reason: string): never { + this.closed = true + // This is a worker failure, not an operator stop. Never stop a replacement. + try { + this.store.finish(this.lease, 'failed', reason) + } + catch { /* A lost lease already fences this attempt. */ } + throw new Error(reason) + } + + private push(kind: 'candidate' | 'active', row: Candidate | Active) { + if (this.closed) throw new Error('Shadow join is closed') + if (row.side !== this.plan.side || row.configHash !== (kind === 'candidate' ? this.plan.candidateConfigHash : this.plan.activeConfigHash) + || row.timestampMs < this.plan.startMs || row.timestampMs >= this.plan.endMs) return this.fail('join_identity_mismatch') + if (row.timestampMs <= this.sealedThrough) return this.fail('join_late_arrival') + for (const metric of this.plan.metrics) { + const reading = row.readings.find(reading => reading.metric === metric) + if (!reading || reading.windowMs !== (metric === 'window_ibi_variability_v1' ? this.plan.windowIbiWindowMs : null)) return this.fail('join_metric_mismatch') + } + const previous = this.pending.get(row.timestampMs) + if (previous?.[kind]) { + if (dataHash(previous[kind]) !== dataHash(row)) return this.fail('join_conflicting_duplicate') + return + } + if (!previous && this.pending.size >= this.policy.maximumPendingPoints) return this.fail('join_queue_quota') + this.pending.set(row.timestampMs, { ...previous, [kind]: row }) + } + + candidate(raw: unknown) { this.push('candidate', shadowWorkerOutputSchema.parse(raw)) } + + active(raw: unknown) { this.push('active', shadowActiveReadingSchema.parse(raw)) } + + advance(nowMs: number) { + if (this.closed) throw new Error('Shadow join is closed') + if (!Number.isSafeInteger(nowMs) || nowMs < this.plan.startMs || nowMs < this.lastClock) return this.fail('join_clock_regression') + this.lastClock = nowMs + const through = Math.min(this.plan.endMs - 1, nowMs - this.policy.maximumLatenessMs) + let emitted = 0 + for (const timestampMs of [...this.pending.keys()].filter(time => time <= through).sort((a, b) => a - b)) { + const pair = this.pending.get(timestampMs) + if (!pair) throw new Error('Missing shadow join point') + const { candidate, active } = pair + const readings = this.plan.metrics.map((metric) => { + const next = candidate?.readings.find(reading => reading.metric === metric) + const previous = active?.readings.find(reading => reading.metric === metric) + return { metric, candidate: next ? { value: next.value, quality: next.quality } : { value: null, quality: 0 }, + active: previous ? { value: previous.value, quality: previous.quality } : { value: null, quality: 0 } } + }) + const inputHash = candidate?.inputHash ?? active?.inputHash ?? null + const result = this.store.append(this.lease, { sequence: this.nextSequence, timestampMs, side: this.plan.side, + candidateConfigHash: this.plan.candidateConfigHash, activeConfigHash: this.plan.activeConfigHash, inputHash, + inputIdentityKind: candidate ? 'candidate_prefix' : inputHash ? 'active_provenance' : 'unavailable', + ...(active?.provenance ? { activeProvenance: active.provenance } : {}), + pairing: { policy: 'exact_timestamp_union_v1', candidatePresent: Boolean(candidate), activePresent: Boolean(active) }, readings, + ...(candidate + ? { candidateProvenance: { sourceSide: candidate.sourceSide, outputSequence: candidate.sequence, flags: candidate.flags, + limitations: active ? candidate.limitations : [...candidate.limitations, 'missing_contemporaneous_active_reading'] } } + : {}) }) + if (result.state === 'quota_reached') { + this.closed = true + return { emitted, stopped: true } + } + // Keep the point and sequence unchanged until storage acknowledges it; + // this permits an exact retry after an uncertain committed write. + this.pending.delete(timestampMs) + this.nextSequence++ + this.sealedThrough = timestampMs + emitted++ + } + this.sealedThrough = Math.max(this.sealedThrough, through) + return { emitted, stopped: false } + } +} diff --git a/src/biometrics/shadow/submission-contracts.ts b/src/biometrics/shadow/submission-contracts.ts new file mode 100644 index 000000000..e43a840cb --- /dev/null +++ b/src/biometrics/shadow/submission-contracts.ts @@ -0,0 +1,10 @@ +import { z } from 'zod' +import { shadowPlanSchema, shadowExpectedBindingsSchema } from './plan-contracts' +import { shadowExecutionPolicySchema } from './execution-policy' + +export const submitShadowSchema = z.object({ + plan: shadowPlanSchema.safeExtend({ candidateRevisionId: z.string().uuid().nullable(), + expectedBindings: shadowExpectedBindingsSchema, execution: shadowExecutionPolicySchema }), + idempotencyKey: z.string().min(1).max(200), +}).strict() + diff --git a/src/biometrics/shadow/submission.ts b/src/biometrics/shadow/submission.ts new file mode 100644 index 000000000..0a34dd128 --- /dev/null +++ b/src/biometrics/shadow/submission.ts @@ -0,0 +1,57 @@ +import { z } from 'zod' +import { dataHash } from '../algorithms/canonical' +import { submitShadowSchema } from './submission-contracts' +import { loadInstalledShadowHost, inspectInstalledShadowHost } from './installed-host' +import { openShadowStateDatabase } from './active-database' +import { installedShadowIdentity } from './device-identity' +import { getSensorFrameTimes } from '@/src/streaming/piezoStream' +import type { runInstalledShadowAttempt } from './installed-attempt' + +export { submitShadowSchema } from './submission-contracts' + +/** Assertions in a proposed plan are checked against installed qualification and + * fresh observations. A request cannot supply paths, resolved configs or health. + * Exact retries return durable evidence before any new Python/health work. */ +export async function submitInstalledShadow(raw: z.input, + installation: Parameters[2], storagePolicy: unknown, busyTimeoutMs: number, signal?: AbortSignal) { + const input = submitShadowSchema.parse(raw), paths = structuredClone(installation), policy = structuredClone(storagePolicy) + z.number().int().min(1).max(1000).parse(busyTimeoutMs) + if (busyTimeoutMs > input.plan.execution.sources.databaseBusyTimeoutMs) throw new Error('Shadow store timeout exceeds saved execution policy') + if (signal?.aborted) throw new Error('Shadow submission cancelled') + const loaded = loadInstalledShadowHost(paths.workerPath) + loaded.assertCurrent() + const database = openShadowStateDatabase(paths.biometricsDatabase, { busyTimeoutMs }) + let owned: Awaited> | undefined + try { + database.assertCurrent() + const store = new loaded.api.ShadowStore(database.db, policy) + const previous = store.findCreated(input.plan, input.idempotencyKey) + if (previous) return previous + const inspected = await inspectInstalledShadowHost(paths.workerPath, paths, signal) + if (signal?.aborted) throw new Error('Shadow submission cancelled') + inspected.assertCurrent() + database.assertCurrent() + const entries = inspected.api.readInstalledShadowQualifications(paths.workerPath).entries.filter(entry => dataHash(entry) === input.plan.qualificationHash) + if (entries.length !== 1) throw new Error('Shadow qualification missing or ambiguous') + const observed = installedShadowIdentity(inspected, entries[0]) + const identity = () => { + database.assertCurrent() + return observed() + } + owned = await inspected.api.openProfiledShadowHost(input.plan, { ...paths, identity, + piezoReceivedAt: () => getSensorFrameTimes()['piezo-dual'] }, input.plan.candidateRevisionId, signal) + if (signal?.aborted) throw new Error('Shadow submission cancelled') + const host = owned.host + return store.create(input.plan, input.idempotencyKey, (plan) => { + inspected.assertCurrent() + database.assertCurrent() + host.admit(plan) + }) + } + finally { + try { + owned?.close() + } + finally { database.close() } + } +} diff --git a/src/biometrics/shadow/tests/active-reader.test.ts b/src/biometrics/shadow/tests/active-reader.test.ts new file mode 100644 index 000000000..1cd55b6e8 --- /dev/null +++ b/src/biometrics/shadow/tests/active-reader.test.ts @@ -0,0 +1,350 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { randomUUID } from 'node:crypto' +import { mkdtempSync, rmSync, renameSync, copyFileSync, symlinkSync, writeFileSync, utimesSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join, dirname } from 'node:path' +import { fileURLToPath } from 'node:url' +import { afterEach, expect, it, vi } from 'vitest' +import { biometricManifest, defaultConfig, manifestHash } from '../../algorithms/registry' +import { configHash } from '../../algorithms/canonical' +import { plannedShadowSourceFactory } from '../planned-sources' +import { shadowPlanSchema } from '../store' +import { openShadowActiveDatabase, openShadowStateDatabase } from '../active-database' +import { openShadowLiveSources } from '../live-sources' +import { publishProspectiveFrame } from '../../../streaming/prospectiveFrameTap' +import { ShadowActiveReader } from '../active-reader' +import { ShadowStore } from '../store' +import { ShadowStreamJoin } from '../stream-join' +const start = 1700000000000, hash = configHash(defaultConfig) +const cleanups: (() => void)[] = [] +afterEach(() => { + cleanups.splice(0).forEach(cleanup => cleanup()) + vi.useRealTimers() +}) +function setup(maximumRows = 10) { + const directory = mkdtempSync(join(tmpdir(), 'shadow-active-')), path = join(directory, 'biometrics.db') + const writer = new Database(path) + writer.pragma('journal_mode = WAL') + migrate(drizzle(writer), { migrationsFolder: fileURLToPath(new URL('../../../db/biometrics-migrations/', import.meta.url)) }) + const readonly = new Database(path, { readonly: true, fileMustExist: true }) + cleanups.push(() => { + readonly.close() + writer.close() + rmSync(directory, { recursive: true, force: true }) + }) + const runId = randomUUID() + const bundle = { schemaVersion: 1, manifestHash, resolved: defaultConfig, generation: 0, revisionId: null, + components: biometricManifest.components, sourceHashes: biometricManifest.sourceHashes } + writer.prepare('INSERT INTO biometric_processing_runs (id, side, kind, bundle, config_hash, calibration_hash, input_identity, started_at_ms) VALUES (?, ?, ?, ?, ?, NULL, ?, ?)') + .run(runId, 'left', 'live', JSON.stringify(bundle), hash, JSON.stringify({ sourceSide: 'right', timestampBasis: 'processing_clock', rawIdentity: 'not_retained', sensorCalibration: 'not_consumed' }), start) + const options = { side: 'left', startMs: start, endMs: start + 10000, activeConfigHash: hash, maximumRows } + let now = start + const reader = () => new ShadowActiveReader(readonly, options, () => now) + const insert = (seconds = 0, side = 'left', provenance: string | null = runId) => { + const id = Number(writer.prepare('INSERT INTO vitals (side,timestamp,heart_rate,hrv,breathing_rate,processing_run_id) VALUES (?,?,60,NULL,12,?)').run(side, start / 1000 + seconds, provenance).lastInsertRowid) + writer.prepare('INSERT INTO vitals_quality (vitals_id,side,timestamp,quality_score,created_at) VALUES (?,?,?,0.9,?)').run(id, side, start / 1000 + seconds, start / 1000 + seconds) + return id + } + return { writer, readonly, path, runId, bundle, options, reader, insert, advance: (milliseconds: number) => { + now += milliseconds + } } +} +it('captures only newly inserted rows with original values, quality and verified live provenance', () => { + const { reader, insert, advance, runId } = setup() + insert() + const capture = reader() + expect(capture.poll()).toEqual([]) + advance(1000) + insert(1) + const rows = capture.poll() + expect(rows).toHaveLength(1) + expect(rows[0]).toMatchObject({ timestampMs: start + 1000, configHash: hash, inputHash: null, + provenance: { processingRunId: runId, sourceSide: 'right', generation: 0, manifestHash }, + readings: [{ metric: 'heart_rate', value: 60, quality: 0.9 }, { metric: 'breathing_rate', value: 12 }, { metric: 'window_ibi_variability_v1', value: null, windowMs: 300000 }] }) + expect(capture.poll()).toEqual([]) +}) +it('requires read-only access and never writes through the capture connection', () => { + const { writer, readonly, options, reader, insert } = setup() + expect(() => new ShadowActiveReader(writer, options)).toThrow(/read-only/) + const capture = reader() + insert() + expect(capture.poll()).toHaveLength(1) + expect(() => readonly.prepare('DELETE FROM vitals').run()).toThrow(/readonly/) +}) +it('advances past other-side and out-of-window inserts without labelling them active', () => { + const { reader, insert } = setup() + const capture = reader() + insert(0, 'right') + insert(-1) + expect(capture.poll()).toEqual([]) + insert() + expect(capture.poll()).toHaveLength(1) +}) +it('refuses a batch above the poll quota without silently taking a prefix', () => { + const { reader, insert, advance } = setup(1) + const capture = reader() + insert() + advance(1000) + insert(1) + expect(() => capture.poll()).toThrow(/quota/) + expect(() => capture.poll()).toThrow(/quota/) +}) +it.each(['legacy', 'quality_missing', 'quality_duplicate', 'quality_side', 'quality_timestamp', 'quality_invalid', 'config', 'bundle', 'kind', 'source', 'future'])('fails closed for %s evidence', (mode) => { + const { reader, insert, writer, runId } = setup() + const capture = reader() + const id = insert(mode === 'future' ? 1 : 0, 'left', mode === 'legacy' ? null : runId) + if (mode === 'quality_missing') writer.prepare('DELETE FROM vitals_quality').run() + if (mode === 'quality_duplicate') writer.prepare('INSERT INTO vitals_quality (vitals_id,side,timestamp,quality_score,created_at) SELECT vitals_id,side,timestamp,quality_score,created_at FROM vitals_quality').run() + if (mode === 'quality_side') writer.prepare('UPDATE vitals_quality SET side=\'right\' WHERE vitals_id=?').run(id) + if (mode === 'quality_timestamp') writer.prepare('UPDATE vitals_quality SET timestamp=timestamp+1').run() + if (mode === 'quality_invalid') writer.prepare('UPDATE vitals_quality SET quality_score=2').run() + if (mode === 'config') writer.prepare('UPDATE biometric_processing_runs SET config_hash=?').run('b'.repeat(64)) + if (mode === 'bundle') writer.prepare('UPDATE biometric_processing_runs SET bundle=?').run('x'.repeat(65537)) + if (mode === 'kind') writer.prepare('UPDATE biometric_processing_runs SET kind=\'replay\'').run() + if (mode === 'source') writer.prepare('UPDATE biometric_processing_runs SET input_identity=?').run('{}') + expect(() => capture.poll()).toThrow() + expect(() => capture.poll()).toThrow() +}) +it('detects deletion or replacement of its cursor instead of resuming through a reset', () => { + const { reader, insert, writer } = setup() + insert() + const capture = reader() + writer.prepare('UPDATE vitals SET heart_rate=99').run() + expect(() => capture.poll()).toThrow(/cursor changed/) + writer.prepare('DELETE FROM vitals').run() + expect(() => capture.poll()).toThrow(/cursor changed/) +}) +it('retains active run provenance when the stream join persists an active-only point', () => { + const { reader, insert, writer, runId } = setup() + const capture = reader() + const store = new ShadowStore(writer, { maximumRuns: 10, maximumPoints: 10, maximumBytes: 100000 }, () => start) + const run = store.create({ schema: 'biometric-shadow-plan-v1', subjectId: 'test', side: 'left', candidateConfigHash: hash, activeConfigHash: hash, + qualificationHash: hash, engineBuildHash: hash, startMs: start, endMs: start + 1000, maximumPoints: 10, maximumBytes: 10000, metrics: ['heart_rate'], + joining: { maximumPendingPoints: 10, maximumLatenessMs: 1, drainGraceMs: 100 }, monitoring: { expectedIntervalMs: 1000, maximumGapMs: 1000, heartbeatStaleMs: 1000, + minimumCandidateCoverage: 0.8, maximumCoverageLoss: 0.1, minimumQuality: 0.8, alignment: { version: 'reference-alignment-v1', pointToleranceMs: 0, clockOffsetMs: 0, offsetSource: 'fixed' } } }, 'test') + const lease = store.claim(run.id, 1000, () => {}) + const stream = new ShadowStreamJoin(store, lease) + insert() + stream.active(capture.poll()[0]) + stream.advance(start + 1) + expect(store.snapshot(run.id, { maximumPoints: 10, maximumBytes: 10000 }).samples[0]).toMatchObject({ + inputHash: null, inputIdentityKind: 'unavailable', activeProvenance: { processingRunId: runId, sourceSide: 'right' }, pairing: { activePresent: true, candidatePresent: false } }) +}) +it.each(['manifestHash', 'components', 'sourceHashes', 'resolved'])('rejects substituted %s in a live run bundle', (field) => { + const { reader, insert, writer, bundle } = setup() + const capture = reader() + insert() + writer.prepare('UPDATE biometric_processing_runs SET bundle=?').run(JSON.stringify({ ...bundle, [field]: field === 'manifestHash' ? 'a'.repeat(64) : {} })) + expect(() => capture.poll()).toThrow() + writer.prepare('UPDATE biometric_processing_runs SET bundle=?').run(JSON.stringify(bundle)) + // Failed validation did not advance the cursor or hide this observation. + expect(capture.poll()).toHaveLength(1) +}) +function liveSetup() { + vi.useFakeTimers() + vi.setSystemTime(start) + const fixture = setup() + const controller = new AbortController(), active = vi.fn(), fail = vi.fn(), databaseCurrent = vi.fn() + const options = { db: fixture.readonly, side: 'left' as const, startMs: start, endMs: start + 10000, activeConfigHash: hash, + maximumActiveRows: 10, maximumLatenessMs: 100, pollMs: 50, + frameBudgets: { maximumFrames: 2, maximumBytes: 10000, maximumEventAgeMs: 1000, maximumFutureSkewMs: 0 }, + context: () => ({ activeSides: ['left'], leftOccupied: true, rightOccupied: false, pumpContextKnown: false }), + assertDatabaseCurrent: databaseCurrent, signal: controller.signal, active, fail } + const open = () => { + const source = openShadowLiveSources(options) + // Close subscribers before the owning database connections in afterEach. + cleanups.unshift(source.close) + return source + } + return { ...fixture, controller, active, fail, databaseCurrent, options, open } +} +it('composes real waveform capture and read-only active polling with an explicit final flush', async () => { + const { insert, open, active, fail } = liveSetup() + insert() + const source = open(), frames = source.frames[Symbol.asyncIterator]() + const frame = frames.next() + publishProspectiveFrame({ type: 'piezo-dual', ts: start / 1000, freq: 500, left1: [1, 2], right1: [3, 4] }, false, start) + expect((await frame).value).toMatchObject({ observedAtMs: start, context: { activeSides: ['left'] } }) + expect(active).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(1000) + insert(1) + source.flush?.() + expect(active).toHaveBeenCalledOnce() + expect(active.mock.calls[0][0]).toMatchObject({ timestampMs: start + 1000 }) + await vi.advanceTimersByTimeAsync(50) + expect(active).toHaveBeenCalledOnce() + expect(fail).not.toHaveBeenCalled() +}) +it('keeps active polling available during final drain after waveform collection ends', async () => { + const { insert, open, active, fail } = liveSetup() + const source = open(), frames = source.frames[Symbol.asyncIterator]() + const done = frames.next() + await vi.advanceTimersByTimeAsync(10000) + expect((await done).done).toBe(true) + insert(9) + source.flush?.() + expect(active.mock.calls[0][0]).toMatchObject({ timestampMs: start + 9000 }) + expect(fail).not.toHaveBeenCalled() +}) +it('closes both captures on a database identity failure without delivering old-connection data', async () => { + const { insert, open, active, fail, databaseCurrent } = liveSetup() + const source = open(), frames = source.frames[Symbol.asyncIterator]() + const pending = frames.next() + insert() + databaseCurrent.mockImplementation(() => { + throw new Error('database inode changed') + }) + await vi.advanceTimersByTimeAsync(50) + expect((await pending).done).toBe(true) + expect(active).not.toHaveBeenCalled() + expect(fail).toHaveBeenCalledOnce() + expect(vi.getTimerCount()).toBe(0) +}) +it('clears polling and subscriptions on abort and refuses asynchronous delivery', async () => { + const fixture = liveSetup() + const source = fixture.open() + fixture.controller.abort() + expect(vi.getTimerCount()).toBe(0) + expect(() => source.flush?.()).toThrow(/closed/) + expect(fixture.fail).not.toHaveBeenCalled() + const next = liveSetup() + next.active.mockImplementation(async () => {}) + next.open() + next.insert() + await vi.advanceTimersByTimeAsync(50) + expect(next.fail).toHaveBeenCalledOnce() + expect(vi.getTimerCount()).toBe(0) +}) +it('opens a bounded read-only active connection and detects path replacement', () => { + const { path } = setup() + const connection = openShadowActiveDatabase(path, { busyTimeoutMs: 50 }) + cleanups.unshift(connection.close) + expect(connection.db.readonly).toBe(true) + expect(connection.db.pragma('busy_timeout', { simple: true })).toBe(50) + connection.assertCurrent() + renameSync(path, path + '.old') + copyFileSync(path + '.old', path) + expect(() => connection.assertCurrent()).toThrow(/replaced/) + connection.close() + expect(() => connection.assertCurrent()).toThrow(/closed/) +}) +it('refuses missing files, symlinks and unbounded busy timeouts', () => { + const { path } = setup() + expect(() => openShadowActiveDatabase(path + '.missing', { busyTimeoutMs: 50 })).toThrow() + symlinkSync(path, path + '.link') + expect(() => openShadowActiveDatabase(path + '.link', { busyTimeoutMs: 50 })).toThrow(/regular file/) + expect(() => openShadowActiveDatabase(path, { busyTimeoutMs: 1001 })).toThrow() +}) +it('rechecks database identity after the snapshot and before active delivery', async () => { + const { insert, open, active, fail, databaseCurrent } = liveSetup() + databaseCurrent.mockImplementationOnce(() => {}).mockImplementationOnce(() => {}).mockImplementation(() => { + throw new Error('database replaced during snapshot') + }) + open() + insert() + await vi.advanceTimersByTimeAsync(50) + expect(active).not.toHaveBeenCalled() + expect(fail).toHaveBeenCalledOnce() +}) +function plannedSetup() { + vi.useFakeTimers() + vi.setSystemTime(start) + const fixture = setup() + const configPath = join(dirname(fixture.path), 'configuration.db'), presencePath = join(dirname(fixture.path), 'presence.json') + const config = new Database(configPath) + config.exec('CREATE TABLE side_settings(side TEXT, away_mode INTEGER); INSERT INTO side_settings VALUES(\'left\',0),(\'right\',1); CREATE TABLE device_settings(id INTEGER, bed_mode TEXT); INSERT INTO device_settings VALUES(1,\'two\');') + cleanups.unshift(() => config.close()) + writeFileSync(presencePath, JSON.stringify({ version: 1, left: { vitals_presence: true, vitals_evidence_ts: start / 1000 } })) + utimesSync(presencePath, start / 1000, start / 1000) + const plan = shadowPlanSchema.parse({ schema: 'biometric-shadow-plan-v1', subjectId: 'test', side: 'left', startMs: start, endMs: start + 10000, + candidateConfigHash: hash, activeConfigHash: hash, qualificationHash: hash, engineBuildHash: hash, maximumPoints: 100, maximumBytes: 100000, metrics: ['heart_rate'], + joining: { maximumPendingPoints: 10, maximumLatenessMs: 100, drainGraceMs: 500 }, + monitoring: { expectedIntervalMs: 1000, maximumGapMs: 1000, heartbeatStaleMs: 1000, minimumCandidateCoverage: 0.8, maximumCoverageLoss: 0.1, minimumQuality: 0.8, + alignment: { version: 'reference-alignment-v1', pointToleranceMs: 0, clockOffsetMs: 0, offsetSource: 'fixed' } }, + execution: { version: 'shadow-execution-policy-v1', configHashes: { left: hash, right: hash }, + maximumRecords: 10, maximumOutputs: 10, maximumCpuMs: 1000, maximumInputBytes: 100000, maximumOutputBytes: 100000, + budgets: { cpuSeconds: 2, addressSpaceBytes: 100000000, maximumRssBytes: 100000000, wallMs: 15000, acknowledgmentMs: 1000, idleMs: 1000, rssIntervalMs: 100, rssTimeoutMs: 100 }, + leaseMs: 1500, heartbeatMs: 200, sources: { pollMs: 50, maximumActiveRows: 10, databaseBusyTimeoutMs: 50, + frames: { maximumFrames: 1, maximumBytes: 10000, maximumEventAgeMs: 1000, maximumFutureSkewMs: 0 }, + context: { presenceStaleMs: 300000, pumpStaleMs: 1000, maximumPresenceBytes: 4096 } } } }) + const paths = { biometricsDatabase: fixture.path, configurationDatabase: configPath, presenceFile: presencePath } + const abort = new AbortController(), active = vi.fn(), fail = vi.fn() + const execution = plan.execution + if (!execution) throw new Error('Fixture execution missing') + const input = { execution, collectionEndsAtMs: plan.endMs, signal: abort.signal, active, fail } + const factory = plannedShadowSourceFactory(plan, paths) + const open = () => { + const source = factory(input) + cleanups.unshift(source.close) + return source + } + return { ...fixture, paths, plan, input, factory, open, abort, active, fail } +} +it('applies the saved context and capture policy to real connections and subscriptions', async () => { + const { open, insert, active, fail } = plannedSetup() + const source = open(), frames = source.frames[Symbol.asyncIterator]() + const next = frames.next() + publishProspectiveFrame({ type: 'piezo-dual', ts: start / 1000, freq: 500, left1: [1], right1: [2] }, false, start) + expect((await next).value.context).toEqual({ activeSides: ['left'], leftOccupied: true, rightOccupied: null, pumpContextKnown: false }) + insert() + source.flush?.() + expect(active).toHaveBeenCalledOnce() + expect(fail).not.toHaveBeenCalled() +}) +it('refuses changed source policies before opening resources', () => { + const { factory, input } = plannedSetup() + const changed = structuredClone(input.execution) + changed.sources.frames.maximumFrames++ + expect(() => factory({ ...input, execution: changed })).toThrow(/saved plan/) + expect(vi.getTimerCount()).toBe(0) +}) +it('closes both owned read-only handles on abort exactly once', () => { + const { open, abort } = plannedSetup() + const close = vi.spyOn(Database.prototype, 'close') + const source = open() + abort.abort() + expect(close).toHaveBeenCalledTimes(2) + source.close() + expect(close).toHaveBeenCalledTimes(2) + expect(vi.getTimerCount()).toBe(0) + close.mockRestore() +}) +it('unwinds the first connection if the second database cannot open', () => { + const { plan, paths, input } = plannedSetup() + const factory = plannedShadowSourceFactory(plan, { ...paths, configurationDatabase: paths.configurationDatabase + '.missing' }) + const close = vi.spyOn(Database.prototype, 'close') + expect(() => factory(input)).toThrow() + expect(close).toHaveBeenCalledOnce() + expect(vi.getTimerCount()).toBe(0) + close.mockRestore() +}) +it('enforces the saved frame count and releases owned resources on overflow', () => { + const { open, fail } = plannedSetup() + open() + const frame = { type: 'piezo-dual', ts: start / 1000, freq: 500, left1: [1], right1: [2] } + const close = vi.spyOn(Database.prototype, 'close') + publishProspectiveFrame(frame, false, start) + publishProspectiveFrame(frame, false, start) + expect(fail).toHaveBeenCalledOnce() + expect(close).toHaveBeenCalledTimes(2) + expect(vi.getTimerCount()).toBe(0) + close.mockRestore() +}) + +it('opens owned shadow state writable while keeping active capture read-only and detecting replacement', () => { + const { path, readonly } = setup() + const state = openShadowStateDatabase(path, { busyTimeoutMs: 50 }) + try { + expect(state.db.readonly).toBe(false) + state.db.exec('CREATE TABLE shadow_connection_test (value INTEGER)') + state.db.prepare('INSERT INTO shadow_connection_test VALUES (1)').run() + expect(() => readonly.prepare('INSERT INTO shadow_connection_test VALUES (2)').run()).toThrow(/readonly/) + state.assertCurrent() + renameSync(path, path + '.moved') + expect(state.assertCurrent).toThrow() + } + finally { state.close() } +}) diff --git a/src/biometrics/shadow/tests/admission.test.ts b/src/biometrics/shadow/tests/admission.test.ts new file mode 100644 index 000000000..7ffe609ca --- /dev/null +++ b/src/biometrics/shadow/tests/admission.test.ts @@ -0,0 +1,94 @@ +// @vitest-environment node +import { mkdtempSync, rmSync, writeFileSync, symlinkSync } from 'node:fs' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { afterEach, expect, it } from 'vitest' +import { dataHash } from '../../algorithms/canonical' +import { manifestHash } from '../../algorithms/registry' +import { installedShadowAdmitter, readInstalledShadowQualifications } from '../admission' +const a = 'a'.repeat(64), b = 'b'.repeat(64), now = 1700000000000 +const directories: string[] = [] +afterEach(() => directories.splice(0).forEach(path => rmSync(path, { recursive: true, force: true }))) +function setup() { + const directory = mkdtempSync(join(tmpdir(), 'shadow-admission-')) + directories.push(directory) + const workerPath = join(directory, 'worker.cjs'), file = join(directory, 'shadow-qualifications.json') + const execution = { version: 'shadow-execution-policy-v1', configHashes: { left: a, right: a }, + maximumRecords: 10, maximumOutputs: 10, maximumCpuMs: 1000, maximumInputBytes: 100000, maximumOutputBytes: 100000, + budgets: { cpuSeconds: 2, addressSpaceBytes: 100000000, maximumRssBytes: 100000000, wallMs: 10000, acknowledgmentMs: 1000, idleMs: 1000, rssIntervalMs: 100, rssTimeoutMs: 100 }, + leaseMs: 1500, heartbeatMs: 200, sources: { pollMs: 50, maximumActiveRows: 10, databaseBusyTimeoutMs: 50, + frames: { maximumFrames: 10, maximumBytes: 100000, maximumEventAgeMs: 1000, maximumFutureSkewMs: 0 }, + context: { presenceStaleMs: 300000, pumpStaleMs: 5000, maximumPresenceBytes: 4096 } } } + const entry = { qualificationId: 'reviewed-test-only', manifestHash, engineBuildHash: a, pythonBuildHash: b, hardwareId: 'fixture-hardware', sampleRateHz: 500, + activeConfigHash: a, executionPolicyHash: dataHash(execution), fixtureEvidenceHash: a, podEvidenceHash: b, heldOutEvidenceHash: a, + contextEvidence: 'host_observations_not_live_processor_cache_attestation', maximumRuntimeHeartbeatAgeMs: 1000, maximumHardwareObservationAgeMs: 1000, + healthPolicy: { qualificationId: 'reviewed-test-only', minimumAvailableMemoryBytes: 100, minimumFreeDiskBytes: 100, + maximumLoadPerCpu: 1, maximumEventLoopDelayMs: 100, maximumPiezoAgeMs: 1000, maximumTemperatureC: 70, maximumSampleAgeMs: 1000 } } + const plan = { schema: 'biometric-shadow-plan-v1', side: 'left', subjectId: 'test', execution, startMs: now, endMs: now + 10000, + candidateConfigHash: a, activeConfigHash: a, engineBuildHash: a, qualificationHash: dataHash(entry), maximumPoints: 10, maximumBytes: 10000, metrics: ['heart_rate'], + joining: { maximumPendingPoints: 10, maximumLatenessMs: 100, drainGraceMs: 500 }, + monitoring: { expectedIntervalMs: 1000, maximumGapMs: 2000, heartbeatStaleMs: 1000, minimumCandidateCoverage: 0.8, maximumCoverageLoss: 0.1, minimumQuality: 0.8, + alignment: { version: 'reference-alignment-v1', pointToleranceMs: 0, clockOffsetMs: 0, offsetSource: 'fixed' } } } + const identity = { engineBuildHash: a, pythonBuildHash: b, hardwareId: 'fixture-hardware', sampleRateHz: 500 } + const binding = { side: 'left', generation: 3, resolvedHash: a, appliedHash: a, appliedGeneration: 3, state: 'applied', heartbeatAtMs: now } + const health = { measuredAtMs: now, availableMemoryBytes: 1000, freeDiskBytes: 1000, loadPerCpu: 0.1, eventLoopDelayMs: 1, piezoReceivedAtMs: now, temperatureC: 40 } + const options = { workerPath, expectedBinding: { side: 'left' as const, generation: 3, resolvedHash: a }, + identity: () => identity, binding: () => binding, health: () => health, now: () => now } + const install = (entries: unknown[] = [entry]) => writeFileSync(file, JSON.stringify({ version: 'biometric-shadow-qualifications-v1', entries })) + install() + return { entry, plan, options, identity, binding, health, install, file, workerPath, admit: installedShadowAdmitter(options) } +} +it('admits only an exact installed qualification with applied current binding and fresh health', () => { + const { admit, plan, binding } = setup() + expect(admit(plan)).toBeUndefined() + binding.generation++ + binding.appliedGeneration++ + expect(() => admit(plan)).toThrow(/binding changed/) +}) +it.each(['engineBuildHash', 'pythonBuildHash', 'hardwareId', 'sampleRateHz'] as const)('refuses changed installed %s', (field) => { + const fixture = setup() + Object.assign(fixture.identity, { [field]: field === 'sampleRateHz' ? 250 : field === 'pythonBuildHash' ? a : b }) + expect(() => fixture.admit(fixture.plan)).toThrow() +}) +it.each(['appliedHash', 'appliedGeneration', 'state', 'heartbeatAtMs', 'side'])('refuses invalid runtime %s', (field) => { + const fixture = setup() + Object.assign(fixture.binding, { [field]: field === 'appliedGeneration' ? 2 : field === 'heartbeatAtMs' ? now - 1001 : field === 'state' ? 'pending_boundary' : field === 'side' ? 'right' : b }) + expect(() => fixture.admit(fixture.plan)).toThrow() +}) +it.each(['availableMemoryBytes', 'freeDiskBytes', 'measuredAtMs', 'piezoReceivedAtMs', 'temperatureC', 'loadPerCpu'])('refuses unhealthy or stale %s', (field) => { + const fixture = setup() + Object.assign(fixture.health, { [field]: field.endsWith('Ms') ? now - 1001 : field.endsWith('Bytes') ? 0 : 100 }) + expect(() => fixture.admit(fixture.plan)).toThrow(/admission refused/) +}) +it('binds all resource limits and refuses substitution or duplicate qualification entries', () => { + const { admit, plan, entry, install } = setup() + plan.execution.budgets.maximumRssBytes++ + expect(() => admit(plan)).toThrow(/installed execution/) + plan.execution.budgets.maximumRssBytes-- + install([entry, entry]) + expect(() => admit(plan)).toThrow(/ambiguous/) +}) +it('reads revocation on subsequent admission instead of caching authorization', () => { + const { admit, plan, install } = setup() + admit(plan) + install([]) + expect(() => admit(plan)).toThrow(/missing/) +}) +it.each(['missing', 'oversize', 'symlink', 'malformed'])('rejects a %s installed file', (mode) => { + const { file, workerPath } = setup() + if (mode === 'missing') rmSync(file) + if (mode === 'oversize') writeFileSync(file, 'x'.repeat(256 * 1024 + 1)) + if (mode === 'malformed') writeFileSync(file, '{}') + if (mode === 'symlink') { + rmSync(file) + writeFileSync(file + '.target', '{}') + symlinkSync(file + '.target', file) + } + expect(() => readInstalledShadowQualifications(workerPath)).toThrow(/unavailable/) +}) +it('rejects asynchronous host observations without leaking rejected promises', async () => { + const { options, plan } = setup() + const admit = installedShadowAdmitter({ ...options, identity: () => Promise.reject(new Error('collector failed')) }) + expect(() => admit(plan)).toThrow(/synchronous/) + await Promise.resolve() +}) diff --git a/src/biometrics/shadow/tests/attempt.test.ts b/src/biometrics/shadow/tests/attempt.test.ts new file mode 100644 index 000000000..cb3e15e28 --- /dev/null +++ b/src/biometrics/shadow/tests/attempt.test.ts @@ -0,0 +1,474 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { fileURLToPath } from 'node:url' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { defaultConfig, resolveConfig } from '../../algorithms/registry' +import { configHash } from '../../algorithms/canonical' +import { reportShadowRun } from '../report' +import { shadowReportSchema } from '../report-contracts' +import { type ShadowPartialReceipt, validateShadowPartialReceipt, validateShadowWorkerReceipt } from '../receipt' +import { ShadowStore } from '../store' +import { runShadowAttempt, type ShadowAttemptHost } from '../attempt' + +const worker = vi.hoisted(() => ({ calls: 0, frames: 0, cleaned: false, fail: false, corruptReceipt: false, cleanupUnconfirmed: false })) +vi.mock('../worker-child', () => ({ runShadowChild: async (_location: unknown, init: { side: string, configHashes: Record }, frames: AsyncIterable<{ observedAtMs: number }>, output: (row: unknown) => void, options: { budgets: ShadowPartialReceipt['budgets'], onResources?: (value: ShadowPartialReceipt) => void }) => { + worker.calls++ + let result + try { + let sequence = 0 + for await (const frame of frames) { + worker.frames++ + if (worker.fail) throw new Error('processor_failed') + output({ side: init.side, sourceSide: init.side, timestampMs: frame.observedAtMs, configHash: init.configHashes[init.side], sequence: sequence++, + inputHash: 'a'.repeat(64), flags: null, limitations: [], readings: [ + { metric: 'heart_rate', unit: 'bpm', windowMs: null, value: 60, quality: 1 }, + { metric: 'breathing_rate', unit: 'breaths/min', windowMs: null, value: 12, quality: 1 }, + { metric: 'window_ibi_variability_v1', unit: 'ms', windowMs: 300000, value: null, quality: 0 }, + ] }) + } + result = { health: { state: 'completed', reason: 'window_complete', records: sequence, outputs: sequence, cpu_ms: 1, + input_hash: 'a'.repeat(64), counts: { records: sequence, accepted_frames: sequence, empty_channel_frames: 0, + pump_gated_frames: 0, preroll_outputs: 0, scored_outputs: sequence }, limitations: [] }, + inputBytes: worker.corruptReceipt ? 100001 : 100, outputBytes: 100, peakObservedRssBytes: 1000, rssSamples: 2, maximumObservationGapMs: 100, + budgets: options.budgets, memoryScope: 'sum_rss_of_shadow_process_group', exactKernelPeak: false } + } + catch (error) { + if (!worker.cleanupUnconfirmed) throw error + } + finally { + worker.cleaned = true + options.onResources?.({ kind: 'partial_supervisor_observation', health: null, inputBytes: 100, outputBytes: 100, diagnosticBytes: 0, + peakObservedRssBytes: 1000, rssSamples: 2, maximumObservationGapMs: 100, budgets: options.budgets, + cleanupConfirmed: !worker.cleanupUnconfirmed, memoryScope: 'sum_rss_of_shadow_process_group', exactKernelPeak: false }) + } + if (worker.cleanupUnconfirmed) throw Object.assign(new Error('shadow_worker_cleanup_unconfirmed'), { code: 'SHADOW_CLEANUP_UNCONFIRMED' }) + return result +} })) +const start = 1700000000000, hash = configHash(defaultConfig) +const databases: Database.Database[] = [] +beforeEach(() => { + vi.useFakeTimers() + vi.setSystemTime(start) + worker.calls = 0 + worker.frames = 0 + worker.cleaned = false + worker.fail = false + worker.corruptReceipt = false + worker.cleanupUnconfirmed = false +}) +afterEach(() => { + vi.useRealTimers() + databases.splice(0).forEach(db => db.close()) +}) +function setup(maximumPoints = 10, duration = 2000) { + const db = new Database(':memory:') + databases.push(db) + migrate(drizzle(db), { migrationsFolder: fileURLToPath(new URL('../../../db/biometrics-migrations/', import.meta.url)) }) + const store = new ShadowStore(db, { maximumRuns: 10, maximumPoints: 100, maximumBytes: 100000 }) + const execution = { version: 'shadow-execution-policy-v1', configHashes: { left: hash, right: hash }, + maximumRecords: 10, maximumOutputs: 10, maximumCpuMs: 1000, maximumInputBytes: 100000, maximumOutputBytes: 100000, + budgets: { cpuSeconds: 2, addressSpaceBytes: 100000000, maximumRssBytes: 100000000, wallMs: 10000, acknowledgmentMs: 1000, idleMs: 1000, rssIntervalMs: 100, rssTimeoutMs: 100 }, + leaseMs: 1500, heartbeatMs: 200, + sources: { pollMs: 50, maximumActiveRows: 10, databaseBusyTimeoutMs: 50, + frames: { maximumFrames: 10, maximumBytes: 100000, maximumEventAgeMs: 1000, maximumFutureSkewMs: 0 }, + context: { presenceStaleMs: 300000, pumpStaleMs: 5000, maximumPresenceBytes: 4096 } } } + const run = store.create({ execution, schema: 'biometric-shadow-plan-v1', subjectId: 'test', side: 'left', startMs: start, endMs: start + duration, + candidateConfigHash: hash, activeConfigHash: hash, qualificationHash: 'c'.repeat(64), engineBuildHash: 'd'.repeat(64), + maximumPoints, maximumBytes: 10000, metrics: ['heart_rate'], joining: { maximumPendingPoints: 10, maximumLatenessMs: 100, drainGraceMs: 500 }, + monitoring: { expectedIntervalMs: 1000, maximumGapMs: 2000, heartbeatStaleMs: 1000, minimumCandidateCoverage: 0.8, maximumCoverageLoss: 0.1, minimumQuality: 0.8, + alignment: { version: 'reference-alignment-v1', pointToleranceMs: 0, clockOffsetMs: 0, offsetSource: 'fixed' } } }, 'test') + let active: (reading: unknown) => void = () => { + throw new Error('Sources not open') + } + const close = vi.fn() + const host: ShadowAttemptHost = { + location: { pythonPath: '/trusted/python', modulesPath: '/trusted/modules' }, configs: { left: defaultConfig, right: defaultConfig }, + admit: vi.fn(), + openSources: (input) => { + active = input.active + let sent = false + return { close, frames: { [Symbol.asyncIterator]: () => ({ next: () => { + if (sent) return new Promise>(() => {}) + sent = true + return Promise.resolve({ done: false, value: { type: 'frame', observedAtMs: start } }) + } }) } } + }, + } + return { db, store, run, host, close, active: (timestampMs = start) => active({ side: 'left', timestampMs, configHash: hash, + readings: [{ metric: 'heart_rate', unit: 'bpm', windowMs: null, value: 61, quality: 1 }, + { metric: 'breathing_rate', unit: 'breaths/min', windowMs: null, value: 12, quality: 1 }, + { metric: 'window_ibi_variability_v1', unit: 'ms', windowMs: 300000, value: null, quality: 0 }] }) } +} +describe('leased shadow attempt lifecycle', () => { + it('retains both streams, renews the lease and completes only after final drain', async () => { + const { db, store, run, host, close, active } = setup() + for (const operation of ['INSERT', 'UPDATE', 'DELETE']) db.exec(`CREATE TRIGGER deny_${operation} BEFORE ${operation} ON vitals BEGIN SELECT RAISE(ABORT, 'canonical write forbidden'); END`) + const task = runShadowAttempt(store, run.id, host) + await vi.advanceTimersByTimeAsync(0) + active() + await vi.advanceTimersByTimeAsync(2000) + expect(store.get(run.id).state).toBe('running') + expect(worker.cleaned).toBe(true) + active(start + 1999) + await vi.advanceTimersByTimeAsync(101) + expect((await task).run).toMatchObject({ state: 'completed', point_count: 2, reason: 'observation_window_complete' }) + const sample = store.snapshot(run.id, { maximumPoints: 10, maximumBytes: 10000 }).samples[0] + expect(sample).toMatchObject({ readings: [{ active: { value: 61 }, candidate: { value: 60 } }] }) + expect(close).toHaveBeenCalledOnce() + expect(vi.mocked(host.admit).mock.calls.length).toBeGreaterThan(5) + }) + it('rejects asynchronous admission without claiming or opening sources', async () => { + const { store, run, host } = setup() + host.admit = async () => {} + await expect(runShadowAttempt(store, run.id, host)).rejects.toThrow(/synchronously/) + expect(store.get(run.id).state).toBe('pending') + expect(worker.calls).toBe(0) + }) + it('cancels, waits for worker cleanup, and ignores late source callbacks', async () => { + const { store, run, host, close, active } = setup() + const abort = new AbortController() + const task = runShadowAttempt(store, run.id, host, abort.signal) + const rejected = expect(task).rejects.toThrow('shadow_cancelled') + await vi.advanceTimersByTimeAsync(0) + abort.abort() + await rejected + expect(worker.cleaned).toBe(true) + expect(close).toHaveBeenCalledOnce() + expect(store.get(run.id)).toMatchObject({ state: 'stopped', reason: 'shadow_cancelled', point_count: 0 }) + active() + expect(store.get(run.id).point_count).toBe(0) + }) + it('rechecks binding/resource admission during observation', async () => { + const { store, run, host } = setup() + const task = runShadowAttempt(store, run.id, host) + const rejected = expect(task).rejects.toThrow('shadow_admission_or_lease_lost') + await vi.advanceTimersByTimeAsync(0) + host.admit = () => { + throw new Error('active binding changed') + } + await vi.advanceTimersByTimeAsync(200) + await rejected + expect(store.get(run.id)).toMatchObject({ state: 'failed', reason: 'shadow_admission_or_lease_lost' }) + expect(worker.cleaned).toBe(true) + }) + it('never finishes or writes over a replacement attempt', async () => { + const { store, run, host } = setup(10, 5000) + const task = runShadowAttempt(store, run.id, host) + const rejected = expect(task).rejects.toThrow(/lease_lost/) + await vi.advanceTimersByTimeAsync(0) + vi.setSystemTime(start + 1600) + const replacement = store.claim(run.id, 1500, () => {}) + await vi.advanceTimersByTimeAsync(200) + await rejected + expect(store.get(run.id)).toMatchObject({ state: 'running', attempt_id: replacement.attemptId, point_count: 0 }) + }) + it('preserves storage-quota termination instead of marking completion', async () => { + const { store, run, host, active } = setup(1) + const task = runShadowAttempt(store, run.id, host) + const rejected = expect(task).rejects.toThrow('storage_quota') + await vi.advanceTimersByTimeAsync(1) + active(start + 1) + await vi.advanceTimersByTimeAsync(200) + await rejected + expect(store.get(run.id)).toMatchObject({ state: 'stopped', reason: 'storage_quota', point_count: 1 }) + }) + it('forwards an in-window frame that was dequeued as the wall clock reached collection end', async () => { + const { store, run, host, close } = setup() + const last = new Promise>(resolve => setTimeout(() => resolve({ done: false, value: { type: 'frame', observedAtMs: start + 1999 } }), 2000)) + host.openSources = () => { + let pulls = 0 + return { close, frames: { [Symbol.asyncIterator]: () => ({ next: () => { + pulls++ + if (pulls === 1) return Promise.resolve({ done: false, value: { type: 'frame', observedAtMs: start } }) + if (pulls === 2) return last + return new Promise>(() => {}) + } }) } } + } + const task = runShadowAttempt(store, run.id, host) + await vi.advanceTimersByTimeAsync(0) + await vi.advanceTimersByTimeAsync(2000) + await vi.advanceTimersByTimeAsync(101) + expect((await task).run).toMatchObject({ state: 'completed', reason: 'observation_window_complete' }) + expect(worker.frames).toBe(2) + }) + it('fails when the source ends before the prospective window', async () => { + const { store, run, host, close } = setup() + host.openSources = () => ({ close, frames: { async* [Symbol.asyncIterator]() {} } }) + await expect(runShadowAttempt(store, run.id, host)).rejects.toThrow('shadow_source_ended_early') + expect(store.get(run.id).state).toBe('failed') + expect(close).toHaveBeenCalledOnce() + }) + it('retains a processor failure and cleans up both sources', async () => { + const { store, run, host, close } = setup() + worker.fail = true + await expect(runShadowAttempt(store, run.id, host)).rejects.toThrow('processor_failed') + expect(store.get(run.id)).toMatchObject({ state: 'failed', reason: 'processor_failed' }) + expect(close).toHaveBeenCalledOnce() + }) +}) +it('records an expired attempt as failed only after worker cleanup', async () => { + const { store, run, host } = setup(10, 5000) + const task = runShadowAttempt(store, run.id, host) + const rejected = expect(task).rejects.toThrow(/lease_lost/) + await vi.advanceTimersByTimeAsync(0) + vi.setSystemTime(start + 1600) + await vi.advanceTimersByTimeAsync(200) + await rejected + expect(worker.cleaned).toBe(true) + expect(store.get(run.id)).toMatchObject({ state: 'failed', reason: 'shadow_admission_or_lease_lost' }) +}) +it('preserves an operator stop and reaps the attempt', async () => { + const { store, run, host } = setup() + const task = runShadowAttempt(store, run.id, host) + const rejected = expect(task).rejects.toThrow(/lease_lost/) + await vi.advanceTimersByTimeAsync(0) + store.stop(run.id, 'operator_stop') + await vi.advanceTimersByTimeAsync(200) + await rejected + expect(worker.cleaned).toBe(true) + expect(store.get(run.id)).toMatchObject({ state: 'stopped', reason: 'operator_stop' }) +}) +it('does not mark completion if source cleanup fails', async () => { + const { store, run, host, close } = setup() + close.mockImplementation(() => { + throw new Error('unsubscribe failed') + }) + const task = runShadowAttempt(store, run.id, host) + const rejected = expect(task).rejects.toThrow('shadow_source_cleanup_failed') + await vi.advanceTimersByTimeAsync(2101) + await rejected + expect(store.get(run.id).state).toBe('failed') +}) +it('refuses to finish an unexpired lease through expired-attempt recovery', () => { + const { store, run } = setup() + const lease = store.claim(run.id, 1500, () => {}) + expect(store.failExpired(lease, 'worker_gone')).toBe(false) + expect(store.get(run.id).state).toBe('running') +}) +it('stops immediately when the active source reports a disconnect', async () => { + const { store, run, host } = setup() + const open = host.openSources + let disconnect = () => {} + host.openSources = (input) => { + disconnect = input.fail + return open(input) + } + const task = runShadowAttempt(store, run.id, host) + const rejected = expect(task).rejects.toThrow('shadow_source_failed') + await vi.advanceTimersByTimeAsync(0) + disconnect() + await rejected + expect(worker.cleaned).toBe(true) + expect(store.get(run.id)).toMatchObject({ state: 'failed', reason: 'shadow_source_failed' }) +}) +it('rechecks admission after cleanup before publishing completion', async () => { + const { store, run, host, close } = setup() + close.mockImplementation(() => { + host.admit = () => { + throw new Error('active profile changed during cleanup') + } + }) + const task = runShadowAttempt(store, run.id, host) + const rejected = expect(task).rejects.toThrow('shadow_admission_changed_before_completion') + await vi.advanceTimersByTimeAsync(2101) + await rejected + expect(store.get(run.id).state).toBe('failed') +}) +it('rejects future active timestamps before buffering them', async () => { + const { store, run, host, active } = setup() + const task = runShadowAttempt(store, run.id, host) + const rejected = expect(task).rejects.toThrow('shadow_active_stream_failed') + await vi.advanceTimersByTimeAsync(0) + active(start + 1) + await rejected + expect(store.get(run.id)).toMatchObject({ state: 'failed', point_count: 0 }) +}) +it('flushes active observations before sealing the final drain', async () => { + const { store, run, host, active } = setup() + const open = host.openSources + let emitted = false + host.openSources = input => ({ ...open(input), flush: () => { + if (!emitted && Date.now() >= start + 2100) { + active(start + 1999) + emitted = true + } + } }) + const task = runShadowAttempt(store, run.id, host) + await vi.advanceTimersByTimeAsync(2101) + await task + expect(store.get(run.id)).toMatchObject({ state: 'completed', point_count: 2 }) + expect(store.snapshot(run.id, { maximumPoints: 10, maximumBytes: 10000 }).samples[1]).toMatchObject({ timestampMs: start + 1999, pairing: { activePresent: true, candidatePresent: false } }) +}) +it('refuses asynchronous final-drain polling', async () => { + const { store, run, host } = setup() + const open = host.openSources + host.openSources = input => ({ ...open(input), flush: async () => {} }) + const task = runShadowAttempt(store, run.id, host) + const rejected = expect(task).rejects.toThrow(/admission_or_lease_lost/) + await vi.advanceTimersByTimeAsync(200) + await rejected + expect(store.get(run.id).state).toBe('failed') +}) +it('records lease expiry during final cleanup as failure rather than leaving a running attempt', async () => { + const { store, run, host, close } = setup() + close.mockImplementation(() => { + vi.setSystemTime(start + 3000) + }) + const task = runShadowAttempt(store, run.id, host) + const rejected = expect(task).rejects.toThrow(/Stale shadow lease/) + await vi.advanceTimersByTimeAsync(2101) + await rejected + expect(store.get(run.id)).toMatchObject({ state: 'failed', reason: 'shadow_completion_lease_lost' }) +}) +it('keeps legacy plans inspectable but refuses to execute without a frozen policy', async () => { + const { store, run, host } = setup() + const legacy = { ...run.plan } + delete legacy.execution + const old = store.create(legacy, 'legacy') + expect(store.get(old.id).plan.execution).toBeUndefined() + await expect(runShadowAttempt(store, old.id, host)).rejects.toThrow(/execution policies/) + expect(store.get(old.id).state).toBe('pending') + expect(worker.calls).toBe(0) +}) +it('binds the other physical channel configuration before launching a worker', async () => { + const { store, run, host } = setup() + host.configs.right = resolveConfig([{ path: 'heart_rate.trackingToleranceBpm', value: 20 }]).resolved + await expect(runShadowAttempt(store, run.id, host)).rejects.toThrow(/frozen plan/) + expect(worker.calls).toBe(0) + expect(store.get(run.id).state).toBe('pending') +}) +it('source factories receive a policy copy and cannot change worker lease budgets', async () => { + const { store, run, host } = setup() + const open = host.openSources + host.openSources = (input) => { + expect(input.execution).toEqual(run.plan.execution) + input.execution.leaseMs = 3600000 + input.execution.budgets.wallMs = 1 + return open(input) + } + const abort = new AbortController() + const task = runShadowAttempt(store, run.id, host, abort.signal) + const rejected = expect(task).rejects.toThrow('shadow_cancelled') + await vi.advanceTimersByTimeAsync(0) + expect(store.get(run.id).lease_until_ms).toBe(start + 1500) + expect(store.get(run.id).plan.execution).toEqual(run.plan.execution) + abort.abort() + await rejected +}) +it('rejects incompatible poll cadence and freezes policy changes into plan idempotency', () => { + const { store, run } = setup() + const execution = structuredClone(run.plan.execution) + if (!execution) throw new Error('Missing fixture policy') + execution.sources.pollMs = 100 + expect(() => store.create({ ...run.plan, execution }, 'invalid')).toThrow(/Invalid shadow window/) + execution.sources.pollMs = 50 + execution.maximumRecords++ + expect(() => store.create({ ...run.plan, execution }, 'test')).toThrow(/idempotency/) +}) + +it('persists the receipt with completion and rejects later corruption', async () => { + const { db, store, run, host } = setup() + const task = runShadowAttempt(store, run.id, host) + await vi.advanceTimersByTimeAsync(2101) + const result = await task + const reopened = new ShadowStore(db, { maximumRuns: 10, maximumPoints: 100, maximumBytes: 100000 }) + expect(reopened.get(run.id).workerReceipt).toMatchObject({ runId: run.id, attemptId: result.lease.attemptId, + planHash: run.plan_hash, receipt: result.receipt }) + expect(reopened.snapshot(run.id, { maximumPoints: 10, maximumBytes: 10000 }).run.workerReceipt).not.toBeNull() + const execution = result.run.plan.execution + if (!execution) throw new Error('Missing execution') + for (const bad of [ + { ...result.receipt, inputBytes: 100001 }, { ...result.receipt, outputBytes: 100001 }, + { ...result.receipt, peakObservedRssBytes: 100000001 }, { ...result.receipt, rssSamples: 0 }, + { ...result.receipt, budgets: { ...result.receipt.budgets, cpuSeconds: 3 } }, + { ...result.receipt, health: { ...result.receipt.health, cpu_ms: 1001 } }, + { ...result.receipt, health: { ...result.receipt.health, records: 11 } }, + ]) expect(() => validateShadowWorkerReceipt(bad, execution)).toThrow() + db.prepare('UPDATE biometric_shadow_runs SET worker_receipt_hash = \'corrupted\' WHERE id = ?').run(run.id) + expect(() => reopened.get(run.id)).toThrow(/receipt integrity/) +}) +it('does not complete an executable plan without a worker receipt', () => { + const { store, run } = setup() + const lease = store.claim(run.id, 1500, () => {}) + expect(() => store.finish(lease, 'completed', 'missing')).toThrow() + expect(store.get(run.id)).toMatchObject({ state: 'running', workerReceipt: null }) +}) +it('fails completion after cleanup when the worker receipt violates the saved budget', async () => { + const { store, run, host } = setup() + worker.corruptReceipt = true + const task = runShadowAttempt(store, run.id, host) + const rejected = expect(task).rejects.toThrow(/receipt does not match/) + await vi.advanceTimersByTimeAsync(2101) + await rejected + expect(worker.cleaned).toBe(true) + expect(store.get(run.id)).toMatchObject({ state: 'failed', workerReceipt: { receipt: { kind: 'partial_supervisor_observation' } }, reason: 'shadow_completion_failed' }) +}) + +it('preserves cleanup uncertainty through cancellation and durable failure evidence', async () => { + const { store, run, host } = setup() + worker.cleanupUnconfirmed = true + const abort = new AbortController() + const task = runShadowAttempt(store, run.id, host, abort.signal) + const rejected = expect(task).rejects.toMatchObject({ code: 'SHADOW_CLEANUP_UNCONFIRMED' }) + await vi.advanceTimersByTimeAsync(0) + abort.abort() + await rejected + expect(store.get(run.id)).toMatchObject({ state: 'failed', reason: 'shadow_worker_cleanup_unconfirmed', workerReceipt: { receipt: { cleanupConfirmed: false } } }) +}) + +it('attaches partial evidence after an operator stop without changing terminal authority', async () => { + const { db, store, run, host } = setup() + const abort = new AbortController() + const task = runShadowAttempt(store, run.id, host, abort.signal) + const rejected = expect(task).rejects.toThrow('shadow_cancelled') + await vi.advanceTimersByTimeAsync(0) + const stopped = store.stop(run.id, 'operator') + await vi.advanceTimersByTimeAsync(50) + abort.abort() + await rejected + const reopened = new ShadowStore(db).get(run.id) + expect(reopened).toMatchObject({ state: 'stopped', reason: 'operator', ended_at_ms: stopped.ended_at_ms, + workerReceipt: { schema: 'biometric-shadow-partial-receipt-v1', recordedAtMs: start + 50, receipt: { cleanupConfirmed: true } } }) + expect(shadowReportSchema.parse(reportShadowRun(store, run.id, { maximumPoints: 100, maximumBytes: 100000 })).workerReceipt).toEqual(reopened.workerReceipt) + const receipt = reopened.workerReceipt + if (!receipt) throw new Error('Receipt missing') + const lease = { runId: run.id, attemptId: receipt.attemptId } + expect(store.recordPartialReceipt(lease, receipt.receipt)).toBe(true) + expect(() => store.recordPartialReceipt(lease, { ...receipt.receipt, inputBytes: 200 })).toThrow('already recorded') + expect(store.recordPartialReceipt({ ...lease, attemptId: 'replacement' }, receipt.receipt)).toBe(false) + db.prepare('UPDATE biometric_shadow_runs SET worker_receipt_hash = ? WHERE id = ?').run('corrupted', run.id) + expect(() => store.get(run.id)).toThrow('receipt integrity') +}) +it('retains cleanup quarantine if storing partial evidence also fails', async () => { + const { store, run, host } = setup() + worker.cleanupUnconfirmed = true + worker.fail = true + vi.spyOn(store, 'recordPartialReceipt').mockImplementation(() => { + throw new Error('disk unavailable') + }) + const task = runShadowAttempt(store, run.id, host) + const rejected = expect(task).rejects.toMatchObject({ code: 'SHADOW_CLEANUP_UNCONFIRMED', cause: { message: 'Shadow partial resource evidence could not be stored' } }) + await vi.advanceTimersByTimeAsync(0) + await rejected +}) + +it('preserves overruns as partial evidence but rejects inconsistent availability and policy', () => { + const { store, run } = setup() + const execution = run.plan.execution + if (!execution) throw new Error('Missing execution') + const receipt: ShadowPartialReceipt = { kind: 'partial_supervisor_observation', health: null, inputBytes: 100001, outputBytes: 200001, + diagnosticBytes: 20000, peakObservedRssBytes: 200000000, rssSamples: 1, maximumObservationGapMs: 100, + budgets: execution.budgets, cleanupConfirmed: false, memoryScope: 'sum_rss_of_shadow_process_group', exactKernelPeak: false } + expect(validateShadowPartialReceipt(receipt, execution)).toEqual(receipt) + for (const bad of [{ ...receipt, rssSamples: 0 }, { ...receipt, peakObservedRssBytes: null }, + { ...receipt, budgets: { ...receipt.budgets, cpuSeconds: 3 } }, { ...receipt, outputBytes: -1 }]) { + expect(() => validateShadowPartialReceipt(bad, execution)).toThrow() + } + const lease = store.claim(run.id, 1500, () => {}) + expect(store.recordPartialReceipt(lease, receipt)).toBe(false) + store.stop(run.id, 'operator') + expect(store.recordPartialReceipt(lease, receipt)).toBe(true) + expect(reportShadowRun(store, run.id, { maximumPoints: 100, maximumBytes: 100000 }).issues).toContain('worker_cleanup_unconfirmed') +}) diff --git a/src/biometrics/shadow/tests/build.test.ts b/src/biometrics/shadow/tests/build.test.ts new file mode 100644 index 000000000..e90b3cac4 --- /dev/null +++ b/src/biometrics/shadow/tests/build.test.ts @@ -0,0 +1,65 @@ +// @vitest-environment node +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' +import { expect, it } from 'vitest' +import { configHash } from '../../algorithms/canonical' +import { probeShadowBuild } from '../build' +const location = { pythonPath: resolve('modules/piezo-processor/.venv/bin/python'), modulesPath: resolve('modules') } +it('probes the exact locked interpreter and requires shadow sources and intact identity', async () => { + const build = await probeShadowBuild(location) + expect(build).toEqual(await probeShadowBuild(location)) + expect(build.identity.version).toBe('shadow-python-build-v1') + expect(build.identity.sources).toHaveProperty('common/biometrics/shadow_process.py') + expect(build.identity.sources).toHaveProperty('common/biometrics/signal.py') + const directory = await mkdtemp(join(tmpdir(), 'shadow-build-')) + try { + const executable = join(directory, 'probe') + for (const raw of [{ ...build, hash: '0'.repeat(64) }, { ...build, identity: { ...build.identity, version: 'waveform-python-build-v1' } }]) { + await writeFile(executable, `#!${process.execPath}\nprocess.stdout.write(${JSON.stringify(JSON.stringify(raw))});`, { mode: 0o700 }) + await expect(probeShadowBuild({ ...location, pythonPath: executable })).rejects.toThrow() + } + const missing = structuredClone(build.identity) + delete missing.sources['common/biometrics/shadow_process.py'] + await writeFile(executable, `#!${process.execPath}\nprocess.stdout.write(${JSON.stringify(JSON.stringify({ hash: configHash(missing), identity: missing }))});`, { mode: 0o700 }) + await expect(probeShadowBuild({ ...location, pythonPath: executable })).rejects.toThrow(/incomplete/) + await expect(probeShadowBuild({ ...location, pythonPath: join(directory, 'missing') })).rejects.toThrow(/unavailable/) + const abort = new AbortController() + abort.abort() + await expect(probeShadowBuild(location, abort.signal)).rejects.toThrow(/unavailable/) + } + finally { await rm(directory, { recursive: true, force: true }) } +}) +it('packages the host without live service imports and binds its exact bytes', async () => { + const { execFile } = await import('node:child_process') + const { promisify } = await import('node:util') + const { loadInstalledShadowHost, inspectInstalledShadowHost } = await import('../installed-host') + const { readFile } = await import('node:fs/promises') + const { shadowEngineBuildIdentity } = await import('../build') + const directory = await mkdtemp(resolve('node_modules/shadow-package-test-')) + try { + await promisify(execFile)(process.execPath, ['scripts/build/biometrics-worker.mjs', directory], { timeout: 30000 }) + const path = join(directory, 'shadow-host.cjs') + const loaded = loadInstalledShadowHost(path), api = loaded.api + loaded.assertCurrent() + expect(loadInstalledShadowHost(path).api).toBe(api) + expect(Object.keys(api).sort()).toEqual(['ShadowStore', 'openProfiledShadowHost', 'prepareShadowPlan', 'readInstalledShadowQualifications', 'runProfiledShadowAttempt', + 'shadowEngineBuildIdentity', 'shadowHostApiVersion', 'shadowPlanSchema', 'shadowStoragePolicySchema'].sort()) + const dependencies = JSON.parse(await readFile(join(directory, 'shadow-host-dependencies.json'), 'utf8')) as string[] + expect(dependencies).toContain('src/biometrics/algorithms/registry.ts') + expect(dependencies).toContain('src/streaming/prospectiveFrameTap.ts') + expect(dependencies.some(path => /src\/(hardware|automation|scheduler|homekit)\//.test(path) || path.endsWith('/piezoStream.ts'))).toBe(false) + const original = await shadowEngineBuildIdentity(path, location) + expect((await inspectInstalledShadowHost(path, location)).build).toEqual(original) + expect(original.identity.hostHash).toBe(loaded.hostHash) + expect(original.python).toEqual(await probeShadowBuild(location)) + expect(await api.shadowEngineBuildIdentity(path, location)).toEqual(original) + await writeFile(path, '// changed build\n' + await readFile(path, 'utf8')) + expect(() => loaded.assertCurrent()).toThrow(/changed/) + expect(() => loadInstalledShadowHost(path)).toThrow(/changed/) + const changed = await shadowEngineBuildIdentity(path, location) + expect(changed.hash).not.toBe(original.hash) + expect(changed.python.hash).toBe(original.python.hash) + } + finally { await rm(directory, { recursive: true, force: true }) } +}) diff --git a/src/biometrics/shadow/tests/context.test.ts b/src/biometrics/shadow/tests/context.test.ts new file mode 100644 index 000000000..bc2c700bd --- /dev/null +++ b/src/biometrics/shadow/tests/context.test.ts @@ -0,0 +1,91 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { mkdtempSync, rmSync, writeFileSync, utimesSync, symlinkSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, expect, it, vi } from 'vitest' +import { ShadowContextReader } from '../context' +const at = 1700000000000 +const cleanup: (() => void)[] = [] +afterEach(() => cleanup.splice(0).forEach(close => close())) +function fixture() { + const directory = mkdtempSync(join(tmpdir(), 'shadow-context-')), path = join(directory, 'presence.json') + const db = new Database(':memory:') + db.exec('CREATE TABLE side_settings(side TEXT, away_mode INTEGER); INSERT INTO side_settings VALUES (\'left\',0),(\'right\',0); CREATE TABLE device_settings(id INTEGER, bed_mode TEXT); INSERT INTO device_settings VALUES(1,\'two\');') + db.pragma('query_only = ON') + cleanup.push(() => { + db.close() + rmSync(directory, { recursive: true, force: true }) + }) + const assertCurrent = vi.fn() + const reader = new ShadowContextReader(db, path, { presenceStaleMs: 300000, pumpStaleMs: 5000, maximumPresenceBytes: 4096 }, assertCurrent) + const write = (value: unknown, mtime = at) => { + writeFileSync(path, JSON.stringify(value)) + utimesSync(path, mtime / 1000, mtime / 1000) + } + const sql = (value: string) => { + db.pragma('query_only = OFF') + db.exec(value) + db.pragma('query_only = ON') + } + return { db, path, reader, write, sql, assertCurrent } +} +const state = { version: 1, left: { vitals_presence: true, vitals_evidence_ts: at / 1000 }, right: { vitals_presence: false, vitals_evidence_ts: at / 1000 } } +it('uses committed per-side evidence and active sleeper settings rather than virtual UI occupancy', () => { + const { reader, write, sql } = fixture() + write(state) + expect(reader.snapshot({ type: 'piezo-dual' }, at)).toEqual({ activeSides: ['left', 'right'], leftOccupied: true, rightOccupied: false, pumpContextKnown: false }) + sql('UPDATE side_settings SET away_mode=1 WHERE side=\'right\'') + expect(reader.snapshot({}, at).activeSides).toEqual(['left']) + sql('UPDATE device_settings SET bed_mode=\'solo-right\'') + expect(reader.snapshot({}, at).activeSides).toEqual([]) + sql('UPDATE side_settings SET away_mode=0') + expect(reader.snapshot({}, at).activeSides).toEqual(['right']) +}) +it.each(['missing', 'malformed', 'version', 'stale_file', 'future_file', 'stale_evidence', 'future_evidence', 'oversize', 'symlink'])('preserves unknown presence for %s evidence', (mode) => { + const { reader, write, path } = fixture() + if (mode !== 'missing') write(state) + if (mode === 'malformed') writeFileSync(path, '{') + if (mode === 'version') write({ ...state, version: 2 }) + if (mode === 'stale_file') write(state, at - 300001) + if (mode === 'future_file') write(state, at + 1000) + if (mode === 'stale_evidence') write({ ...state, left: { ...state.left, vitals_evidence_ts: at / 1000 - 301 }, right: null }) + if (mode === 'future_evidence') write({ ...state, left: { ...state.left, vitals_evidence_ts: at / 1000 + 1 }, right: null }) + if (mode === 'oversize') write({ ...state, extra: 'x'.repeat(4096) }) + if (mode === 'symlink') { + rmSync(path) + writeFileSync(path + '.target', JSON.stringify(state)) + symlinkSync(path + '.target', path) + } + expect(reader.snapshot({}, at)).toMatchObject({ leftOccupied: null, rightOccupied: null }) +}) +it('does not let one missing side erase valid evidence on the other side', () => { + const { reader, write } = fixture() + write({ ...state, right: { vitals_presence: false, vitals_evidence_ts: null } }) + expect(reader.snapshot({}, at)).toMatchObject({ leftOccupied: true, rightOccupied: null }) +}) +it('marks incomplete or invalid settings unknown rather than guessing two sleepers', () => { + const { reader, sql } = fixture() + sql('DELETE FROM side_settings WHERE side=\'right\'') + expect(reader.snapshot({}, at).activeSides).toBeNull() + sql('INSERT INTO side_settings VALUES(\'right\',0); UPDATE device_settings SET bed_mode=\'unexpected\'') + expect(reader.snapshot({}, at).activeSides).toBeNull() +}) +it('requires both pump sides in this attempt and expires or invalidates incomplete updates', () => { + const { reader } = fixture() + expect(reader.snapshot({ type: 'frzHealth', left: { pump: { rpm: 1200 } } }, at).pumpContextKnown).toBe(false) + expect(reader.snapshot({ type: 'frzHealth', left: { pumpRpm: 0 }, right: { pump: { duty: '10' } } }, at).pumpContextKnown).toBe(true) + expect(reader.snapshot({}, at + 5000).pumpContextKnown).toBe(true) + expect(reader.snapshot({}, at + 5001).pumpContextKnown).toBe(false) + expect(reader.snapshot({ type: 'frzHealth', left: { rpm: 0 }, right: { rpm: 0 } }, at + 5001).pumpContextKnown).toBe(true) + expect(reader.snapshot({ type: 'frzHealth', left: { rpm: 0 }, right: {} }, at + 5002).pumpContextKnown).toBe(false) +}) +it('propagates replacement and clock errors instead of reporting a healthy context', () => { + const { reader, assertCurrent } = fixture() + reader.snapshot({}, at) + expect(() => reader.snapshot({}, at - 1)).toThrow(/clock/) + assertCurrent.mockImplementation(() => { + throw new Error('database replaced') + }) + expect(() => reader.snapshot({}, at)).toThrow('database replaced') +}) diff --git a/src/biometrics/shadow/tests/device-identity.test.ts b/src/biometrics/shadow/tests/device-identity.test.ts new file mode 100644 index 000000000..66c4b40c7 --- /dev/null +++ b/src/biometrics/shadow/tests/device-identity.test.ts @@ -0,0 +1,58 @@ +// @vitest-environment node +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import type { inspectInstalledShadowHost } from '../installed-host' +import { observedShadowHardware, installedShadowIdentity } from '../device-identity' +import { getDacMonitorIfRunning } from '@/src/hardware/dacMonitor.instance' +vi.mock('@/src/hardware/dacMonitor.instance', () => ({ getDacMonitorIfRunning: vi.fn() })) +const now = 1700000000000 +function monitor(version = 'J00', sensorLabel = '20600-0003-J55-B0708DE3', at: number | null = now, state = 'running') { + vi.mocked(getDacMonitorIfRunning).mockReturnValue({ getStatus: () => state, getLastPollAt: () => at, + getLastStatus: () => ({ podVersion: version, sensorLabel }) } as ReturnType) +} +beforeEach(() => vi.resetAllMocks()) +afterEach(() => vi.restoreAllMocks()) +it.each([['H00', 'H00'], ['I00', 'I11'], ['J00', 'J55']])('identifies %s by exact revision while discarding serials', (version, revision) => { + monitor(version, `20600-${revision}-SERIAL`) + expect(observedShadowHardware(1000, now)).toEqual({ hardwareId: `pod-sensor-v1/${version}/${revision}`, sampleRateHz: 500 }) +}) +it.each(['UNKNOWN', 'SERIAL-H00-JUNK-I11', '20600-K00-SERIAL', '20600-J550-SERIAL'])('rejects unknown or inconsistent sensor label %s', (label) => { + monitor('J00', label) + expect(() => observedShadowHardware(1000, now)).toThrow(/revision/) +}) +it.each([null, now + 1, now - 1001])('rejects missing, future or stale poll %s', (at) => { + monitor('J00', '20600-J55-SERIAL', at) + expect(() => observedShadowHardware(1000, now)).toThrow() +}) +it.each(['stopped', 'degraded'])('rejects %s monitors', (state) => { + monitor('J00', '20600-J55-SERIAL', now, state) + expect(() => observedShadowHardware(1000, now)).toThrow(/unavailable/) +}) +it('rejects an absent monitor without initializing one', () => { + vi.mocked(getDacMonitorIfRunning).mockReturnValue(null) + expect(() => observedShadowHardware(1000, now)).toThrow(/unavailable/) + expect(getDacMonitorIfRunning).toHaveBeenCalledOnce() +}) + +it('combines the inspected build with fresh qualified hardware and repeats artifact checks', () => { + vi.spyOn(Date, 'now').mockReturnValue(now) + const a = 'a'.repeat(64), b = 'b'.repeat(64) + const guard = vi.fn() + const inspection = { build: { hash: a, python: { hash: b } }, assertCurrent: guard } as unknown as Awaited> + const qualification = { qualificationId: 'test', manifestHash: a, engineBuildHash: a, pythonBuildHash: b, + hardwareId: 'pod-sensor-v1/J00/J55', sampleRateHz: 500, activeConfigHash: a, executionPolicyHash: a, + fixtureEvidenceHash: a, podEvidenceHash: a, heldOutEvidenceHash: a, + contextEvidence: 'host_observations_not_live_processor_cache_attestation', maximumRuntimeHeartbeatAgeMs: 1000, maximumHardwareObservationAgeMs: 1000, + healthPolicy: { qualificationId: 'test', minimumAvailableMemoryBytes: 1, minimumFreeDiskBytes: 1, maximumLoadPerCpu: 1, + maximumEventLoopDelayMs: 100, maximumPiezoAgeMs: 1000, maximumTemperatureC: 70, maximumSampleAgeMs: 1000 } } + expect(() => installedShadowIdentity(inspection, { ...qualification, pythonBuildHash: a })).toThrow(/build differs/) + const identity = installedShadowIdentity(inspection, qualification) + monitor() + expect(identity()).toEqual({ engineBuildHash: a, pythonBuildHash: b, hardwareId: qualification.hardwareId, sampleRateHz: 500 }) + monitor('I00', '20600-I11-SERIAL') + expect(identity).toThrow(/hardware differs/) + expect(guard).toHaveBeenCalledTimes(2) + guard.mockImplementation(() => { + throw new Error('artifact changed') + }) + expect(identity).toThrow(/artifact changed/) +}) diff --git a/src/biometrics/shadow/tests/fixtures/submission.ts b/src/biometrics/shadow/tests/fixtures/submission.ts new file mode 100644 index 000000000..022a94d46 --- /dev/null +++ b/src/biometrics/shadow/tests/fixtures/submission.ts @@ -0,0 +1,22 @@ +import { dataHash } from '../../../algorithms/canonical' +import { submitShadowSchema } from '../../submission-contracts' +export const submissionTime = 1700000000000 +export const submissionEntry = { qualificationId: 'synthetic-test-only' } +export function submissionFixture() { + const hash = 'a'.repeat(64) + return submitShadowSchema.parse({ idempotencyKey: 'submit', plan: { + schema: 'biometric-shadow-plan-v1', side: 'left', subjectId: 'fixture', candidateRevisionId: null, + expectedBindings: { left: { generation: 1, resolvedHash: hash }, right: { generation: 2, resolvedHash: hash } }, + execution: { version: 'shadow-execution-policy-v1', configHashes: { left: hash, right: hash }, + maximumRecords: 10, maximumOutputs: 10, maximumCpuMs: 1000, maximumInputBytes: 100000, maximumOutputBytes: 100000, + budgets: { cpuSeconds: 2, addressSpaceBytes: 100000000, maximumRssBytes: 100000000, wallMs: 10000, acknowledgmentMs: 1000, idleMs: 1000, rssIntervalMs: 100, rssTimeoutMs: 100 }, + leaseMs: 1500, heartbeatMs: 200, sources: { pollMs: 50, maximumActiveRows: 10, databaseBusyTimeoutMs: 50, + frames: { maximumFrames: 10, maximumBytes: 100000, maximumEventAgeMs: 1000, maximumFutureSkewMs: 0 }, + context: { presenceStaleMs: 300000, pumpStaleMs: 5000, maximumPresenceBytes: 4096 } } }, + startMs: submissionTime + 1000, endMs: submissionTime + 10000, candidateConfigHash: hash, activeConfigHash: hash, + engineBuildHash: hash, qualificationHash: dataHash(submissionEntry), maximumPoints: 10, maximumBytes: 10000, metrics: ['heart_rate'], + joining: { maximumPendingPoints: 10, maximumLatenessMs: 100, drainGraceMs: 500 }, + monitoring: { expectedIntervalMs: 1000, maximumGapMs: 2000, heartbeatStaleMs: 1000, minimumCandidateCoverage: 0.8, maximumCoverageLoss: 0.1, minimumQuality: 0.8, + alignment: { version: 'reference-alignment-v1', pointToleranceMs: 0, clockOffsetMs: 0, offsetSource: 'fixed' } }, + } }) +} diff --git a/src/biometrics/shadow/tests/frame-source.test.ts b/src/biometrics/shadow/tests/frame-source.test.ts new file mode 100644 index 000000000..0e944a6e0 --- /dev/null +++ b/src/biometrics/shadow/tests/frame-source.test.ts @@ -0,0 +1,206 @@ +// @vitest-environment node +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import { openShadowFrameSource } from '../frame-source' +import { publishProspectiveFrame, subscribeProspectiveFrames } from '../../../streaming/prospectiveFrameTap' +const start = 1700000000000 +const cleanups: (() => void)[] = [] +const resetArrivalClock = () => { + (globalThis as { __sleepypodProspectiveObservedAtMs?: number }).__sleepypodProspectiveObservedAtMs = undefined +} +beforeEach(() => { + vi.useFakeTimers() + vi.setSystemTime(start) +}) +afterEach(() => { + cleanups.splice(0).forEach(close => close()) + resetArrivalClock() + vi.useRealTimers() +}) +function setup(maximumFrames = 2, maximumBytes = 100000) { + const fail = vi.fn(), context = { activeSides: ['left'], leftOccupied: true, rightOccupied: false, pumpContextKnown: false } + const controller = new AbortController() + const source = openShadowFrameSource({ startMs: start, endMs: start + 1000, signal: controller.signal, + budgets: { maximumFrames, maximumBytes, maximumEventAgeMs: 10000, maximumFutureSkewMs: 0 }, context: () => context, fail }) + cleanups.push(source.close) + return { source, iterator: source.frames[Symbol.asyncIterator](), fail, context, controller } +} +const frame = () => ({ type: 'piezo-dual', ts: start / 1000, freq: 500, left1: [-2147483648, 0, 2147483647], right1: [1, -2, 3] }) +function publish(value = frame(), backlog = false) { + publishProspectiveFrame(value, backlog, Date.now()) +} +it('snapshots signed int32 waveforms and arrival context without retaining mutable stream data', async () => { + const { iterator, context } = setup() + const value = frame() + publish(value) + value.left1[0] = 99 + context.activeSides[0] = 'right' + const row = (await iterator.next()).value + expect(row).toMatchObject({ type: 'frame', observedAtMs: start, record: { type: 'piezo-dual', freq: 500 }, context: { activeSides: ['left'] } }) + const bytes = Buffer.from(row.record.left1, 'base64') + expect([0, 4, 8].map(offset => bytes.readInt32LE(offset))).toEqual([-2147483648, 0, 2147483647]) +}) +it('excludes backlog and unrelated sensors from prospective evidence', async () => { + const { iterator, fail } = setup() + publish(frame(), true) + publishProspectiveFrame({ type: 'capSense' }, false, start) + const next = iterator.next() + publish() + expect((await next).value.record.type).toBe('piezo-dual') + expect(fail).not.toHaveBeenCalled() +}) +it('preserves only the bounded pump fields used by the shared gate', async () => { + const { iterator } = setup() + publishProspectiveFrame({ type: 'frzHealth', ts: start / 1000, left: { pumpRpm: 0, pump: { rpm: '1200', duty: 30 }, unrelated: 'discard' }, right: { duty: 0 } }, false, start) + expect((await iterator.next()).value.record).toEqual({ type: 'frzHealth', ts: start / 1000, left: { pumpRpm: 0, pump: { rpm: '1200', duty: 30 } }, right: { duty: 0 } }) +}) +it.each(['frames', 'bytes'])('fails and closes on %s queue overflow', async (kind) => { + const { iterator, fail } = setup(1, kind === 'bytes' ? 1 : 100000) + publish() + if (kind === 'frames') publish() + expect(fail).toHaveBeenCalledOnce() + expect((await iterator.next()).done).toBe(true) + publish() + expect(fail).toHaveBeenCalledOnce() +}) +it.each([Array(5001).fill(0), [2147483648], [1.5], [NaN]])('rejects malformed or oversized channel before queueing', async (left1) => { + const { iterator, fail } = setup() + publish({ ...frame(), left1 }) + expect(fail).toHaveBeenCalledOnce() + expect((await iterator.next()).done).toBe(true) +}) +it('ends pending reads and unsubscribes on abort', async () => { + const { iterator, controller, fail } = setup() + const pending = iterator.next() + controller.abort() + expect((await pending).done).toBe(true) + publish() + expect(fail).not.toHaveBeenCalled() +}) +it('reports unread input at collection end instead of silently dropping it', async () => { + const { iterator, fail } = setup() + publish() + await vi.advanceTimersByTimeAsync(1000) + expect(fail).toHaveBeenCalledOnce() + expect((await iterator.next()).done).toBe(true) +}) +it('ends an empty collection normally and rejects concurrent consumers', async () => { + const { source, iterator, fail } = setup() + expect(() => source.frames[Symbol.asyncIterator]()).toThrow(/one consumer/) + const pending = iterator.next() + await expect(iterator.next()).rejects.toThrow(/Concurrent/) + await vi.advanceTimersByTimeAsync(1000) + expect((await pending).done).toBe(true) + expect(fail).not.toHaveBeenCalled() +}) +it('isolates subscriber faults so live ingestion and other subscribers continue', () => { + const failed = vi.fn(), healthy = vi.fn() + cleanups.push(subscribeProspectiveFrames({ frame: () => { + throw new Error('consumer failed') + }, failed })) + cleanups.push(subscribeProspectiveFrames({ frame: healthy, failed: () => {} })) + expect(() => publish()).not.toThrow() + publish() + expect(failed).toHaveBeenCalledOnce() + expect(healthy).toHaveBeenCalledTimes(2) +}) +it('unwinds subscriptions and timers when subscriber admission fails', () => { + for (let i = 0; i < 4; i++) cleanups.push(subscribeProspectiveFrames({ frame: () => {}, failed: () => {} })) + expect(() => setup()).toThrow(/subscriber limit/) + expect(vi.getTimerCount()).toBe(0) +}) +it.each([-20, 1])('fails on stale/future event timestamps (%s seconds)', async (seconds) => { + const { iterator, fail } = setup() + publish({ ...frame(), ts: start / 1000 + seconds }) + expect(fail).toHaveBeenCalledOnce() + expect((await iterator.next()).done).toBe(true) +}) +it('keeps null pump values, omits missing sides and skips the frequency when absent', async () => { + const { iterator, fail } = setup(4) + publishProspectiveFrame({ type: 'frzHealth', ts: start / 1000, left: { pumpRpm: null, rpm: 0 } }, false, start) + publishProspectiveFrame({ type: 'frzHealth', ts: start / 1000, right: { duty: 1 } }, false, start) + publishProspectiveFrame({ type: 'piezo-dual', ts: start / 1000, left1: [1], right1: [2] }, false, start) + expect((await iterator.next()).value.record).toEqual({ type: 'frzHealth', ts: start / 1000, left: { pumpRpm: null, rpm: 0 } }) + expect((await iterator.next()).value.record).toEqual({ type: 'frzHealth', ts: start / 1000, right: { duty: 1 } }) + const waveform = (await iterator.next()).value.record + expect(waveform).not.toHaveProperty('freq') + expect(Buffer.from(waveform.left1, 'base64').readInt32LE(0)).toBe(1) + expect(fail).not.toHaveBeenCalled() +}) +it.each([ + { left: { pumpRpm: 'fast' } }, { left: { pumpRpm: true } }, { left: 'left' }, { left: [] }, { right: { pump: 'none' } }, { right: { pump: [1] } }, +])('fails on malformed pump context %#', async (sides) => { + const { iterator, fail } = setup() + publishProspectiveFrame({ type: 'frzHealth', ts: start / 1000, ...sides }, false, start) + expect(fail).toHaveBeenCalledOnce() + expect((await iterator.next()).done).toBe(true) +}) +it('rejects windows that are malformed or too far ahead', () => { + const budgets = { maximumFrames: 1, maximumBytes: 1000, maximumEventAgeMs: 10000, maximumFutureSkewMs: 0 } + const open = (startMs: number, endMs: number) => openShadowFrameSource({ startMs, endMs, signal: new AbortController().signal, budgets, context: () => null, fail: () => {} }) + expect(() => open(-1, start)).toThrow('Invalid prospective source window') + expect(() => open(start, start)).toThrow('Invalid prospective source window') + expect(() => open(start, start + 8 * 86400000)).toThrow('Invalid prospective source window') + expect(() => open(start + 86400000, start + 7 * 86400000 + 1000)).toThrow('Invalid prospective source window') + expect(() => open(1.5, start)).toThrow('Invalid prospective source window') + expect(vi.getTimerCount()).toBe(0) +}) +it('starts closed when the signal is already aborted', async () => { + const controller = new AbortController() + controller.abort() + const fail = vi.fn() + const source = openShadowFrameSource({ startMs: start, endMs: start + 1000, signal: controller.signal, + budgets: { maximumFrames: 1, maximumBytes: 1000, maximumEventAgeMs: 10000, maximumFutureSkewMs: 0 }, context: () => null, fail }) + expect((await source.frames[Symbol.asyncIterator]().next()).done).toBe(true) + publish() + expect(fail).not.toHaveBeenCalled() + expect(vi.getTimerCount()).toBe(0) +}) +it('ignores arrivals outside the window and fails on future or regressing arrival clocks', async () => { + const { iterator, fail } = setup(4) + resetArrivalClock() + publishProspectiveFrame(frame(), false, start - 1) + vi.setSystemTime(start + 1000) + publishProspectiveFrame({ ...frame(), ts: (start + 1000) / 1000 }, false, start + 1000) + expect(fail).not.toHaveBeenCalled() + vi.setSystemTime(start + 500) + publishProspectiveFrame({ ...frame(), ts: (start + 500) / 1000 }, false, start + 600) + expect(fail).toHaveBeenCalledOnce() + expect((await iterator.next()).done).toBe(true) + const regressing = setup(4) + resetArrivalClock() + publishProspectiveFrame({ ...frame(), ts: (start + 500) / 1000 }, false, start + 500) + expect(regressing.fail).not.toHaveBeenCalled() + resetArrivalClock() + publishProspectiveFrame({ ...frame(), ts: (start + 100) / 1000 }, false, start + 100) + expect(regressing.fail).toHaveBeenCalledOnce() + expect((await regressing.iterator.next()).done).toBe(true) +}) +it('returns early from iteration and reports unread frames only once the window has ended', async () => { + const early = setup(4) + publish() + expect((await early.iterator.return?.())?.done).toBe(true) + expect(early.fail).not.toHaveBeenCalled() + expect((await early.iterator.next()).done).toBe(true) + const late = setup(4) + publish() + vi.setSystemTime(start + 1000) + expect((await late.iterator.return?.())?.done).toBe(true) + expect(late.fail).toHaveBeenCalledOnce() +}) +it('releases queue byte capacity when a frame is consumed', async () => { + const { iterator, fail } = setup(1, 1000) + publish() + await iterator.next() + publish() + expect((await iterator.next()).done).toBe(false) + expect(fail).not.toHaveBeenCalled() +}) +it('isolates a failing consumer notification from the collection timer', async () => { + const { iterator, fail } = setup() + fail.mockImplementation(() => { + throw new Error('consumer failed') + }) + publish() + await expect(vi.advanceTimersByTimeAsync(1000)).resolves.toBeDefined() + expect((await iterator.next()).done).toBe(true) +}) diff --git a/src/biometrics/shadow/tests/health-monitor.test.ts b/src/biometrics/shadow/tests/health-monitor.test.ts new file mode 100644 index 000000000..1d2f46a17 --- /dev/null +++ b/src/biometrics/shadow/tests/health-monitor.test.ts @@ -0,0 +1,102 @@ +// @vitest-environment node +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import { startShadowHealthMonitor, openShadowHealthMonitor } from '../health-monitor' +import { collectAdmissionSample } from '../../backtest/admission' +import type * as Admission from '../../backtest/admission' +vi.mock('../../backtest/admission', async original => ({ ...await original(), collectAdmissionSample: vi.fn() })) +const now = 1700000000000 +const policy = { qualificationId: 'test', minimumAvailableMemoryBytes: 100, minimumFreeDiskBytes: 100, + maximumLoadPerCpu: 1, maximumEventLoopDelayMs: 100, maximumPiezoAgeMs: 1000, maximumTemperatureC: 70, maximumSampleAgeMs: 1000 } +const sample = () => ({ measuredAtMs: Date.now(), availableMemoryBytes: 1000, freeDiskBytes: 1000, + loadPerCpu: 0.1, eventLoopDelayMs: 1, piezoReceivedAtMs: Date.now(), temperatureC: 40 }) +beforeEach(() => { + vi.useFakeTimers() + vi.setSystemTime(now) +}) +afterEach(() => { + vi.clearAllTimers() + vi.useRealTimers() + vi.clearAllMocks() +}) +it('refreshes real collector arguments and isolates published snapshots', async () => { + vi.mocked(collectAdmissionSample).mockImplementation(async (directory, piezo) => { + expect(directory).toBe('/trusted/data') + expect(piezo()).toBe(Date.now() - 10) + return sample() + }) + const monitor = await openShadowHealthMonitor('/trusted/data', policy, () => Date.now() - 10) + const first = monitor.read() + expect(first).not.toBeNull() + if (first) first.availableMemoryBytes = 0 + expect(monitor.read()?.availableMemoryBytes).toBe(1000) + await vi.advanceTimersByTimeAsync(500) + expect(collectAdmissionSample).toHaveBeenCalledTimes(2) + expect(monitor.read()?.measuredAtMs).toBe(now + 500) + monitor.close() + expect(monitor.read()).toBeNull() + await vi.advanceTimersByTimeAsync(1000) + expect(collectAdmissionSample).toHaveBeenCalledTimes(2) +}) +it('makes a stalled refresh terminal without overlapping I/O or accepting its late result', async () => { + let resolve!: (value: ReturnType) => void + const collect = vi.fn().mockResolvedValueOnce(sample()).mockImplementationOnce(() => new Promise((done) => { + resolve = done + })) + const monitor = await startShadowHealthMonitor(collect, policy) + await vi.advanceTimersByTimeAsync(1000) + expect(monitor.read()).toBeNull() + resolve(sample()) + await vi.advanceTimersByTimeAsync(10000) + expect(monitor.read()).toBeNull() + expect(collect).toHaveBeenCalledTimes(2) +}) +it('rejects initial timeout and cancellation without waiting for blocked collection', async () => { + const pending = startShadowHealthMonitor(() => new Promise(() => {}), policy) + const rejected = expect(pending).rejects.toThrow(/deadline/) + await vi.advanceTimersByTimeAsync(500) + await rejected + const abort = new AbortController() + const task = startShadowHealthMonitor(() => new Promise(() => {}), policy, abort.signal) + const cancelled = expect(task).rejects.toThrow(/closed/) + abort.abort() + await cancelled +}) +it('denies stale or unhealthy evidence and clears previous health on failed refresh', async () => { + const collect = vi.fn().mockResolvedValueOnce(sample()).mockResolvedValueOnce(null).mockResolvedValueOnce({ ...sample(), temperatureC: 100 }) + const monitor = await startShadowHealthMonitor(collect, policy) + vi.setSystemTime(now + 1001) + expect(monitor.read()).toBeNull() + await vi.advanceTimersByTimeAsync(500) + expect(monitor.read()).toBeNull() + await vi.advanceTimersByTimeAsync(500) + expect(monitor.read()).toBeNull() + monitor.close() +}) +it('closes on collector rejection and observes late rejected promises after timeout', async () => { + const monitor = await startShadowHealthMonitor(vi.fn().mockResolvedValueOnce(sample()).mockRejectedValueOnce(new Error('I/O failed')), policy) + await vi.advanceTimersByTimeAsync(500) + expect(monitor.read()).toBeNull() + expect(vi.getTimerCount()).toBe(0) + let reject!: (error: Error) => void + const pending = startShadowHealthMonitor(() => new Promise((_resolve, fail) => { + reject = fail + }), policy) + const rejected = expect(pending).rejects.toThrow(/deadline/) + await vi.advanceTimersByTimeAsync(500) + await rejected + reject(new Error('late failure')) + await Promise.resolve() +}) +it('does not collect after pre-abort and clears scheduled refresh on later abort', async () => { + const pre = new AbortController(), collect = vi.fn().mockResolvedValue(sample()) + pre.abort() + await expect(startShadowHealthMonitor(collect, policy, pre.signal)).rejects.toThrow(/closed/) + expect(collect).not.toHaveBeenCalled() + const abort = new AbortController() + const monitor = await startShadowHealthMonitor(collect, policy, abort.signal) + abort.abort() + expect(monitor.read()).toBeNull() + expect(vi.getTimerCount()).toBe(0) + await vi.advanceTimersByTimeAsync(2000) + expect(collect).toHaveBeenCalledTimes(1) +}) diff --git a/src/biometrics/shadow/tests/host.test.ts b/src/biometrics/shadow/tests/host.test.ts new file mode 100644 index 000000000..fa89709b9 --- /dev/null +++ b/src/biometrics/shadow/tests/host.test.ts @@ -0,0 +1,178 @@ +// @vitest-environment node +import { beforeEach, expect, it, vi } from 'vitest' +import { resolveShadowProfiles } from '../profiles' +import { dataHash } from '../../algorithms/canonical' +import { openPlannedShadowHost, openProfiledShadowHost, runPlannedShadowAttempt, runProfiledShadowAttempt } from '../host' +import { openShadowActiveDatabase } from '../active-database' +import { openShadowHealthMonitor } from '../health-monitor' +import { installedShadowAdmitter, readInstalledShadowQualifications } from '../admission' +import { plannedShadowSourceFactory } from '../planned-sources' +import { shadowRuntimeBindingReader } from '../runtime-binding' +import { runShadowAttempt } from '../attempt' +import type { ShadowStore } from '../store' +vi.mock('../profiles', () => ({ resolveShadowProfiles: vi.fn() })) +vi.mock('../active-database', () => ({ openShadowActiveDatabase: vi.fn() })) +vi.mock('../health-monitor', () => ({ openShadowHealthMonitor: vi.fn() })) +vi.mock('../admission', () => ({ installedShadowAdmitter: vi.fn(), readInstalledShadowQualifications: vi.fn() })) +vi.mock('../planned-sources', () => ({ plannedShadowSourceFactory: vi.fn() })) +vi.mock('../runtime-binding', () => ({ shadowRuntimeBindingReader: vi.fn() })) +vi.mock('../attempt', () => ({ runShadowAttempt: vi.fn() })) +const a = 'a'.repeat(64), now = 1700000000000 +function setup() { + const entry = { healthPolicy: { qualificationId: 'test' } } + const execution = { version: 'shadow-execution-policy-v1', configHashes: { left: a, right: a }, + maximumRecords: 10, maximumOutputs: 10, maximumCpuMs: 1000, maximumInputBytes: 100000, maximumOutputBytes: 100000, + budgets: { cpuSeconds: 2, addressSpaceBytes: 100000000, maximumRssBytes: 100000000, wallMs: 10000, acknowledgmentMs: 1000, idleMs: 1000, rssIntervalMs: 100, rssTimeoutMs: 100 }, + leaseMs: 1500, heartbeatMs: 200, sources: { pollMs: 50, maximumActiveRows: 10, databaseBusyTimeoutMs: 50, + frames: { maximumFrames: 10, maximumBytes: 100000, maximumEventAgeMs: 1000, maximumFutureSkewMs: 0 }, + context: { presenceStaleMs: 300000, pumpStaleMs: 5000, maximumPresenceBytes: 4096 } } } + const plan = { schema: 'biometric-shadow-plan-v1', side: 'left', subjectId: 'test', execution, startMs: now, endMs: now + 10000, + candidateConfigHash: a, activeConfigHash: a, engineBuildHash: a, qualificationHash: dataHash(entry), maximumPoints: 10, maximumBytes: 10000, metrics: ['heart_rate'], + joining: { maximumPendingPoints: 10, maximumLatenessMs: 100, drainGraceMs: 500 }, + monitoring: { expectedIntervalMs: 1000, maximumGapMs: 2000, heartbeatStaleMs: 1000, minimumCandidateCoverage: 0.8, maximumCoverageLoss: 0.1, minimumQuality: 0.8, + alignment: { version: 'reference-alignment-v1', pointToleranceMs: 0, clockOffsetMs: 0, offsetSource: 'fixed' } } } + const databases = [0, 1].map(() => ({ db: {}, close: vi.fn(), assertCurrent: vi.fn() })) + vi.mocked(openShadowActiveDatabase).mockReturnValueOnce(databases[0] as unknown as ReturnType) + .mockReturnValueOnce(databases[1] as unknown as ReturnType) + vi.mocked(readInstalledShadowQualifications).mockReturnValue({ entries: [entry] } as unknown as ReturnType) + const health = { read: vi.fn(), close: vi.fn() }, admit = vi.fn(), sources = vi.fn() + vi.mocked(openShadowHealthMonitor).mockResolvedValue(health) + vi.mocked(installedShadowAdmitter).mockReturnValue(admit) + vi.mocked(plannedShadowSourceFactory).mockReturnValue(sources) + vi.mocked(shadowRuntimeBindingReader).mockReturnValue(vi.fn()) + const installation = { workerPath: '/trusted/worker.cjs', pythonPath: '/trusted/python', modulesPath: '/trusted/modules', + configurationDatabase: '/trusted/config.db', biometricsDatabase: '/trusted/bio.db', presenceFile: '/trusted/presence.json', + configs: { left: {}, right: {} }, expectedBindings: { left: { generation: 1, resolvedHash: a }, right: { generation: 2, resolvedHash: a } }, identity: vi.fn(), piezoReceivedAt: vi.fn(() => now) } + return { plan, installation, databases, health, admit, sources } +} +beforeEach(() => vi.resetAllMocks()) +it('composes frozen policy, both binding guards and sampled health before returning a host', async () => { + const { plan, installation, databases, health, admit, sources } = setup() + const owned = await openPlannedShadowHost(plan, installation) + expect(admit).toHaveBeenCalledOnce() + expect(openShadowActiveDatabase).toHaveBeenCalledWith('/trusted/config.db', { busyTimeoutMs: 50 }) + const bindingOptions = vi.mocked(shadowRuntimeBindingReader).mock.calls[0][2] + bindingOptions.assertCurrent() + databases.forEach(db => expect(db.assertCurrent).toHaveBeenCalledOnce()) + expect(bindingOptions.expected.right.generation).toBe(2) + expect(openShadowHealthMonitor).toHaveBeenCalledWith('/trusted', { qualificationId: 'test' }, installation.piezoReceivedAt, owned.signal) + expect(owned.host.openSources).toBe(sources) + installation.configs.left = { changed: true } + expect(owned.host.configs.left).toEqual({}) + expect(() => owned.host.admit({ ...plan, endMs: plan.endMs + 1 } as Parameters[0])).toThrow(/plan changed/) + owned.close() + owned.close() + expect(owned.signal.aborted).toBe(true) + databases.forEach(db => expect(db.close).toHaveBeenCalledOnce()) + expect(health.close).toHaveBeenCalledOnce() + expect(() => owned.host.admit(plan as Parameters[0])).toThrow(/closed/) +}) +it('unwinds the first connection when the second database fails', async () => { + const { plan, installation, databases, health } = setup() + vi.mocked(openShadowActiveDatabase).mockReset().mockReturnValueOnce(databases[0] as unknown as ReturnType) + .mockImplementationOnce(() => { throw new Error('open failed') }) + await expect(openPlannedShadowHost(plan, installation)).rejects.toThrow(/open failed/) + expect(databases[0].close).toHaveBeenCalledOnce() + expect(openShadowHealthMonitor).not.toHaveBeenCalled() + expect(health.close).not.toHaveBeenCalled() +}) +it('unwinds both connections and health when initial admission fails', async () => { + const { plan, installation, databases, health, admit } = setup() + admit.mockImplementation(() => { + throw new Error('unqualified') + }) + await expect(openPlannedShadowHost(plan, installation)).rejects.toThrow(/unqualified/) + databases.forEach(db => expect(db.close).toHaveBeenCalledOnce()) + expect(health.close).toHaveBeenCalledOnce() +}) +it('cleans up a late health handle after cancellation during startup', async () => { + const { plan, installation, databases, health } = setup() + let resolve!: (value: typeof health) => void + vi.mocked(openShadowHealthMonitor).mockImplementation(() => new Promise((done) => { + resolve = done + })) + const abort = new AbortController() + const pending = openPlannedShadowHost(plan, installation, abort.signal) + abort.abort() + resolve(health) + await expect(pending).rejects.toThrow(/cancelled/) + databases.forEach(db => expect(db.close).toHaveBeenCalledOnce()) + expect(health.close).toHaveBeenCalledOnce() +}) +it('rejects pre-cancellation and missing qualification before opening resources', async () => { + const { plan, installation } = setup() + const abort = new AbortController() + abort.abort() + await expect(openPlannedShadowHost(plan, installation, abort.signal)).rejects.toThrow(/cancelled/) + vi.mocked(readInstalledShadowQualifications).mockReturnValue({ version: 'biometric-shadow-qualifications-v1', entries: [] }) + await expect(openPlannedShadowHost(plan, installation)).rejects.toThrow(/qualification/) + expect(openShadowActiveDatabase).not.toHaveBeenCalled() +}) +it('closes host resources when the attempt rejects after startup', async () => { + const { plan, installation, databases, health } = setup() + vi.mocked(runShadowAttempt).mockRejectedValue(new Error('claim failed')) + const store = { get: () => ({ plan }) } as unknown as ShadowStore + await expect(runPlannedShadowAttempt(store, 'test', installation)).rejects.toThrow(/claim failed/) + databases.forEach(db => expect(db.close).toHaveBeenCalledOnce()) + expect(health.close).toHaveBeenCalledOnce() +}) +it('closes admission connections when health collection rejects', async () => { + const { plan, installation, databases, health } = setup() + vi.mocked(openShadowHealthMonitor).mockRejectedValue(new Error('health unavailable')) + await expect(openPlannedShadowHost(plan, installation)).rejects.toThrow(/health unavailable/) + databases.forEach(db => expect(db.close).toHaveBeenCalledOnce()) + expect(health.close).not.toHaveBeenCalled() + expect(plannedShadowSourceFactory).not.toHaveBeenCalled() +}) + +it('resolves profiles using temporary handles then reopens admission with captured generations', async () => { + const { plan, installation, databases } = setup() + const capture = [0, 1].map(() => ({ db: {}, close: vi.fn(), assertCurrent: vi.fn() })) + vi.mocked(openShadowActiveDatabase).mockReset() + for (const db of [...capture, ...databases]) vi.mocked(openShadowActiveDatabase).mockReturnValueOnce(db as unknown as ReturnType) + vi.mocked(resolveShadowProfiles).mockReturnValue({ revisionId: null, configs: installation.configs, + expectedBindings: installation.expectedBindings, heartbeatAtMs: now } as ReturnType) + vi.mocked(runShadowAttempt).mockImplementation(async () => { + capture.forEach(db => expect(db.close).toHaveBeenCalledOnce()) + databases.forEach(db => expect(db.close).not.toHaveBeenCalled()) + throw new Error('test attempt finished') + }) + const store = { get: () => ({ plan }) } as unknown as ShadowStore + await expect(runProfiledShadowAttempt(store, 'test', installation, null)).rejects.toThrow(/test attempt finished/) + expect(resolveShadowProfiles).toHaveBeenCalledWith(capture[0].db, capture[1].db, plan, null, expect.any(Function)) + databases.forEach(db => expect(db.close).toHaveBeenCalledOnce()) +}) + +it('checks both captured database identities during profile resolution and closes them on rejection', async () => { + const { plan, installation, databases } = setup() + vi.mocked(resolveShadowProfiles).mockImplementation((_config, _bio, _plan, _revision, assertCurrent) => { + assertCurrent() + throw new Error('profile incompatible') + }) + await expect(openProfiledShadowHost(plan, installation, null)).rejects.toThrow('profile incompatible') + databases.forEach((db) => { + expect(db.assertCurrent).toHaveBeenCalledOnce() + expect(db.close).toHaveBeenCalledOnce() + }) + expect(openShadowHealthMonitor).not.toHaveBeenCalled() + expect(runShadowAttempt).not.toHaveBeenCalled() +}) + +it('unwinds profile capture if the biometrics database cannot be opened', async () => { + const { plan, installation, databases } = setup() + vi.mocked(openShadowActiveDatabase).mockReset().mockReturnValueOnce(databases[0] as unknown as ReturnType) + .mockImplementationOnce(() => { throw new Error('open failed') }) + await expect(openProfiledShadowHost(plan, installation, null)).rejects.toThrow('open failed') + expect(databases[0].close).toHaveBeenCalledOnce() + expect(resolveShadowProfiles).not.toHaveBeenCalled() +}) + +it('requires execution policy before opening either kind of host and honors pre-cancellation', async () => { + const { plan, installation } = setup() + const withoutExecution: Partial = { ...plan } + delete withoutExecution.execution + await expect(openPlannedShadowHost(withoutExecution, installation)).rejects.toThrow(/execution policy/) + await expect(openProfiledShadowHost(withoutExecution, installation, null)).rejects.toThrow(/execution policy/) + await expect(openProfiledShadowHost(plan, installation, null, AbortSignal.abort())).rejects.toThrow(/cancelled/) + expect(openShadowActiveDatabase).not.toHaveBeenCalled() +}) diff --git a/src/biometrics/shadow/tests/installed-attempt.test.ts b/src/biometrics/shadow/tests/installed-attempt.test.ts new file mode 100644 index 000000000..5f5b93204 --- /dev/null +++ b/src/biometrics/shadow/tests/installed-attempt.test.ts @@ -0,0 +1,120 @@ +import { withBiometricBackgroundSlot } from '../../background-admission' +// @vitest-environment node +import { beforeEach, expect, it, vi } from 'vitest' +import { dataHash } from '../../algorithms/canonical' +import { inspectInstalledShadowHost, loadInstalledShadowHost } from '../installed-host' +import { installedShadowIdentity } from '../device-identity' +import { openShadowStateDatabase } from '../active-database' +import { runInstalledShadowAttempt, reconcileInstalledShadowRuns, pendingInstalledShadowRun } from '../installed-attempt' +vi.mock('../installed-host', () => ({ inspectInstalledShadowHost: vi.fn(), loadInstalledShadowHost: vi.fn() })) +vi.mock('../device-identity', () => ({ installedShadowIdentity: vi.fn() })) +vi.mock('../active-database', () => ({ openShadowStateDatabase: vi.fn() })) +vi.mock('@/src/streaming/piezoStream', () => ({ getSensorFrameTimes: () => ({ 'piezo-dual': 1234 }) })) +const runId = '11111111-1111-4111-8111-111111111111' +const paths = { workerPath: '/trusted/shadow-host.cjs', pythonPath: '/trusted/python', modulesPath: '/trusted/modules', + configurationDatabase: '/trusted/config.db', biometricsDatabase: '/trusted/bio.db', presenceFile: '/trusted/presence' } +function setup() { + const entry = { qualificationId: 'test' } + const plan = { expectedBindings: { left: { generation: 1, resolvedHash: 'a'.repeat(64) }, right: { generation: 1, resolvedHash: 'a'.repeat(64) } }, candidateRevisionId: null as string | null | undefined, qualificationHash: dataHash(entry), execution: { sources: { databaseBusyTimeoutMs: 50 } } } + const get = vi.fn(() => ({ plan })), execute = vi.fn().mockResolvedValue('completed') + const registry = vi.fn(() => ({ entries: [entry] })), guard = vi.fn() + const recover = vi.fn(() => ({ scanned: 1, recovered: [], nextCursor: null })) + const pending = vi.fn(() => ({ run: { runId, revisionId: null }, nextCursor: runId })) + class Store { + pending = pending + get = get + reconcileExpired = recover + } + const inspected = { api: { ShadowStore: Store, runProfiledShadowAttempt: execute, readInstalledShadowQualifications: registry }, assertCurrent: guard } + vi.mocked(inspectInstalledShadowHost).mockResolvedValue(inspected as unknown as Awaited>) + vi.mocked(loadInstalledShadowHost).mockReturnValue(inspected as unknown as ReturnType) + const database = { db: {}, close: vi.fn(), assertCurrent: vi.fn() } + vi.mocked(openShadowStateDatabase).mockReturnValue(database as unknown as ReturnType) + const identity = vi.fn(() => ({ hardwareId: 'test', sampleRateHz: 500 as const, engineBuildHash: 'a'.repeat(64), pythonBuildHash: 'b'.repeat(64) })) + vi.mocked(installedShadowIdentity).mockReturnValue(identity) + return { plan, execute, registry, guard, database, identity, recover, pending } +} +beforeEach(() => vi.resetAllMocks()) +it('executes the loaded package with real frame-clock wiring and guarded state ownership', async () => { + const { execute, guard, database, identity } = setup() + execute.mockImplementation(async (_store, id, installation, revision) => { + expect(id).toBe(runId) + expect(revision).toBeNull() + expect(installation.piezoReceivedAt()).toBe(1234) + expect(installation.identity()).toMatchObject({ hardwareId: 'test', sampleRateHz: 500 }) + expect(database.close).not.toHaveBeenCalled() + return 'completed' + }) + expect(await runInstalledShadowAttempt(runId, null, paths, {}, 50)).toBe('completed') + expect(guard).toHaveBeenCalledOnce() + expect(database.assertCurrent).toHaveBeenCalledTimes(2) + expect(identity).toHaveBeenCalledTimes(2) + expect(database.close).toHaveBeenCalledOnce() +}) +it.each(['qualification', 'timeout', 'identity', 'execution'])('closes state and refuses invalid %s', async (mode) => { + const { registry, plan, identity, execute, database } = setup() + if (mode === 'qualification') registry.mockReturnValue({ entries: [] }) + if (mode === 'timeout') plan.execution.sources.databaseBusyTimeoutMs = 49 + if (mode === 'identity') identity.mockImplementation(() => { + throw new Error('identity unavailable') + }) + if (mode === 'execution') execute.mockRejectedValue(new Error('worker failed')) + await expect(runInstalledShadowAttempt(runId, null, paths, {}, 50)).rejects.toThrow() + expect(database.close).toHaveBeenCalledOnce() + if (mode !== 'execution') expect(execute).not.toHaveBeenCalled() +}) +it('does not open state after cancelled inspection', async () => { + setup() + const abort = new AbortController() + const pending = runInstalledShadowAttempt(runId, null, paths, {}, 50, abort.signal) + abort.abort() + await expect(pending).rejects.toThrow(/cancelled/) + expect(openShadowStateDatabase).not.toHaveBeenCalled() +}) + +it('reconciles through the installed store without starting probes, hardware checks or workers', () => { + const { recover, execute, database, guard } = setup() + expect(reconcileInstalledShadowRuns(paths, {}, 50, { maximumRuns: 10 })).toEqual({ scanned: 1, recovered: [], nextCursor: null }) + expect(recover).toHaveBeenCalledWith({ maximumRuns: 10 }) + expect(database.assertCurrent).toHaveBeenCalledTimes(2) + expect(guard).toHaveBeenCalledTimes(2) + expect(database.close).toHaveBeenCalledOnce() + expect(inspectInstalledShadowHost).not.toHaveBeenCalled() + expect(installedShadowIdentity).not.toHaveBeenCalled() + expect(execute).not.toHaveBeenCalled() +}) + +it.each([undefined, '22222222-2222-4222-8222-222222222222'])('refuses missing or substituted saved revision %s', async (revision) => { + const { plan, execute, database } = setup() + plan.candidateRevisionId = revision + await expect(runInstalledShadowAttempt(runId, null, paths, {}, 50)).rejects.toThrow('revision differs') + expect(execute).not.toHaveBeenCalled() + expect(database.close).toHaveBeenCalledOnce() +}) +it('reads pending identity from the installed store without worker inspection', () => { + const { pending, database } = setup() + expect(pendingInstalledShadowRun(paths, {}, 50, { maximumRuns: 10 })).toEqual({ run: { runId, revisionId: null }, nextCursor: runId }) + expect(pending).toHaveBeenCalledWith({ maximumRuns: 10 }) + expect(inspectInstalledShadowHost).not.toHaveBeenCalled() + expect(database.close).toHaveBeenCalledOnce() +}) + +it('defers before inspection or claim while coordinator work owns application capacity', async () => { + const { execute, database } = setup() + let finish!: () => void + const busy = withBiometricBackgroundSlot('work', () => new Promise((resolve) => { + finish = resolve + })) + try { + await expect(runInstalledShadowAttempt(runId, null, paths, {}, 50)).rejects.toMatchObject({ code: 'RESOURCE_LIMIT' }) + expect(inspectInstalledShadowHost).not.toHaveBeenCalled() + expect(openShadowStateDatabase).not.toHaveBeenCalled() + expect(execute).not.toHaveBeenCalled() + } + finally { + finish() + await busy + } + expect(await runInstalledShadowAttempt(runId, null, paths, {}, 50)).toBe('completed') + expect(database.close).toHaveBeenCalledOnce() +}) diff --git a/src/biometrics/shadow/tests/installed-host.test.ts b/src/biometrics/shadow/tests/installed-host.test.ts new file mode 100644 index 000000000..1b4d21c77 --- /dev/null +++ b/src/biometrics/shadow/tests/installed-host.test.ts @@ -0,0 +1,85 @@ +// @vitest-environment node +import { mkdtempSync, rmSync, writeFileSync, symlinkSync, renameSync } from 'node:fs' +import { createRequire } from 'node:module' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { createHash } from 'node:crypto' +import { readFileSync } from 'node:fs' +import { afterEach, expect, it, vi } from 'vitest' +import { inspectInstalledShadowHost, loadInstalledShadowHost } from '../installed-host' +import { guardShadowPythonInstallation } from '../python-installation' +vi.mock('../python-installation', () => ({ guardShadowPythonInstallation: vi.fn() })) +const directories: string[] = [] +const fixture = `module.exports = { shadowHostApiVersion: 'shadow-host-api-v1', runProfiledShadowAttempt() {}, openProfiledShadowHost() {}, prepareShadowPlan() {}, ShadowStore() {}, +shadowEngineBuildIdentity() {}, readInstalledShadowQualifications() {}, shadowPlanSchema: {}, shadowStoragePolicySchema: {} };` +function file(contents = fixture) { + const directory = mkdtempSync(join(tmpdir(), 'installed-shadow-')) + directories.push(directory) + const path = join(directory, 'shadow-host.cjs') + writeFileSync(path, contents) + return path +} +afterEach(() => { + delete (globalThis as typeof globalThis & { __sleepypodInstalledShadowHostV1?: unknown }).__sleepypodInstalledShadowHostV1 + directories.splice(0).forEach(path => rmSync(path, { recursive: true, force: true })) +}) +it.each(['symlink', 'oversize', 'directory', 'missing'])('rejects a %s artifact before executing it', (mode) => { + const path = file() + if (mode === 'oversize') writeFileSync(path, 'x'.repeat(16 * 1024 * 1024 + 1)) + if (mode === 'symlink') { + renameSync(path, path + '.target') + symlinkSync(path + '.target', path) + } + if (mode === 'missing') rmSync(path) + expect(() => loadInstalledShadowHost(mode === 'directory' ? directories[0] : path)).toThrow() +}) +it('refuses preloaded Node cache entries with no verified loading history', () => { + const path = file() + createRequire(import.meta.url)(path) + expect(() => loadInstalledShadowHost(path)).toThrow(/without installed identity/) +}) +it('refuses incomplete or unsupported API exports', () => { + const path = file('module.exports = { shadowHostApiVersion: \'other\' };') + expect(() => loadInstalledShadowHost(path)).toThrow(/API/) +}) +it('inspects the loaded artifact through its own build probe and guards the Python installation', async () => { + const probe = `module.exports = { shadowHostApiVersion: 'shadow-host-api-v1', runProfiledShadowAttempt() {}, openProfiledShadowHost() {}, prepareShadowPlan() {}, ShadowStore() {}, +async shadowEngineBuildIdentity(hostPath, location, signal) { return { identity: { hostHash: process.env.SHADOW_TEST_HOST_HASH, hostPath, location, cancelled: Boolean(signal?.aborted) }, python: { probed: true } } }, +readInstalledShadowQualifications() {}, shadowPlanSchema: {}, shadowStoragePolicySchema: {} };` + const path = file(probe) + const hostHash = createHash('sha256').update(readFileSync(path)).digest('hex') + const location = { pythonPath: '/opt/python', modulesPath: '/opt/modules' } + const aborted = new AbortController() + aborted.abort() + await expect(inspectInstalledShadowHost(path, location, aborted.signal)).rejects.toThrow('cancelled') + process.env.SHADOW_TEST_HOST_HASH = 'f'.repeat(64) + try { + await expect(inspectInstalledShadowHost(path, location)).rejects.toThrow('differs from inspected artifact') + expect(guardShadowPythonInstallation).not.toHaveBeenCalled() + process.env.SHADOW_TEST_HOST_HASH = hostHash + const pythonCurrent = vi.fn() + vi.mocked(guardShadowPythonInstallation).mockReturnValue(pythonCurrent) + const inspected = await inspectInstalledShadowHost(path, location) + expect(inspected.hostHash).toBe(hostHash) + expect(inspected.build).toMatchObject({ identity: { hostHash, location, cancelled: false }, python: { probed: true } }) + expect(guardShadowPythonInstallation).toHaveBeenCalledWith(location, { probed: true }) + expect(pythonCurrent).toHaveBeenCalledOnce() + inspected.assertCurrent() + expect(pythonCurrent).toHaveBeenCalledTimes(2) + pythonCurrent.mockImplementationOnce(() => { + throw new Error('Python changed') + }) + expect(() => inspected.assertCurrent()).toThrow('Python changed') + } + finally { + delete process.env.SHADOW_TEST_HOST_HASH + } +}) +it('rejects replacement with identical bytes and a second installation path', () => { + const path = file(), loaded = loadInstalledShadowHost(path) + loaded.assertCurrent() + expect(() => loadInstalledShadowHost(file())).toThrow(/restart/) + renameSync(path, path + '.old') + writeFileSync(path, fixture) + expect(() => loaded.assertCurrent()).toThrow(/restart/) +}) diff --git a/src/biometrics/shadow/tests/preparation.test.ts b/src/biometrics/shadow/tests/preparation.test.ts new file mode 100644 index 000000000..e7806636e --- /dev/null +++ b/src/biometrics/shadow/tests/preparation.test.ts @@ -0,0 +1,91 @@ +// @vitest-environment node +import type Database from 'better-sqlite3' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import { configHash, dataHash } from '../../algorithms/canonical' +import { defaultConfig, manifestHash } from '../../algorithms/registry' +import { captureShadowProfiles } from '../profiles' +import { readInstalledShadowQualifications } from '../admission' +import { prepareShadowPlan } from '../preparation' +import { configuredShadowTemplates, prepareInstalledShadow } from '../installed-preparation' +import { openShadowActiveDatabase } from '../active-database' +import { loadInstalledShadowHost } from '../installed-host' +import { submissionFixture, submissionTime } from './fixtures/submission' +vi.mock('../profiles', () => ({ captureShadowProfiles: vi.fn() })) +vi.mock('../admission', () => ({ readInstalledShadowQualifications: vi.fn() })) +vi.mock('../active-database', () => ({ openShadowActiveDatabase: vi.fn() })) +vi.mock('../installed-host', () => ({ loadInstalledShadowHost: vi.fn() })) + +const paths = { workerPath: '/installed/shadow-host.cjs', pythonPath: '/installed/python', modulesPath: '/installed/modules', + configurationDatabase: '/installed/config.db', biometricsDatabase: '/installed/bio.db', presenceFile: '/run/presence' } +const db = {} as Database.Database +function setup() { + const { plan } = submissionFixture(), hash = configHash(defaultConfig) + plan.execution.configHashes = { left: hash, right: hash } + const entry = { manifestHash, executionPolicyHash: dataHash(plan.execution), activeConfigHash: hash, engineBuildHash: 'b'.repeat(64) } + const template = { id: 'test', label: 'Synthetic fixture', maximumWindowMs: 9000, qualificationHash: dataHash(entry), + execution: plan.execution, joining: plan.joining, monitoring: plan.monitoring, metrics: plan.metrics, + maximumPoints: plan.maximumPoints, maximumBytes: plan.maximumBytes } + const request = { templateId: 'test', side: 'left' as const, subjectId: 'sleeper', candidateRevisionId: null, startMs: submissionTime + 1000, endMs: submissionTime + 9000 } + const captured = { revisionId: null, configs: { left: defaultConfig, right: defaultConfig }, + expectedBindings: { left: { generation: 4, resolvedHash: hash }, right: { generation: 5, resolvedHash: hash } }, heartbeatAtMs: submissionTime, + candidateConfigHash: hash, activeConfigHash: hash } + vi.mocked(captureShadowProfiles).mockReturnValue(captured) + vi.mocked(readInstalledShadowQualifications).mockReturnValue({ entries: [entry] } as unknown as ReturnType) + const config = { version: 'shadow-service-configuration-v1', pythonPath: paths.pythonPath, modulesPath: paths.modulesPath, presenceFile: paths.presenceFile, + storagePolicy: { maximumRuns: 10, maximumPoints: 100, maximumBytes: 100000 }, busyTimeoutMs: 50, recoveryIntervalMs: 1000, recoveryMaximumRuns: 10, + planTemplates: [template] } + vi.stubEnv('SLEEPYPOD_BIOMETRICS_SHADOW_SERVICE', JSON.stringify(config)) + return { template, request, entry, captured, config } +} +beforeEach(() => vi.resetAllMocks()) +afterEach(() => vi.unstubAllEnvs()) + +it('derives candidate/baseline identities and generations from server capture without admitting execution', () => { + const { request, template, captured, entry } = setup(), guard = vi.fn() + const prepared = prepareShadowPlan(db, db, request, template, paths.workerPath, guard, submissionTime) + expect(prepared).toMatchObject({ status: 'prepared_not_admitted', plan: { candidateRevisionId: null, + expectedBindings: captured.expectedBindings, candidateConfigHash: captured.candidateConfigHash, activeConfigHash: captured.activeConfigHash, engineBuildHash: entry.engineBuildHash } }) + expect(prepared.planHash).toBe(dataHash(prepared.plan)) + expect(captureShadowProfiles).toHaveBeenCalledWith(db, db, 'left', null, guard, submissionTime) +}) + +it.each(['past', 'duration', 'wall', 'qualification', 'candidate', 'baseline'])('refuses incompatible preparation: %s', (failure) => { + const { request, template, captured } = setup() + if (failure === 'past') request.startMs = submissionTime - 1 + if (failure === 'duration') request.endMs += 100000 + if (failure === 'wall') template.execution.budgets.wallMs = 1000 + if (failure === 'qualification') template.qualificationHash = 'f'.repeat(64) + if (failure === 'candidate') captured.configs.left = { ...defaultConfig, version: 'invalid' } as unknown as typeof defaultConfig + if (failure === 'baseline') captured.activeConfigHash = 'f'.repeat(64) + expect(() => prepareShadowPlan(db, db, request, template, paths.workerPath, () => {}, submissionTime)).toThrow() +}) + +it('lists only template metadata and owns readonly preparation handles on success and failure', () => { + const { request, template } = setup() + expect(configuredShadowTemplates()).toEqual({ items: [{ id: 'test', label: 'Synthetic fixture', maximumWindowMs: 9000 }] }) + const handles = [0, 1].map(() => ({ db, close: vi.fn(), assertCurrent: vi.fn() })) + const prepare = vi.fn(() => ({ status: 'prepared_not_admitted' })), guard = vi.fn() + vi.mocked(loadInstalledShadowHost).mockReturnValue({ api: { prepareShadowPlan: prepare }, assertCurrent: guard } as unknown as ReturnType) + for (const failed of [false, true]) { + vi.mocked(openShadowActiveDatabase).mockReturnValueOnce(handles[0]).mockReturnValueOnce(handles[1]) + if (failed) prepare.mockImplementationOnce(() => { + throw new Error('changed') + }) + if (failed) expect(() => prepareInstalledShadow(request, paths, 50)).toThrow('changed') + else expect(prepareInstalledShadow(request, paths, 50)).toEqual({ status: 'prepared_not_admitted' }) + } + expect(prepare).toHaveBeenCalledWith(db, db, request, template, paths.workerPath, expect.any(Function)) + handles.forEach(handle => expect(handle.close).toHaveBeenCalledTimes(2)) +}) + +it('refuses templates that exceed deployment quotas or duplicate an ID', () => { + const { config } = setup() + config.storagePolicy.maximumBytes = 1 + vi.stubEnv('SLEEPYPOD_BIOMETRICS_SHADOW_SERVICE', JSON.stringify(config)) + expect(configuredShadowTemplates).toThrow('storage policy') + config.storagePolicy.maximumBytes = 100000 + config.planTemplates.push(config.planTemplates[0]) + vi.stubEnv('SLEEPYPOD_BIOMETRICS_SHADOW_SERVICE', JSON.stringify(config)) + expect(configuredShadowTemplates).toThrow('Duplicate') + expect(openShadowActiveDatabase).not.toHaveBeenCalled() +}) diff --git a/src/biometrics/shadow/tests/process-cleanup.test.ts b/src/biometrics/shadow/tests/process-cleanup.test.ts new file mode 100644 index 000000000..b87ea6a31 --- /dev/null +++ b/src/biometrics/shadow/tests/process-cleanup.test.ts @@ -0,0 +1,78 @@ +// @vitest-environment node +import { afterEach, expect, it, vi } from 'vitest' +import { confirmShadowProcessCleanup } from '../process-cleanup' + +afterEach(() => { + vi.restoreAllMocks() + vi.useRealTimers() +}) +const missing = () => { + throw Object.assign(new Error('missing'), { code: 'ESRCH' }) +} +it('requires disappearance of descendants even after the direct child closes', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'performance'] }) + const kill = vi.spyOn(process, 'kill').mockReturnValue(true) + let settled = false + const result = confirmShadowProcessCleanup(123, () => true).then((value) => { + settled = true + return value + }) + await vi.advanceTimersByTimeAsync(100) + expect(settled).toBe(false) + kill.mockImplementation(missing) + await vi.advanceTimersByTimeAsync(25) + expect(await result).toBe(true) + expect(kill.mock.calls.every(args => args[0] === -123 && args[1] === 0)).toBe(true) +}) +it('requires the close event even after the group disappears', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'performance'] }) + vi.spyOn(process, 'kill').mockImplementation(missing) + let closed = false, settled = false + const result = confirmShadowProcessCleanup(123, () => closed).then((value) => { + settled = true + return value + }) + await vi.advanceTimersByTimeAsync(100) + expect(settled).toBe(false) + closed = true + await vi.advanceTimersByTimeAsync(25) + expect(await result).toBe(true) +}) +it('bounds confirmation when a group still exists and when close is missing', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'performance'] }) + const kill = vi.spyOn(process, 'kill').mockReturnValue(true) + const alive = confirmShadowProcessCleanup(123, () => true) + await vi.advanceTimersByTimeAsync(5000) + expect(await alive).toBe(false) + kill.mockImplementation(missing) + const unclosed = confirmShadowProcessCleanup(123, () => false) + await vi.advanceTimersByTimeAsync(5000) + expect(await unclosed).toBe(false) + expect(vi.getTimerCount()).toBe(0) +}) +it('treats probe denial as unknown and never signals invalid or missing pids', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'performance'] }) + const kill = vi.spyOn(process, 'kill').mockImplementation(() => { + throw Object.assign(new Error('denied'), { code: 'EPERM' }) + }) + const denied = confirmShadowProcessCleanup(123, () => true) + await vi.advanceTimersByTimeAsync(5000) + expect(await denied).toBe(false) + kill.mockClear() + for (const pid of [0, -1, NaN, Infinity, 1.5]) expect(await confirmShadowProcessCleanup(pid, () => true)).toBe(false) + expect(await confirmShadowProcessCleanup(undefined, () => true)).toBe(true) + expect(kill).not.toHaveBeenCalled() +}) + +it('retries transient unknown observations but still requires explicit absence', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'performance'] }) + const kill = vi.spyOn(process, 'kill').mockImplementationOnce(() => { + throw Object.assign(new Error('reaping'), { code: 'EPERM' }) + }).mockImplementation(missing) + const result = confirmShadowProcessCleanup(123, () => true) + expect(kill).toHaveBeenCalledOnce() + await vi.advanceTimersByTimeAsync(25) + expect(await result).toBe(true) + expect(kill).toHaveBeenCalledTimes(2) + expect(vi.getTimerCount()).toBe(0) +}) diff --git a/src/biometrics/shadow/tests/python-installation.test.ts b/src/biometrics/shadow/tests/python-installation.test.ts new file mode 100644 index 000000000..85c9b0200 --- /dev/null +++ b/src/biometrics/shadow/tests/python-installation.test.ts @@ -0,0 +1,61 @@ +// @vitest-environment node +import { copyFileSync, mkdirSync, mkdtempSync, rmSync, writeFileSync, renameSync, symlinkSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { dirname, join, resolve } from 'node:path' +import { afterEach, expect, it } from 'vitest' +import { probeShadowBuild } from '../build' +import { guardShadowPythonInstallation } from '../python-installation' +const directories: string[] = [] +afterEach(() => directories.splice(0).forEach(path => rmSync(path, { recursive: true, force: true }))) +async function setup() { + const original = { pythonPath: resolve('modules/piezo-processor/.venv/bin/python'), modulesPath: resolve('modules') } + const build = await probeShadowBuild(original) + const root = mkdtempSync(join(tmpdir(), 'python-installation-')) + directories.push(root) + for (const relative of Object.keys(build.identity.sources)) { + const path = join(root, relative) + mkdirSync(dirname(path), { recursive: true }) + copyFileSync(join(original.modulesPath, relative), path) + } + for (const name of ['numpy', 'scipy'] as const) { + for (const kind of ['record', 'wheel'] as const) { + const path = join(root, `${name}-${kind}`) + copyFileSync(build.dependencyFiles[name][kind], path) + build.dependencyFiles[name][kind] = path + } + } + const pythonPath = join(root, 'python') + symlinkSync(original.pythonPath, pythonPath) + const location = { pythonPath, modulesPath: root } + return { build, location, source: join(root, 'common/biometrics/shadow.py') } +} +it('validates real source and wheel metadata against the probe, then detects source replacement', async () => { + const { build, location, source } = await setup() + const guard = guardShadowPythonInstallation(location, build) + guard() + renameSync(source, source + '.old') + copyFileSync(source + '.old', source) + expect(guard).toThrow(/changed/) +}) +it('refuses source content changed after probing', async () => { + const { build, location, source } = await setup() + writeFileSync(source, '# changed') + expect(() => guardShadowPythonInstallation(location, build)).toThrow(/differs from probe/) +}) +it('detects metadata mutation and executable link replacement', async () => { + const { build, location } = await setup() + const guard = guardShadowPythonInstallation(location, build) + writeFileSync(build.dependencyFiles.numpy.wheel, 'changed') + expect(guard).toThrow(/changed/) + const other = await setup() + const executableGuard = guardShadowPythonInstallation(other.location, other.build) + rmSync(other.location.pythonPath) + symlinkSync(process.execPath, other.location.pythonPath) + expect(executableGuard).toThrow(/changed/) +}) +it('refuses malformed build hashes and non-absolute dependency paths', async () => { + const { build, location } = await setup() + expect(() => guardShadowPythonInstallation(location, { ...build, hash: 'a'.repeat(64) })).toThrow(/integrity/) + build.dependencyFiles.numpy.record = 'relative/RECORD' + expect(() => guardShadowPythonInstallation(location, build)).toThrow(/absolute/) +}) diff --git a/src/biometrics/shadow/tests/runtime-binding.test.ts b/src/biometrics/shadow/tests/runtime-binding.test.ts new file mode 100644 index 000000000..46fbb191f --- /dev/null +++ b/src/biometrics/shadow/tests/runtime-binding.test.ts @@ -0,0 +1,161 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, expect, it } from 'vitest' +import * as configSchema from '../../../db/schema' +import * as bioSchema from '../../../db/biometrics-schema' +import { configHash } from '../../algorithms/canonical' +import { defaultConfig } from '../../algorithms/registry' +import { requestManualActivation } from '../../profiles/activation' +import { deliverRuntimeOutbox } from '../../profiles/runtime' +import { resolveShadowProfiles } from '../profiles' +import { ProfileStore } from '../../profiles/store' +import { shadowRuntimeBindingReader } from '../runtime-binding' +const at = 1700000000000, hash = configHash(defaultConfig) +const cleanups: (() => void)[] = [] +afterEach(() => cleanups.splice(0).forEach(fn => fn())) +function setup() { + const root = mkdtempSync(join(tmpdir(), 'shadow-binding-')) + const config = new Database(join(root, 'config.db')), bio = new Database(join(root, 'bio.db')) + const configDb = drizzle(config, { schema: configSchema }), bioDb = drizzle(bio, { schema: bioSchema }) + migrate(configDb, { migrationsFolder: 'src/db/migrations' }) + migrate(bioDb, { migrationsFolder: 'src/db/biometrics-migrations' }) + for (const side of ['left', 'right'] as const) { + requestManualActivation(configDb, { side, action: 'reset', revisionId: null, expectedGeneration: 0, + expectedResolvedHash: hash, boundary: 'immediate', requester: 'fixture', idempotencyKey: side, allowUnvalidated: true }, at - 2000) + } + deliverRuntimeOutbox(configDb, bioDb, at - 1000) + for (const side of ['left', 'right'] as const) { + for (const component of ['heart_rate', 'breathing_rate', 'hrv_index']) { + bioDb.insert(bioSchema.biometricRuntimeReceipts).values({ side, component, generation: 1, appliedHash: hash, + effectiveSampleTimeMs: at - 1000, heartbeatAtMs: at, state: 'applied' }).run() + } + } + const configReader = new Database(join(root, 'config.db'), { readonly: true }), bioReader = new Database(join(root, 'bio.db'), { readonly: true }) + cleanups.push(() => { + configReader.close() + bioReader.close() + config.close() + bio.close() + rmSync(root, { recursive: true, force: true }) + }) + const options = { side: 'left' as const, expected: { left: { generation: 1, resolvedHash: hash }, right: { generation: 1, resolvedHash: hash } }, + assertCurrent: () => {}, now: () => at } + return { configDb, config, bio, configReader, bioReader, options, read: shadowRuntimeBindingReader(configReader, bioReader, options) } +} +it('aggregates both channels without requiring an on-demand stage receipt', () => { + const { read, bio, options } = setup() + bio.prepare('UPDATE biometric_runtime_receipts SET heartbeat_at_ms = ? WHERE side = \'right\' AND component = \'hrv_index\'').run(at - 500) + options.expected.right.generation = 99 // caller mutation cannot change captured identity + expect(read()).toEqual({ side: 'left', generation: 1, resolvedHash: hash, appliedGeneration: 1, appliedHash: hash, state: 'applied', heartbeatAtMs: at - 500 }) +}) +it.each(['left', 'right'])('rejects same-hash desired resets on %s', (side) => { + const { config, read } = setup() + config.prepare('UPDATE biometric_profile_bindings SET generation = 2 WHERE side = ?').run(side) + expect(read).toThrow(/generation changed/) +}) +it.each([ + 'DELETE FROM biometric_runtime_receipts WHERE side = \'right\' AND component = \'breathing_rate\'', + 'UPDATE biometric_runtime_receipts SET state = \'pending_boundary\' WHERE component = \'hrv_index\'', + 'UPDATE biometric_runtime_receipts SET state = \'failed\' WHERE component = \'heart_rate\'', + 'UPDATE biometric_runtime_receipts SET effective_sample_time_ms = NULL WHERE component = \'hrv_index\'', + 'UPDATE biometric_runtime_receipts SET generation = 2 WHERE side = \'right\'', + 'UPDATE biometric_runtime_receipts SET applied_hash = \'wrong\' WHERE side = \'right\'', + `UPDATE biometric_runtime_receipts SET heartbeat_at_ms = ${at + 1} WHERE side = 'right'`, + `UPDATE biometric_runtime_receipts SET effective_sample_time_ms = ${at + 1} WHERE side = 'right'`, +])('rejects missing, unapplied, changed or future component evidence: %s', (sql) => { + const { bio, read } = setup() + bio.exec(sql) + expect(read).toThrow() +}) +it.each(['DELETE FROM biometric_runtime_configs WHERE side = \'right\'', + 'UPDATE biometric_runtime_configs SET resolved_hash = \'wrong\' WHERE side = \'right\'', + 'UPDATE biometric_runtime_configs SET payload = \'{}\' WHERE side = \'right\'', + 'UPDATE biometric_runtime_configs SET payload = printf(\'%070000d\', 0) WHERE side = \'right\''])('rejects unavailable or corrupt installed projection: %s', (sql) => { + const { bio, read } = setup() + bio.exec(sql) + expect(read).toThrow() +}) +it('requires read-only handles and propagates post-snapshot identity failure', () => { + const { config, bio, configReader, bioReader, options } = setup() + expect(() => shadowRuntimeBindingReader(config, bio, options)).toThrow(/read-only/) + let checks = 0 + const read = shadowRuntimeBindingReader(configReader, bioReader, { ...options, assertCurrent: () => { + if (++checks === 3) throw new Error('database replaced') + } }) + expect(read).toThrow(/database replaced/) +}) +it('refuses desired hash changes even when all old receipts remain applied', () => { + const { config, read } = setup() + config.prepare('UPDATE biometric_activation_events SET next_hash = \'changed\' WHERE side = \'right\'').run() + expect(read).toThrow(/configuration changed/) +}) +it('does not fall back to an older applied generation behind a pending receipt', () => { + const { bio, read } = setup() + bio.prepare(`INSERT INTO biometric_runtime_receipts + (side, component, generation, applied_hash, effective_sample_time_ms, heartbeat_at_ms, state) + VALUES ('right', 'heart_rate', 2, ?, NULL, ?, 'pending_boundary')`).run(hash, at) + expect(read).toThrow() +}) + +function savedPlan() { + const execution = { version: 'shadow-execution-policy-v1', configHashes: { left: hash, right: hash }, + maximumRecords: 10, maximumOutputs: 10, maximumCpuMs: 1000, maximumInputBytes: 100000, maximumOutputBytes: 100000, + budgets: { cpuSeconds: 2, addressSpaceBytes: 100000000, maximumRssBytes: 100000000, wallMs: 10000, acknowledgmentMs: 1000, idleMs: 1000, rssIntervalMs: 100, rssTimeoutMs: 100 }, + leaseMs: 1500, heartbeatMs: 200, sources: { pollMs: 50, maximumActiveRows: 10, databaseBusyTimeoutMs: 50, + frames: { maximumFrames: 10, maximumBytes: 100000, maximumEventAgeMs: 1000, maximumFutureSkewMs: 0 }, + context: { presenceStaleMs: 300000, pumpStaleMs: 5000, maximumPresenceBytes: 4096 } } } + const plan = { schema: 'biometric-shadow-plan-v1', side: 'left', subjectId: 'test', execution, startMs: at, endMs: at + 10000, + candidateConfigHash: hash, activeConfigHash: hash, engineBuildHash: hash, qualificationHash: 'a'.repeat(64), maximumPoints: 10, maximumBytes: 10000, metrics: ['heart_rate'], + joining: { maximumPendingPoints: 10, maximumLatenessMs: 100, drainGraceMs: 500 }, + monitoring: { expectedIntervalMs: 1000, maximumGapMs: 2000, heartbeatStaleMs: 1000, minimumCandidateCoverage: 0.8, maximumCoverageLoss: 0.1, minimumQuality: 0.8, + alignment: { version: 'reference-alignment-v1', pointToleranceMs: 0, clockOffsetMs: 0, offsetSource: 'fixed' } } } + return plan +} + +it('resolves the saved candidate while preserving acknowledged other-channel config', () => { + const { configDb, configReader, bioReader } = setup() + const revision = new ProfileStore(configDb).createRevision({ side: 'left', overrides: [{ path: 'heart_rate.trackingToleranceBpm', value: 20 }], + origin: 'manual', requester: 'test', idempotencyKey: 'candidate' }) + const plan = savedPlan() + plan.candidateConfigHash = revision.resolvedHash + plan.execution.configHashes.left = revision.resolvedHash + const result = resolveShadowProfiles(configReader, bioReader, plan, revision.id, () => {}, at) + expect(result.configs.left).toEqual(revision.resolved) + expect(result.configs.right).toEqual(defaultConfig) + expect(result.expectedBindings).toEqual({ left: { generation: 1, resolvedHash: hash }, right: { generation: 1, resolvedHash: hash } }) + expect(result.revisionId).toBe(revision.id) +}) +it('allows explicit default candidates but refuses unstored active defaults', () => { + const { config, configReader, bioReader } = setup() + expect(resolveShadowProfiles(configReader, bioReader, savedPlan(), null, () => {}, at).configs.left).toEqual(defaultConfig) + config.prepare('DELETE FROM biometric_profile_bindings WHERE side = \'right\'').run() + expect(() => resolveShadowProfiles(configReader, bioReader, savedPlan(), null, () => {}, at)).toThrow() +}) +it('refuses a same-hash active generation change from a queued plan', () => { + const { configReader, bioReader } = setup() + const expectedBindings = { left: { generation: 1, resolvedHash: hash }, right: { generation: 1, resolvedHash: hash } } + expect(resolveShadowProfiles(configReader, bioReader, { ...savedPlan(), expectedBindings }, null, () => {}, at).expectedBindings).toEqual(expectedBindings) + expectedBindings.right.generation = 2 + expect(() => resolveShadowProfiles(configReader, bioReader, { ...savedPlan(), expectedBindings }, null, () => {}, at)).toThrow('generations differ') +}) +it('rejects wrong-side or corrupted revisions and frozen physical-channel substitution', () => { + const { configDb, config, configReader, bioReader } = setup() + const revision = new ProfileStore(configDb).createRevision({ side: 'right', overrides: [], origin: 'manual', requester: 'test', idempotencyKey: 'candidate' }) + const plan = savedPlan() + expect(() => resolveShadowProfiles(configReader, bioReader, plan, revision.id, () => {}, at)).toThrow(/other side/) + config.prepare('UPDATE biometric_profile_revisions SET side = \'left\', resolved_hash = \'corrupt\' WHERE id = ?').run(revision.id) + expect(() => resolveShadowProfiles(configReader, bioReader, plan, revision.id, () => {}, at)).toThrow(/integrity/) + plan.execution.configHashes.right = 'b'.repeat(64) + expect(() => resolveShadowProfiles(configReader, bioReader, plan, null, () => {}, at)).toThrow(/physical-channel/) +}) +it('refuses unapplied baselines and writable profile resolution handles', () => { + const { config, bio, configReader, bioReader } = setup() + expect(() => resolveShadowProfiles(config, bio, savedPlan(), null, () => {}, at)).toThrow(/read-only/) + bio.prepare('UPDATE biometric_runtime_receipts SET state = \'pending_boundary\' WHERE side = \'right\'').run() + expect(() => resolveShadowProfiles(configReader, bioReader, savedPlan(), null, () => {}, at)).toThrow() +}) diff --git a/src/biometrics/shadow/tests/service-host.test.ts b/src/biometrics/shadow/tests/service-host.test.ts new file mode 100644 index 000000000..b83ea3df7 --- /dev/null +++ b/src/biometrics/shadow/tests/service-host.test.ts @@ -0,0 +1,55 @@ +// @vitest-environment node +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import { runInstalledShadowAttempt, reconcileInstalledShadowRuns, pendingInstalledShadowRun } from '../installed-attempt' +import { startShadowService, stopShadowService } from '../service-host' + +vi.mock('../installed-attempt', () => ({ runInstalledShadowAttempt: vi.fn(), reconcileInstalledShadowRuns: vi.fn(), pendingInstalledShadowRun: vi.fn() })) +const runId = '11111111-1111-4111-8111-111111111111' +const options = () => ({ installation: { workerPath: '/trusted/shadow-host.cjs', pythonPath: '/trusted/python', modulesPath: '/trusted/modules', + configurationDatabase: '/trusted/config.db', biometricsDatabase: '/trusted/bio.db', presenceFile: '/trusted/presence' }, +storagePolicy: { maximumRuns: 10, maximumPoints: 1000, maximumBytes: 1000000 }, busyTimeoutMs: 50, recoveryIntervalMs: 1000, recoveryMaximumRuns: 10 }) +beforeEach(() => { + vi.mocked(pendingInstalledShadowRun).mockReturnValue({ run: null, nextCursor: null }) +}) +afterEach(async () => { + await stopShadowService() + vi.resetAllMocks() +}) + +it('owns one global service, freezes deployment inputs and rejects configuration drift', async () => { + vi.mocked(reconcileInstalledShadowRuns).mockReturnValue({ scanned: 0, recovered: [], nextCursor: null }) + const raw = options(), original = structuredClone(raw) + const service = startShadowService(raw) + expect(startShadowService(options())).toBe(service) + expect(reconcileInstalledShadowRuns).toHaveBeenCalledOnce() + raw.installation.workerPath = '/different' + raw.storagePolicy.maximumRuns = 999 + expect(() => startShadowService(raw)).toThrow('restart required') + await service.run(runId, null) + expect(runInstalledShadowAttempt).toHaveBeenCalledWith(runId, null, original.installation, + original.storagePolicy, 50, expect.any(AbortSignal)) + await stopShadowService() + expect(() => service.run(runId, null)).toThrow('stopped') + expect(startShadowService(options())).not.toBe(service) +}) + +it('keeps the singleton until shutdown finishes worker cleanup', async () => { + vi.mocked(reconcileInstalledShadowRuns).mockReturnValue({ scanned: 0, recovered: [], nextCursor: null }) + let release!: () => void + vi.mocked(runInstalledShadowAttempt).mockImplementation(async (_id, _revision, _paths, _policy, _timeout, signal) => { + await new Promise((resolve) => { + release = resolve + }) + expect(signal?.aborted).toBe(true) + throw new Error('interrupted') + }) + const service = startShadowService(options()) + const failed = expect(service.run(runId, null)).rejects.toThrow('interrupted') + await Promise.resolve() + const stopping = stopShadowService() + expect(startShadowService(options())).toBe(service) + expect(() => service.run(runId, null)).toThrow('stopped') + release() + await Promise.all([stopping, failed]) + expect(startShadowService(options())).not.toBe(service) +}) diff --git a/src/biometrics/shadow/tests/service.test.ts b/src/biometrics/shadow/tests/service.test.ts new file mode 100644 index 000000000..c62f04d41 --- /dev/null +++ b/src/biometrics/shadow/tests/service.test.ts @@ -0,0 +1,176 @@ +// @vitest-environment node +import { afterEach, expect, it, vi } from 'vitest' +import { ShadowService } from '../service' + +const runId = '11111111-1111-4111-8111-111111111111' +const otherId = '22222222-2222-4222-8222-222222222222' +function setup() { + const execute = vi.fn[0]['execute']>().mockResolvedValue('complete') + const recover = vi.fn(() => ({ scanned: 0, recovered: [], nextCursor: null as string | null })) + const onRecoveryError = vi.fn() + return { execute, recover, onRecoveryError, service: new ShadowService({ execute, recover, onRecoveryError, + recoveryIntervalMs: 100, recoveryMaximumRuns: 7 }) } +} +afterEach(() => vi.useRealTimers()) + +it('schedules saved pending identity while maintenance continues during worker cleanup', async () => { + vi.useFakeTimers() + const { execute, recover } = setup() + const pending = vi.fn().mockReturnValueOnce({ run: { runId, revisionId: otherId }, nextCursor: runId }) + .mockReturnValue({ run: null, nextCursor: null }) + let release!: () => void + execute.mockImplementationOnce(async () => { + await new Promise((resolve) => { + release = resolve + }) + }) + const service = new ShadowService({ execute, recover, pending, recoveryIntervalMs: 100, recoveryMaximumRuns: 7 }) + service.start() + await Promise.resolve() + expect(execute).toHaveBeenCalledWith(runId, otherId, expect.any(AbortSignal)) + await vi.advanceTimersByTimeAsync(300) + expect(recover).toHaveBeenCalledTimes(4) + expect(pending).toHaveBeenCalledOnce() + release() + await vi.advanceTimersByTimeAsync(100) + expect(pending).toHaveBeenLastCalledWith({ maximumRuns: 7, afterId: runId }) + await service.stop() +}) + +it('retries failed queue scans and observes scheduled execution errors', async () => { + vi.useFakeTimers() + const { execute, recover } = setup() + const pending = vi.fn().mockImplementationOnce(() => { + throw new Error('database busy') + }).mockReturnValueOnce({ run: { runId, revisionId: null }, nextCursor: runId }) + .mockReturnValue({ run: null, nextCursor: null }) + const onExecutionError = vi.fn(() => { + throw new Error('logging failed') + }) + execute.mockRejectedValueOnce(new Error('admission refused')) + const service = new ShadowService({ execute, recover, pending, onExecutionError, recoveryIntervalMs: 100, recoveryMaximumRuns: 7 }) + service.start() + await vi.advanceTimersByTimeAsync(100) + expect(pending.mock.calls.slice(0, 2)).toEqual([[{ maximumRuns: 7 }], [{ maximumRuns: 7 }]]) + expect(onExecutionError).toHaveBeenCalledTimes(2) + await vi.advanceTimersByTimeAsync(100) + expect(pending).toHaveBeenLastCalledWith({ maximumRuns: 7, afterId: runId }) + await service.stop() +}) + +it('advances bounded sweeps, retains the cursor on failure and stops maintenance', async () => { + vi.useFakeTimers() + const { service, recover, onRecoveryError } = setup() + recover.mockReturnValueOnce({ scanned: 7, recovered: [], nextCursor: runId }) + service.start() + service.start() + expect(recover).toHaveBeenCalledTimes(1) + recover.mockImplementationOnce(() => { + throw new Error('busy') + }) + onRecoveryError.mockImplementationOnce(() => { + throw new Error('logger failed') + }) + await vi.advanceTimersByTimeAsync(100) + expect(onRecoveryError).toHaveBeenCalledOnce() + await vi.advanceTimersByTimeAsync(100) + expect(recover.mock.calls.slice(1)).toEqual([[{ maximumRuns: 7, afterId: runId }], [{ maximumRuns: 7, afterId: runId }]]) + await vi.advanceTimersByTimeAsync(100) + expect(recover).toHaveBeenLastCalledWith({ maximumRuns: 7 }) + await service.stop() + service.start() + await vi.advanceTimersByTimeAsync(1000) + expect(recover).toHaveBeenCalledTimes(4) +}) + +it('retains the execution slot through cancellation and asynchronous cleanup', async () => { + const { service, execute } = setup() + let release!: () => void + let signal!: AbortSignal + execute.mockImplementation(async (_id, _revision, received) => { + signal = received + await new Promise((resolve) => { + release = resolve + }) + throw new Error('cancelled after cleanup') + }) + const run = service.run(runId, otherId) + const failure = expect(run).rejects.toThrow('cancelled after cleanup') + await Promise.resolve() + expect(execute).toHaveBeenCalledWith(runId, otherId, signal) + expect(await service.cancel(otherId)).toBe(false) + expect(signal.aborted).toBe(false) + let cancelled = false + const cancellation = service.cancel(runId).then((result) => { + cancelled = result + }) + expect(signal.aborted).toBe(true) + expect(() => service.run(otherId, null)).toThrow('busy') + await Promise.resolve() + expect(cancelled).toBe(false) + release() + await Promise.all([failure, cancellation]) + expect(cancelled).toBe(true) + execute.mockResolvedValueOnce('next') + expect(await service.run(otherId, null)).toBe('next') + await service.stop() +}) + +it('shutdown aborts, waits for cleanup, rejects new work and is repeatable', async () => { + const { service, execute } = setup() + let release!: () => void + execute.mockImplementation(async (_id, _revision, signal) => { + await new Promise((resolve) => { + release = resolve + }) + expect(signal.aborted).toBe(true) + }) + const run = service.run(runId, null) + await Promise.resolve() + let stopped = false + const stop = service.stop().then(() => { + stopped = true + }) + await Promise.resolve() + expect(stopped).toBe(false) + expect(() => service.run(otherId, null)).toThrow('stopped') + release() + await Promise.all([stop, run, service.stop()]) + expect(stopped).toBe(true) +}) + +it('cancels before starting a deferred worker and releases synchronous failures', async () => { + const { service, execute } = setup() + const run = service.run(runId, null) + const failed = expect(run).rejects.toThrow('cancelled') + await service.cancel(runId) + await failed + expect(execute).not.toHaveBeenCalled() + execute.mockImplementationOnce(() => { + throw new Error('startup failed') + }) + await expect(service.run(runId, null)).rejects.toThrow('startup failed') + expect(await service.run(otherId, null)).toBe('complete') + await service.stop() +}) + +it('quarantines scheduling after unconfirmed cleanup while maintenance continues', async () => { + vi.useFakeTimers() + const { execute, recover } = setup() + const pending = vi.fn(() => ({ run: { runId, revisionId: null }, nextCursor: runId })) + let reject!: (error: Error) => void + execute.mockImplementation(() => new Promise((_resolve, fail) => { + reject = fail + })) + const service = new ShadowService({ execute, recover, pending, recoveryIntervalMs: 100, recoveryMaximumRuns: 7 }) + service.start() + await Promise.resolve() + const cancelled = service.cancel(runId) + reject(Object.assign(new Error('shadow_worker_cleanup_unconfirmed'), { code: 'SHADOW_CLEANUP_UNCONFIRMED' })) + expect(await cancelled).toBe(false) + await vi.advanceTimersByTimeAsync(300) + expect(pending).toHaveBeenCalledOnce() + expect(recover).toHaveBeenCalledTimes(4) + expect(() => service.run(otherId, null)).toThrow('cleanup unconfirmed') + await service.stop() +}) diff --git a/src/biometrics/shadow/tests/startup.test.ts b/src/biometrics/shadow/tests/startup.test.ts new file mode 100644 index 000000000..8f5d7bd85 --- /dev/null +++ b/src/biometrics/shadow/tests/startup.test.ts @@ -0,0 +1,88 @@ +// @vitest-environment node +import { mkdtempSync, rmSync, realpathSync } from 'node:fs' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import * as configSchema from '@/src/db/schema' +import * as biometricsSchema from '@/src/db/biometrics-schema' +import { packagedShadowHostPath } from '../../backtest/location' +import { startShadowService } from '../service-host' +import { configuredShadowService, startConfiguredShadowService } from '../startup' + +vi.mock('../../backtest/location', () => ({ packagedShadowHostPath: vi.fn(() => '/installed/shadow-host.cjs') })) +vi.mock('../service-host', () => ({ startShadowService: vi.fn() })) +const cleanup: (() => void)[] = [] +const policy = () => ({ version: 'shadow-service-configuration-v1', pythonPath: '/installed/python', modulesPath: '/installed/modules', presenceFile: '/run/presence.json', + storagePolicy: { maximumRuns: 10, maximumPoints: 1000, maximumBytes: 1000000 }, busyTimeoutMs: 50, recoveryIntervalMs: 1000, recoveryMaximumRuns: 10 }) +function databases(memory = false) { + const directory = realpathSync(mkdtempSync(join(tmpdir(), 'shadow-startup-'))) + const configPath = join(directory, 'config.db'), bioPath = join(directory, 'bio.db') + const config = new Database(memory ? ':memory:' : configPath), bio = new Database(bioPath) + cleanup.push(() => { + config.close() + bio.close() + rmSync(directory, { recursive: true, force: true }) + }) + return { config: drizzle(config, { schema: configSchema }), bio: drizzle(bio, { schema: biometricsSchema }), configPath, bioPath } +} +beforeEach(() => { + vi.stubEnv('SLEEPYPOD_BIOMETRICS_SHADOW_SERVICE', undefined) + vi.spyOn(console, 'warn').mockImplementation(() => {}) +}) +afterEach(() => { + cleanup.splice(0).forEach(close => close()) + vi.unstubAllEnvs() + vi.restoreAllMocks() + vi.clearAllMocks() +}) + +it('leaves scheduling disabled without explicit configuration', () => { + const { config, bio } = databases() + expect(configuredShadowService()).toBeNull() + expect(startConfiguredShadowService(config, bio)).toBeNull() + expect(packagedShadowHostPath).not.toHaveBeenCalled() + expect(startShadowService).not.toHaveBeenCalled() + expect(console.warn).not.toHaveBeenCalled() +}) + +it('wires deployment policy to the actual application databases and fixed package', () => { + const { config, bio, configPath, bioPath } = databases() + const raw = policy() + vi.stubEnv('SLEEPYPOD_BIOMETRICS_SHADOW_SERVICE', JSON.stringify(raw)) + startConfiguredShadowService(config, bio) + expect(startShadowService).toHaveBeenCalledWith({ installation: { workerPath: '/installed/shadow-host.cjs', + pythonPath: raw.pythonPath, modulesPath: raw.modulesPath, presenceFile: raw.presenceFile, + configurationDatabase: configPath, biometricsDatabase: bioPath }, storagePolicy: raw.storagePolicy, + busyTimeoutMs: 50, recoveryIntervalMs: 1000, recoveryMaximumRuns: 10 }) +}) + +it.each([ + 'null', '{broken', 'x'.repeat(16385), + JSON.stringify({ ...policy(), pythonPath: 'relative/python' }), + JSON.stringify({ ...policy(), recoveryIntervalMs: 0 }), + JSON.stringify({ ...policy(), busyTimeoutMs: 1001 }), + JSON.stringify({ ...policy(), configurationDatabase: '/other.db' }), + JSON.stringify({ ...policy(), workerPath: '/other.cjs' }), +])('refuses invalid startup configuration without echoing its contents (%#)', (raw) => { + const { config, bio } = databases() + vi.stubEnv('SLEEPYPOD_BIOMETRICS_SHADOW_SERVICE', raw) + expect(startConfiguredShadowService(config, bio)).toBeNull() + expect(startShadowService).not.toHaveBeenCalled() + expect(packagedShadowHostPath).not.toHaveBeenCalled() + expect(console.warn).toHaveBeenCalledWith('[biometrics] Shadow service unavailable; check deployment configuration and installed package') +}) + +it('requires persistent databases and contains missing-package startup failure', () => { + vi.stubEnv('SLEEPYPOD_BIOMETRICS_SHADOW_SERVICE', JSON.stringify(policy())) + const memory = databases(true) + expect(startConfiguredShadowService(memory.config, memory.bio)).toBeNull() + expect(startShadowService).not.toHaveBeenCalled() + const persistent = databases() + vi.mocked(packagedShadowHostPath).mockImplementationOnce(() => { + throw new Error('missing') + }) + expect(startConfiguredShadowService(persistent.config, persistent.bio)).toBeNull() + expect(startShadowService).not.toHaveBeenCalled() +}) diff --git a/src/biometrics/shadow/tests/store.test.ts b/src/biometrics/shadow/tests/store.test.ts new file mode 100644 index 000000000..62b8d6cbd --- /dev/null +++ b/src/biometrics/shadow/tests/store.test.ts @@ -0,0 +1,570 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { fileURLToPath } from 'node:url' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { ShadowStore } from '../store' +import { reportShadowRun } from '../report' +import { ShadowWorkerSink } from '../worker-sink' +import { ShadowStreamJoin } from '../stream-join' + +const databases: Database.Database[] = [] +afterEach(() => databases.splice(0).forEach(db => db.close())) +const hash = 'a'.repeat(64), start = 1700000000000 +const plan = { subjectId: 'fixture', monitoring: { expectedIntervalMs: 1000, maximumGapMs: 2000, heartbeatStaleMs: 2000, minimumCandidateCoverage: 0.8, maximumCoverageLoss: 0.1, minimumQuality: 0.8, + alignment: { version: 'reference-alignment-v1', pointToleranceMs: 0, clockOffsetMs: 0, offsetSource: 'fixed' } }, schema: 'biometric-shadow-plan-v1', side: 'left', candidateConfigHash: hash, activeConfigHash: 'b'.repeat(64), +qualificationHash: 'c'.repeat(64), engineBuildHash: 'd'.repeat(64), startMs: start, endMs: start + 60000, +maximumPoints: 3, maximumBytes: 10000, metrics: ['heart_rate'] } +function setup(policy = { maximumRuns: 10, maximumPoints: 10, maximumBytes: 100000 }) { + const db = new Database(':memory:') + databases.push(db) + migrate(drizzle(db), { migrationsFolder: fileURLToPath(new URL('../../../db/biometrics-migrations/', import.meta.url)) }) + let now = start + return { db, store: new ShadowStore(db, policy, () => now), advance: (ms: number) => { + now += ms + } } +} +function sample(sequence = 0) { + return { sequence, side: 'left', timestampMs: start + sequence, inputHash: hash, candidateConfigHash: hash, activeConfigHash: plan.activeConfigHash, + readings: [{ metric: 'heart_rate', candidate: { value: null as number | null, quality: 0 }, active: { value: 60, quality: 1 } }] } +} +const joining = { maximumPendingPoints: 10, maximumLatenessMs: 1000, drainGraceMs: 2000 } +const executable = { ...plan, joining, candidateRevisionId: null, + expectedBindings: { left: { generation: 1, resolvedHash: hash }, right: { generation: 1, resolvedHash: hash } }, + execution: { version: 'shadow-execution-policy-v1', configHashes: { left: hash, right: hash }, + maximumRecords: 10, maximumOutputs: 10, maximumCpuMs: 1000, maximumInputBytes: 100000, maximumOutputBytes: 100000, + budgets: { cpuSeconds: 2, addressSpaceBytes: 100000000, maximumRssBytes: 100000000, wallMs: 10000, acknowledgmentMs: 1000, idleMs: 1000, rssIntervalMs: 100, rssTimeoutMs: 100 }, + leaseMs: 1500, heartbeatMs: 200, sources: { pollMs: 50, maximumActiveRows: 10, databaseBusyTimeoutMs: 50, + frames: { maximumFrames: 10, maximumBytes: 100000, maximumEventAgeMs: 1000, maximumFutureSkewMs: 0 }, + context: { presenceStaleMs: 300000, pumpStaleMs: 5000, maximumPresenceBytes: 4096 } } } } + +it('persists candidate revision identity and rejects identity changes on retry', () => { + const { store } = setup() + const revision = '11111111-1111-4111-8111-111111111111' + const run = store.create({ ...executable, candidateRevisionId: revision }, 'identity') + expect(store.get(run.id).plan.candidateRevisionId).toBe(revision) + expect(() => store.create(executable, 'identity')).toThrow('idempotency') + expect(() => store.create({ ...executable, candidateRevisionId: 'latest' }, 'invalid')).toThrow() +}) + +it('scans bounded pages past legacy, future and running plans using saved identity', () => { + const { store, advance } = setup() + store.create(plan, 'legacy') + const { candidateRevisionId, ...legacyExecution } = executable + expect(candidateRevisionId).toBeNull() + store.create(legacyExecution, 'no-identity') + const future = store.create({ ...executable, startMs: start + 1000 }, 'future') + const running = store.create(executable, 'running') + store.claim(running.id, 10000, () => {}) + const ready = store.create(executable, 'ready') + const scan = () => { + const results: string[] = [] + let cursor: string | undefined + let pages = 0 + do { + const page = store.pending({ maximumRuns: 1, ...(cursor ? { afterId: cursor } : {}) }) + if (page.run) { + expect(page.run.revisionId).toBeNull() + results.push(page.run.runId) + } + cursor = page.nextCursor ?? undefined + expect(++pages).toBeLessThan(8) + } while (cursor) + return results + } + expect(scan()).toEqual([ready.id]) + advance(1000) + expect(scan().sort()).toEqual([ready.id, future.id].sort()) + advance(60000) + expect(scan()).toEqual([]) + expect(store.get(running.id).state).toBe('running') +}) + +it('fails queue inspection on corrupted pending evidence', () => { + const { store, db } = setup() + const run = store.create(executable, 'ready') + db.prepare('UPDATE biometric_shadow_runs SET plan_hash = \'corrupt\' WHERE id = ?').run(run.id) + expect(() => store.pending({ maximumRuns: 1 })).toThrow('integrity') +}) +function workerOutput(timestampMs: number, sequence = 0) { + return { sequence, inputHash: hash, side: 'left', sourceSide: 'left', timestampMs, configHash: hash, + readings: [{ metric: 'heart_rate', unit: 'bpm', windowMs: null, value: 60, quality: 1 }, + { metric: 'breathing_rate', unit: 'breaths/min', windowMs: null, value: 15, quality: 1 }, + { metric: 'window_ibi_variability_v1', unit: 'ms', windowMs: 300000, value: null, quality: 0 }], flags: null, limitations: [] } +} +function activeOutput(timestampMs: number) { + return { side: 'left', timestampMs, configHash: plan.activeConfigHash, readings: workerOutput(timestampMs).readings } +} +describe('separate prospective shadow storage', () => { + it('uses the generated additive migration and never writes canonical observations', () => { + const { db, store } = setup() + for (const operation of ['INSERT', 'UPDATE', 'DELETE']) db.exec(`CREATE TRIGGER deny_vitals_${operation} BEFORE ${operation} ON vitals BEGIN SELECT RAISE(ABORT, 'canonical write forbidden'); END`) + const before = db.prepare('SELECT * FROM vitals').all() + const run = store.create(plan, 'create') + const lease = store.claim(run.id, 10000, (admitted) => { + expect(admitted.qualificationHash).toBe(plan.qualificationHash) + }) + expect(store.append(lease, sample())).toEqual({ state: 'recorded' }) + expect(store.append(lease, sample())).toEqual({ state: 'already_recorded' }) + expect(store.get(run.id)).toMatchObject({ point_count: 1, state: 'running' }) + expect(db.prepare('SELECT * FROM vitals').all()).toEqual(before) + const payload = JSON.parse((db.prepare('SELECT payload FROM biometric_shadow_samples').get() as { payload: string }).payload) + expect(payload).toMatchObject({ attemptId: lease.attemptId, readings: [{ candidate: { value: null }, active: { value: 60 } }] }) + }) + it('rejects retroactive windows and changed idempotency requests, while exact retries survive time passing', () => { + const { store, advance } = setup() + const run = store.create(plan, 'first') + advance(100) + expect(store.create(plan, 'first').id).toBe(run.id) + expect(() => store.create(plan, 'second')).toThrow('prospective') + expect(() => store.create({ ...plan, side: 'right' }, 'first')).toThrow('idempotency') + }) + it('requires admission and fences replaced workers without losing retained samples', () => { + const { store, advance } = setup() + const run = store.create(plan, 'first') + expect(() => store.claim(run.id, 1000, () => { + throw new Error('Unqualified build') + })).toThrow('Unqualified build') + expect(store.get(run.id).state).toBe('pending') + const old = store.claim(run.id, 1000, () => undefined) + store.append(old, sample()) + advance(1000) + expect(() => store.append(old, sample(1))).toThrow('Stale') + const current = store.claim(run.id, 1000, () => undefined) + expect(() => store.renew(old, 1000)).toThrow('Stale') + expect(() => store.finish(old, 'completed', 'stale')).toThrow('Stale') + expect(store.append(current, sample(1))).toEqual({ state: 'recorded' }) + expect(store.get(run.id).point_count).toBe(2) + }) + it('limits concurrent workers per side but permits the other side', () => { + const { store } = setup() + const first = store.create(plan, 'first'), second = store.create(plan, 'second'), right = store.create({ ...plan, side: 'right' }, 'right') + store.claim(first.id, 1000, () => undefined) + expect(() => store.claim(second.id, 1000, () => undefined)).toThrow('Side already') + expect(store.claim(right.id, 1000, () => undefined).runId).toBe(right.id) + }) + it('rejects configuration changes, duplicate metrics, future points, reordered points and altered retries', () => { + const { store, advance } = setup() + const lease = store.claim(store.create(plan, 'first').id, 10000, () => undefined) + for (const invalid of [{ ...sample(), side: 'right' }, { ...sample(), candidateConfigHash: 'f'.repeat(64) }, { ...sample(), activeConfigHash: 'f'.repeat(64) }, + { ...sample(), timestampMs: start - 1 }, { ...sample(), timestampMs: start + 1 }, { ...sample(), readings: [...sample().readings, ...sample().readings] }]) { + expect(() => store.append(lease, invalid)).toThrow('frozen plan') + } + store.append(lease, sample()) + expect(() => store.append(lease, { ...sample(), inputHash: 'f'.repeat(64) })).toThrow('idempotency') + advance(100) + expect(() => store.append(lease, sample(2))).toThrow('Out-of-order') + expect(() => store.append(lease, { ...sample(1), timestampMs: start })).toThrow('Out-of-order') + }) + it('atomically stops at per-run and global quotas and keeps stored evidence', () => { + const { store, advance } = setup({ maximumRuns: 2, maximumPoints: 3, maximumBytes: 100000 }) + const first = store.create({ ...plan, maximumPoints: 1 }, 'first'), second = store.create({ ...plan, side: 'right' }, 'second') + expect(() => store.create(plan, 'third')).toThrow('run quota') + const lease = store.claim(first.id, 10000, () => undefined) + const other = store.claim(second.id, 10000, () => undefined) + store.append(lease, sample()) + advance(1) + expect(store.append(lease, sample(1))).toEqual({ state: 'quota_reached' }) + expect(store.get(first.id)).toMatchObject({ state: 'stopped', reason: 'storage_quota', point_count: 1 }) + store.append(other, { ...sample(), side: 'right' }) + store.append(other, { ...sample(1), side: 'right' }) + advance(1) + expect(store.append(other, { ...sample(2), side: 'right' })).toEqual({ state: 'quota_reached' }) + expect(store.get(second.id).point_count).toBe(2) + }) + it('can cancel after deadline and cannot accept asynchronous admission', () => { + const { store, advance } = setup() + const run = store.create(plan, 'first') + expect(() => store.claim(run.id, 1000, async () => undefined)).toThrow('synchronously') + expect(store.get(run.id).state).toBe('pending') + const lease = store.claim(run.id, 1000, () => undefined) + store.append(lease, sample()) + advance(60000) + expect(() => store.renew(lease, 1000)).toThrow('Stale') + expect(store.stop(run.id, 'window_elapsed')).toMatchObject({ state: 'stopped', reason: 'window_elapsed', point_count: 1 }) + expect(store.stop(run.id, 'retry')).toMatchObject({ reason: 'window_elapsed' }) + expect(() => store.claim(run.id, 1000, () => undefined)).toThrow('unavailable') + }) + it('reports dropped candidate values, common reference support and current worker health', () => { + const { store, advance } = setup() + const lease = store.claim(store.create(plan, 'first').id, 10000, () => undefined) + const first = sample() + first.readings[0].candidate = { value: 61, quality: 1 } + store.append(lease, first) + advance(1000) + store.append(lease, { ...sample(1), timestampMs: start + 1000 }) + const reference = { subjectId: 'fixture', side: 'left', source: 'test', rows: [0, 1000].map((offset, index) => ({ + recordKey: String(index), metric: 'heart_rate', value: 60, unit: 'bpm', timestampMs: start + offset })) } + const report = reportShadowRun(store, lease.runId, { maximumPoints: 10, maximumBytes: 10000 }, reference) + expect(report).toMatchObject({ expectedSlots: 2, observedSamples: 2, health: 'issues_detected', issues: ['coverage_degraded'], + metrics: [{ candidate: { coverage: 0.5, usableCount: 1 }, current: { coverage: 1 }, coverageDifference: -0.5, + comparison: { commonObservationCount: 1, lostObservationTimestamps: 1 } }] }) + expect(report.reportHash).toMatch(/^[a-f0-9]{64}$/) + advance(10000) + expect(report.contextEvidence).toBe('host_observations_not_live_processor_cache_attestation') + const stale = reportShadowRun(store, lease.runId, { maximumPoints: 10, maximumBytes: 10000 }) + expect(stale.issues).toEqual(expect.arrayContaining(['worker_stale', 'observation_gap'])) + }) + it('does not inflate coverage from bursts within one cadence slot', () => { + const { store, advance } = setup() + const lease = store.claim(store.create(plan, 'first').id, 10000, () => undefined) + for (let index = 0; index < 3; index++) { + const point = sample(index) + point.readings[0].candidate = { value: 60, quality: 1 } + store.append(lease, point) + advance(1) + } + advance(1997) + const report = reportShadowRun(store, lease.runId, { maximumPoints: 10, maximumBytes: 10000 }) + expect(report).toMatchObject({ expectedSlots: 3, metrics: [{ candidate: { usableCount: 3, usableSlots: 1, coverage: 1 / 3 } }] }) + }) + it('requires matching reference identity and never compares RMSSD to the variability index', () => { + const { store } = setup() + const lease = store.claim(store.create({ ...plan, metrics: ['window_ibi_variability_v1'], windowIbiWindowMs: 300000 }, 'first').id, 10000, () => undefined) + store.append(lease, { ...sample(), readings: [{ metric: 'window_ibi_variability_v1', active: { value: 20, quality: 1 }, candidate: { value: 20, quality: 1 } }] }) + const reference = { subjectId: 'fixture', side: 'left', source: 'test', rows: [{ recordKey: '0', metric: 'hrv_rmssd', value: 20, unit: 'ms', windowMs: 300000, timestampMs: start }] } + const limits = { maximumPoints: 10, maximumBytes: 10000 } + expect(reportShadowRun(store, lease.runId, limits, reference).metrics[0]).toMatchObject({ + referenceExclusions: { incompatible_metric_definition: 1 }, comparison: { commonSupport: { candidate: { status: 'incompatible', metrics: null } } }, + }) + expect(() => reportShadowRun(store, lease.runId, limits, { ...reference, subjectId: 'other' })).toThrow('subject or side') + expect(() => reportShadowRun(store, lease.runId, limits, { ...reference, side: 'right' })).toThrow('subject or side') + expect(() => store.create({ ...plan, metrics: ['window_ibi_variability_v1'] }, 'invalid')).toThrow() + }) + it('refuses oversized or corrupted snapshots instead of scoring partial history', () => { + const { db, store } = setup() + const lease = store.claim(store.create(plan, 'first').id, 10000, () => undefined) + store.append(lease, sample()) + expect(() => store.snapshot(lease.runId, { maximumPoints: 10, maximumBytes: 1 })).toThrow('input limit') + db.prepare('UPDATE biometric_shadow_samples SET content_hash = ? WHERE run_id = ?').run('f'.repeat(64), lease.runId) + expect(() => store.snapshot(lease.runId, { maximumPoints: 10, maximumBytes: 10000 })).toThrow('integrity') + }) + it('reports clock skew and missing observations without converting unknown values to zero', () => { + const { store, advance } = setup() + const run = store.create(plan, 'first') + expect(reportShadowRun(store, run.id, { maximumPoints: 10, maximumBytes: 10000 }).issues).toContain('no_observations') + const lease = store.claim(run.id, 10000, () => undefined) + store.append(lease, sample()) + advance(-1) + expect(reportShadowRun(store, run.id, { maximumPoints: 10, maximumBytes: 10000 })).toMatchObject({ + health: 'issues_detected', issues: expect.arrayContaining(['clock_skew']), metrics: [{ candidate: { coverage: null } }], + }) + }) + it('bridges typed worker readings with exact active identity and preserves missing data and limitations', () => { + const { store, advance } = setup() + const lease = store.claim(store.create(plan, 'first').id, 1000, () => undefined) + const sink = new ShadowWorkerSink(store, lease) + const output = { sequence: 0, inputHash: hash, side: 'left', sourceSide: 'right', timestampMs: start, configHash: hash, + readings: [{ metric: 'heart_rate', unit: 'bpm', windowMs: null, value: 60, quality: 1 }, + { metric: 'breathing_rate', unit: 'breaths/min', windowMs: null, value: 15, quality: 1 }, + { metric: 'window_ibi_variability_v1', unit: 'ms', windowMs: 300000, value: null, quality: 0 }], + flags: null, limitations: ['cold_start_without_serialized_state'] } + const active = { side: 'left', timestampMs: start, configHash: plan.activeConfigHash, readings: output.readings } + for (const changed of [{ ...active, side: 'right' }, { ...active, timestampMs: start + 1 }, { ...active, configHash: hash }]) { + expect(() => sink.append(output, changed)).toThrow('not contemporaneous') + } + expect(store.get(lease.runId).point_count).toBe(0) + expect(sink.append(output, null)).toEqual({ state: 'recorded' }) + expect(sink.append(output, null)).toEqual({ state: 'already_recorded' }) + const snapshot = store.snapshot(lease.runId, { maximumPoints: 10, maximumBytes: 10000 }) + expect(snapshot.samples[0]).toMatchObject({ readings: [{ active: { value: null, quality: 0 }, candidate: { value: 60 } }], + candidateProvenance: { sourceSide: 'right', limitations: expect.arrayContaining(['missing_contemporaneous_active_reading']) } }) + expect(reportShadowRun(store, lease.runId, { maximumPoints: 10, maximumBytes: 10000 }).limitations).toContain('cold_start_without_serialized_state') + advance(1000) + expect(() => sink.append({ ...output, sequence: 1, timestampMs: start + 1000 }, null)).toThrow('Stale') + }) + it('retains active-only times so dropped candidate readings reduce reported coverage', () => { + const { store, advance } = setup() + const lease = store.claim(store.create({ ...plan, joining }, 'joined').id, 10000, () => undefined) + const join = new ShadowStreamJoin(store, lease) + for (const offset of [2000, 0, 1000]) join.active(activeOutput(start + offset)) + join.candidate(workerOutput(start + 1000)) + advance(2999) + expect(join.advance(start + 2999)).toEqual({ emitted: 2, stopped: false }) + advance(1) + expect(join.advance(start + 3000)).toEqual({ emitted: 1, stopped: false }) + const report = reportShadowRun(store, lease.runId, { maximumPoints: 10, maximumBytes: 10000 }) + expect(report).toMatchObject({ pairing: { activeOnly: 2, candidateOnly: 0, unknownInputIdentity: 2 }, + metrics: [{ current: { usableCount: 3 }, candidate: { usableCount: 1 }, issues: expect.arrayContaining(['coverage_regression']) }] }) + const points = store.snapshot(lease.runId, { maximumPoints: 10, maximumBytes: 10000 }).samples + expect(points.map(row => row.timestampMs)).toEqual([start, start + 1000, start + 2000]) + expect(points[0]).toMatchObject({ inputHash: null, inputIdentityKind: 'unavailable', readings: [{ candidate: { value: null } }] }) + }) + it('does not match nearby timestamps and fails explicitly on late or conflicting arrivals', () => { + const { store, advance } = setup() + const lease = store.claim(store.create({ ...plan, joining }, 'joined').id, 10000, () => undefined) + const join = new ShadowStreamJoin(store, lease) + join.active(activeOutput(start)) + join.active(activeOutput(start)) + join.candidate(workerOutput(start + 1)) + advance(1001) + expect(join.advance(start + 1001).emitted).toBe(2) + expect(reportShadowRun(store, lease.runId, { maximumPoints: 10, maximumBytes: 10000 }).pairing).toMatchObject({ activeOnly: 1, candidateOnly: 1 }) + expect(() => join.candidate(workerOutput(start))).toThrow('join_late_arrival') + expect(store.get(lease.runId)).toMatchObject({ state: 'failed', reason: 'join_late_arrival' }) + const nextPlan = { ...plan, startMs: start + 1001, joining } + const secondLease = store.claim(store.create(nextPlan, 'second').id, 10000, () => undefined) + const second = new ShadowStreamJoin(store, secondLease) + second.active(activeOutput(start + 1001)) + const changed = activeOutput(start + 1001) + changed.readings[0].value = 80 + expect(() => second.active(changed)).toThrow('join_conflicting_duplicate') + }) + it('bounds pending points and preserves uncertain-write retry identity', () => { + const { store, advance } = setup() + const lease = store.claim(store.create({ ...plan, joining: { ...joining, maximumPendingPoints: 1 } }, 'joined').id, 10000, () => undefined) + const join = new ShadowStreamJoin(store, lease) + join.active(activeOutput(start)) + join.candidate(workerOutput(start)) + const append = store.append.bind(store) + vi.spyOn(store, 'append').mockImplementationOnce((lease, point) => { + append(lease, point) + throw new Error('lost acknowledgment') + }) + advance(1000) + expect(() => join.advance(start + 1000)).toThrow('lost acknowledgment') + expect(store.get(lease.runId).point_count).toBe(1) + expect(join.advance(start + 1000).emitted).toBe(1) + expect(store.get(lease.runId).point_count).toBe(1) + join.active(activeOutput(start + 2000)) + expect(() => join.active(activeOutput(start + 3000))).toThrow('join_queue_quota') + }) + it('drains final buffered samples during explicit grace without extending the sampling window', () => { + const { store, advance } = setup() + const selected = { ...plan, endMs: start + 5000, joining } + const run = store.create(selected, 'joined') + const lease = store.claim(run.id, 10000, () => undefined) + const join = new ShadowStreamJoin(store, lease) + join.active(activeOutput(start + 4999)) + advance(6000) + expect(join.advance(start + 6000)).toEqual({ emitted: 1, stopped: false }) + expect(store.get(run.id).point_count).toBe(1) + store.renew(lease, 10000) + expect(store.get(run.id).lease_until_ms).toBe(start + 7000) + advance(1000) + expect(() => store.renew(lease, 10000)).toThrow('Stale') + }) + it('enforces byte quotas and terminal states', () => { + const { store } = setup() + const lease = store.claim(store.create({ ...plan, maximumBytes: 1 }, 'first').id, 1000, () => undefined) + expect(store.append(lease, sample()).state).toBe('quota_reached') + expect(() => store.append(lease, sample())).toThrow('Stale') + const next = store.claim(store.create(plan, 'second').id, 1000, () => undefined) + store.finish(next, 'failed', 'processor_failure') + expect(store.get(next.runId)).toMatchObject({ state: 'failed', reason: 'processor_failure' }) + expect(() => store.renew(next, 1000)).toThrow('Stale') + }) +}) + +describe('bounded expired shadow recovery', () => { + it('revokes expired leases while retaining observations and exposing unconfirmed cleanup', () => { + const { store, advance } = setup() + const run = store.create(plan, 'recover') + const lease = store.claim(run.id, 1000, () => {}) + store.append(lease, sample()) + advance(1000) + expect(store.reconcileExpired({ maximumRuns: 10 }).recovered).toEqual([{ runId: run.id, attemptId: lease.attemptId, reason: 'shadow_lease_expired_cleanup_unconfirmed' }]) + expect(store.get(run.id)).toMatchObject({ state: 'failed', point_count: 1, attempt_id: lease.attemptId, workerReceipt: null }) + expect(() => store.append(lease, sample(1))).toThrow(/Stale/) + expect(() => store.renew(lease, 1000)).toThrow(/Stale/) + expect(store.reconcileExpired({ maximumRuns: 10 }).recovered).toEqual([]) + const report = reportShadowRun(store, run.id, { maximumPoints: 10, maximumBytes: 10000 }) + expect(report.issues).toContain('worker_cleanup_unconfirmed') + expect(report.observedSamples).toBe(1) + }) + it('marks missed pending windows without claiming processor execution', () => { + const { store, advance } = setup() + const run = store.create(plan, 'missed') + advance(60000) + expect(store.reconcileExpired({ maximumRuns: 10 }).recovered).toEqual([{ runId: run.id, attemptId: null, reason: 'shadow_observation_window_missed' }]) + expect(store.get(run.id)).toMatchObject({ state: 'failed', point_count: 0, attempt_id: null }) + const report = reportShadowRun(store, run.id, { maximumPoints: 10, maximumBytes: 10000 }) + expect(report.issues).toContain('observation_window_missed') + expect(report.issues).not.toContain('processor_failed') + }) + it('preserves renewed replacement attempts and terminal evidence', () => { + const { store, advance } = setup() + const run = store.create(plan, 'replacement'), stopped = store.create({ ...plan, side: 'right' }, 'stopped') + store.claim(run.id, 1000, () => {}) + store.stop(stopped.id, 'operator') + advance(1000) + const replacement = store.claim(run.id, 1000, () => {}) + expect(store.reconcileExpired({ maximumRuns: 10 }).recovered).toEqual([]) + expect(store.get(run.id)).toMatchObject({ state: 'running', attempt_id: replacement.attemptId }) + expect(store.get(stopped.id)).toMatchObject({ state: 'stopped', reason: 'operator' }) + }) + it('advances a bounded keyset scan past future runs without starving later rows', () => { + const { store, advance } = setup() + const runs = Array.from({ length: 5 }, (_, index) => store.create({ ...plan, endMs: start + (index % 2 ? 120000 : 60000) }, String(index))) + advance(60000) + const recovered: string[] = [], visited: number[] = [] + let cursor: string | undefined + do { + const batch = store.reconcileExpired({ maximumRuns: 2, ...(cursor ? { afterId: cursor } : {}) }) + recovered.push(...batch.recovered.map(row => row.runId)) + visited.push(batch.scanned) + cursor = batch.nextCursor ?? undefined + } while (cursor) + expect(visited).toEqual([2, 2, 1]) + expect(recovered.sort()).toEqual([runs[0].id, runs[2].id, runs[4].id].sort()) + expect(() => store.reconcileExpired({ maximumRuns: 101 })).toThrow() + }) +}) + +it('recovery preserves final drain authority until the actual lease deadline', () => { + const { store, advance } = setup() + const run = store.create({ ...plan, joining }, 'draining') + store.claim(run.id, 100000, () => {}) + advance(61000) + expect(store.reconcileExpired({ maximumRuns: 10 }).recovered).toEqual([]) + advance(1000) + expect(store.reconcileExpired({ maximumRuns: 10 }).recovered).toHaveLength(1) +}) +it('rolls back the entire recovery batch if a plan fails integrity validation', () => { + const { db, store, advance } = setup() + const ids = [store.create(plan, 'one').id, store.create(plan, 'two').id].sort() + db.prepare('UPDATE biometric_shadow_runs SET plan_hash = \'corrupt\' WHERE id = ?').run(ids[1]) + advance(60000) + expect(() => store.reconcileExpired({ maximumRuns: 10 })).toThrow(/integrity/) + expect(store.get(ids[0]).state).toBe('pending') +}) + +describe('storage policy and integrity guards', () => { + const receipt = { health: { state: 'completed', reason: 'window_complete', records: 2, outputs: 2, cpu_ms: 1, input_hash: hash, + counts: { records: 2, accepted_frames: 2, empty_channel_frames: 0, pump_gated_frames: 0, preroll_outputs: 0, scored_outputs: 2 }, limitations: [] }, + inputBytes: 100, outputBytes: 100, peakObservedRssBytes: 1000, rssSamples: 2, maximumObservationGapMs: 100, + budgets: executable.execution.budgets, memoryScope: 'sum_rss_of_shadow_process_group', exactKernelPeak: false } + const partial = { kind: 'partial_supervisor_observation', health: null, inputBytes: 100, outputBytes: 100, diagnosticBytes: 0, + peakObservedRssBytes: 1000, rssSamples: 2, maximumObservationGapMs: 100, budgets: executable.execution.budgets, + cleanupConfirmed: true, memoryScope: 'sum_rss_of_shadow_process_group', exactKernelPeak: false } + const oversized = Array.from({ length: 32 }, () => '€'.repeat(200)) + + it('pages run listings per side and refuses plans beyond the storage policy', () => { + const { store } = setup() + const ids = ['one', 'two', 'three'].map(key => store.create(plan, key).id).sort() + const first = store.list({ side: 'left', maximumRuns: 2 }) + expect(first.items.map(row => row.id)).toEqual(ids.slice(0, 2)) + expect(first.nextCursor).toBe(ids[1]) + const rest = store.list({ side: 'left', maximumRuns: 2, afterId: ids[1] }) + expect(rest.items.map(row => row.id)).toEqual([ids[2]]) + expect(rest.nextCursor).toBeNull() + expect(store.list({ side: 'right', maximumRuns: 2 }).items).toEqual([]) + expect(() => store.list({ side: 'left', maximumRuns: 51 })).toThrow() + expect(() => store.create({ ...plan, maximumPoints: 11 }, 'points')).toThrow('exceeds storage policy') + expect(() => store.create({ ...plan, maximumBytes: 100001 }, 'bytes')).toThrow('exceeds storage policy') + expect(() => new ShadowStore(setup().db).create(plan, 'no-policy')).toThrow('storage policy required') + }) + + it('refuses asynchronous admission, windows that pass during admission and claims outside the window', () => { + const { store, advance } = setup() + const later = async () => { + throw new Error('later') + } + expect(() => store.create(plan, 'async', later)).toThrow('synchronously') + expect(store.list({ side: 'left', maximumRuns: 5 }).items).toEqual([]) + const run = store.create(plan, 'claimed') + expect(() => store.claim(run.id, 1000, later)).toThrow('synchronously') + const future = store.create({ ...plan, startMs: start + 1000 }, 'future') + expect(() => store.claim(future.id, 1000, () => {})).toThrow('Outside shadow window') + const lease = store.claim(run.id, 1000, () => {}) + expect(() => store.claim(run.id, 1000, () => {})).toThrow('Shadow run unavailable') + expect(store.failExpired(lease, 'early')).toBe(false) + store.renew(lease, 2000) + expect(store.get(run.id).lease_until_ms).toBe(start + 2000) + expect(() => store.finish(lease, 'failed', 'with_receipt', receipt)).toThrow('requires a completed executable plan') + expect(() => store.create(plan, 'late', () => advance(1))).toThrow('prospective') + advance(60000) + expect(() => store.claim(future.id, 1000, () => {})).toThrow('Outside shadow window') + store.stop(run.id, 'operator') + expect(() => store.recordPartialReceipt(lease, partial)).toThrow('terminal executable attempt') + }) + + it('detects corrupted receipts, totals and queue authority', () => { + const { store, db, advance } = setup() + const run = store.create(executable, 'corrupt') + const lease = store.claim(run.id, 1000, () => {}) + store.append(lease, sample()) + db.prepare('UPDATE biometric_shadow_runs SET worker_receipt_hash = ? WHERE id = ?').run('x', run.id) + expect(() => store.get(run.id)).toThrow('receipt integrity') + db.prepare('UPDATE biometric_shadow_runs SET worker_receipt = ?, worker_receipt_hash = ? WHERE id = ?').run('x'.repeat(16385), 'x', run.id) + expect(() => store.get(run.id)).toThrow('exceeds storage limit') + db.prepare('UPDATE biometric_shadow_runs SET worker_receipt = NULL, worker_receipt_hash = NULL, point_count = point_count + 1 WHERE id = ?').run(run.id) + expect(() => store.snapshot(run.id, { maximumPoints: 10, maximumBytes: 10000 })).toThrow('totals integrity') + expect(() => store.cursor(run.id)).toThrow('cursor integrity') + db.prepare('UPDATE biometric_shadow_runs SET point_count = point_count - 1 WHERE id = ?').run(run.id) + expect(store.cursor(run.id)).toEqual({ nextSequence: 1, lastTimestampMs: start }) + const pending = store.create(executable, 'pending') + db.prepare('UPDATE biometric_shadow_runs SET attempt_id = ? WHERE id = ?').run('11111111-1111-4111-8111-111111111111', pending.id) + expect(() => store.pending({ maximumRuns: 10 })).toThrow('Invalid pending shadow authority') + advance(60000) + expect(() => store.reconcileExpired({ maximumRuns: 10 })).toThrow('Invalid pending shadow authority') + }) + + it('fences sink and join construction, identity and window mismatches', () => { + const { store, advance } = setup() + const run = store.create({ ...plan, joining }, 'joined'), unjoinedRun = store.create({ ...plan, side: 'right' }, 'unjoined') + const stale = store.claim(run.id, 1000, () => undefined) + advance(1000) + const lease = store.claim(run.id, 10000, () => undefined) + expect(() => new ShadowWorkerSink(store, stale)).toThrow('attempt mismatch') + expect(() => new ShadowStreamJoin(store, stale)).toThrow('live attempt') + const unjoined = store.claim(unjoinedRun.id, 10000, () => undefined) + expect(() => new ShadowStreamJoin(store, unjoined)).toThrow('frozen join policy') + const sink = new ShadowWorkerSink(store, lease) + const output = workerOutput(start + 1000) + expect(() => sink.append({ ...output, side: 'right' }, null)).toThrow('identity mismatch') + expect(() => sink.append({ ...output, sequence: 5 }, null)).toThrow('identity mismatch') + const active = { ...activeOutput(start + 1000), inputHash: hash, provenance: { processingRunId: '11111111-1111-4111-8111-111111111111', sourceSide: 'left', generation: 1, manifestHash: hash } } + expect(sink.append(output, active)).toEqual({ state: 'recorded' }) + expect(store.snapshot(run.id, { maximumPoints: 10, maximumBytes: 10000 }).samples[0]).toMatchObject({ activeProvenance: active.provenance, + readings: [{ active: { value: 60, quality: 1 } }], candidateProvenance: { limitations: [] } }) + const join = new ShadowStreamJoin(store, lease) + expect(() => join.active({ ...activeOutput(start + 2000), side: 'right' })).toThrow('join_identity_mismatch') + expect(store.get(run.id).state).toBe('failed') + expect(() => join.active(activeOutput(start + 2000))).toThrow('closed') + expect(() => join.advance(start + 2000)).toThrow('closed') + }) + + it('records active-only identity and fails on metric gaps or clock regression', () => { + const { store, advance } = setup() + const windowed = { ...plan, joining, metrics: ['heart_rate', 'window_ibi_variability_v1'], windowIbiWindowMs: 300000 } + const lease = store.claim(store.create(windowed, 'windowed').id, 10000, () => undefined) + const mismatchedRun = store.create({ ...windowed, side: 'right', windowIbiWindowMs: 60000 }, 'mismatched') + const join = new ShadowStreamJoin(store, lease) + join.active({ ...activeOutput(start), inputHash: hash }) + advance(1000) + expect(join.advance(start + 1000)).toEqual({ emitted: 1, stopped: false }) + expect(store.snapshot(lease.runId, { maximumPoints: 10, maximumBytes: 10000 }).samples[0]).toMatchObject({ inputIdentityKind: 'active_provenance', inputHash: hash }) + expect(() => join.advance(start + 999)).toThrow('join_clock_regression') + const mismatched = store.claim(mismatchedRun.id, 10000, () => undefined) + const output = { ...workerOutput(start + 1000), side: 'right' } + expect(() => new ShadowWorkerSink(store, mismatched).append(output, null)).toThrow('window mismatch') + expect(() => new ShadowStreamJoin(store, mismatched).candidate(output)).toThrow('join_metric_mismatch') + }) + + it('pages the pending queue past non-executable plans', () => { + const { store } = setup() + const ids = [store.create(plan, 'legacy-one').id, store.create(plan, 'legacy-two').id].sort() + expect(store.pending({ maximumRuns: 1 })).toEqual({ run: null, nextCursor: ids[0] }) + expect(store.pending({ maximumRuns: 1, afterId: ids[0] })).toEqual({ run: null, nextCursor: null }) + }) + + it('verifies stored receipts against the terminal state and bounds their size', () => { + const { store, db } = setup() + const completed = store.create(executable, 'completed') + const lease = store.claim(completed.id, 1000, () => {}) + expect(() => store.finish(lease, 'completed', 'done', { ...receipt, health: { ...receipt.health, limitations: oversized } })).toThrow('exceeds storage limit') + store.finish(lease, 'completed', 'done', receipt) + expect(store.get(completed.id).workerReceipt).toMatchObject({ schema: 'biometric-shadow-receipt-v1', receipt }) + db.prepare('UPDATE biometric_shadow_runs SET ended_at_ms = ended_at_ms + 1 WHERE id = ?').run(completed.id) + expect(() => store.get(completed.id)).toThrow('receipt integrity') + const stopped = store.create({ ...executable, side: 'right' }, 'stopped') + const stoppedLease = store.claim(stopped.id, 1000, () => {}) + store.stop(stopped.id, 'operator') + expect(() => store.recordPartialReceipt(stoppedLease, { ...partial, health: { ...receipt.health, state: 'stopped', limitations: oversized } })).toThrow('exceeds storage limit') + expect(store.recordPartialReceipt(stoppedLease, partial)).toBe(true) + db.prepare('UPDATE biometric_shadow_runs SET state = \'completed\' WHERE id = ?').run(stopped.id) + expect(() => store.get(stopped.id)).toThrow('receipt integrity') + }) +}) diff --git a/src/biometrics/shadow/tests/submission.test.ts b/src/biometrics/shadow/tests/submission.test.ts new file mode 100644 index 000000000..f078ba1e0 --- /dev/null +++ b/src/biometrics/shadow/tests/submission.test.ts @@ -0,0 +1,115 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import { ShadowStore } from '../store' +import { submitInstalledShadow } from '../submission' +import { inspectInstalledShadowHost, loadInstalledShadowHost } from '../installed-host' +import { openShadowStateDatabase } from '../active-database' +import { installedShadowIdentity } from '../device-identity' +import { submissionFixture, submissionEntry, submissionTime } from './fixtures/submission' +vi.mock('../installed-host', () => ({ inspectInstalledShadowHost: vi.fn(), loadInstalledShadowHost: vi.fn() })) +vi.mock('../active-database', () => ({ openShadowStateDatabase: vi.fn() })) +vi.mock('../device-identity', () => ({ installedShadowIdentity: vi.fn() })) +vi.mock('@/src/streaming/piezoStream', () => ({ getSensorFrameTimes: () => ({ 'piezo-dual': submissionTime }) })) +const paths = { workerPath: '/installed/shadow-host.cjs', pythonPath: '/installed/python', modulesPath: '/installed/modules', + configurationDatabase: '/installed/config.db', biometricsDatabase: '/installed/bio.db', presenceFile: '/run/presence' } +const policy = { maximumRuns: 10, maximumPoints: 100, maximumBytes: 100000 } +let db: Database.Database, now: number +beforeEach(() => { + vi.resetAllMocks() + db = new Database(':memory:') + migrate(drizzle(db), { migrationsFolder: 'src/db/biometrics-migrations' }) + now = submissionTime +}) +afterEach(() => db.close()) +function setup() { + const owned = { host: { admit: vi.fn() }, close: vi.fn() } + const open = vi.fn().mockResolvedValue(owned), guard = vi.fn() + class Store extends ShadowStore { + constructor(raw: Database.Database, limits: unknown) { super(raw, limits, () => now) } + } + const inspected = { api: { ShadowStore: Store, openProfiledShadowHost: open, + readInstalledShadowQualifications: () => ({ entries: [submissionEntry] }) }, assertCurrent: guard } + vi.mocked(loadInstalledShadowHost).mockReturnValue(inspected as unknown as ReturnType) + vi.mocked(inspectInstalledShadowHost).mockResolvedValue(inspected as unknown as Awaited>) + const handle = { db, assertCurrent: vi.fn(), close: vi.fn() } + vi.mocked(openShadowStateDatabase).mockReturnValue(handle) + vi.mocked(installedShadowIdentity).mockReturnValue(vi.fn()) + return { owned, open, guard, handle, inspected: inspected as unknown as Awaited> } +} + +it('checks admission inside the creation transaction and closes owned admission resources', async () => { + const { owned, open, handle } = setup(), input = submissionFixture() + owned.host.admit.mockImplementation(() => { + expect(db.inTransaction).toBe(true) + expect(db.prepare('SELECT count(*) AS n FROM biometric_shadow_runs').get()).toEqual({ n: 0 }) + }) + const run = await submitInstalledShadow(input, paths, policy, 50) + expect(run).toMatchObject({ state: 'pending', plan: input.plan }) + expect(open).toHaveBeenCalledWith(input.plan, expect.objectContaining(paths), null, undefined) + expect(owned.host.admit).toHaveBeenCalledOnce() + expect(owned.close).toHaveBeenCalledOnce() + expect(handle.close).toHaveBeenCalledOnce() +}) + +it('returns exact retries before probing changed health and rejects plan substitution', async () => { + setup() + const input = submissionFixture(), first = await submitInstalledShadow(input, paths, policy, 50) + now += 100000 + vi.mocked(inspectInstalledShadowHost).mockClear().mockRejectedValue(new Error('unavailable')) + expect((await submitInstalledShadow(input, paths, policy, 50)).id).toBe(first.id) + await expect(submitInstalledShadow({ ...input, plan: { ...input.plan, subjectId: 'other' } }, paths, policy, 50)).rejects.toThrow('idempotency') + expect(inspectInstalledShadowHost).not.toHaveBeenCalled() +}) + +it('rejects an over-long store timeout, cancelled submissions and missing qualification before admission', async () => { + const { open, handle, inspected } = setup(), input = submissionFixture() + await expect(submitInstalledShadow(input, paths, policy, input.plan.execution.sources.databaseBusyTimeoutMs + 1)).rejects.toThrow('timeout exceeds') + const aborted = new AbortController() + aborted.abort() + await expect(submitInstalledShadow(input, paths, policy, 50, aborted.signal)).rejects.toThrow('cancelled') + expect(loadInstalledShadowHost).not.toHaveBeenCalled() + const late = new AbortController() + vi.mocked(inspectInstalledShadowHost).mockImplementationOnce(async () => { + late.abort() + return inspected + }) + await expect(submitInstalledShadow(input, paths, policy, 50, late.signal)).rejects.toThrow('cancelled') + expect(open).not.toHaveBeenCalled() + await expect(submitInstalledShadow({ ...input, plan: { ...input.plan, qualificationHash: 'f'.repeat(64) } }, paths, policy, 50)).rejects.toThrow('qualification missing') + expect(open).not.toHaveBeenCalled() + expect(handle.close).toHaveBeenCalledTimes(2) + expect(db.prepare('SELECT count(*) AS n FROM biometric_shadow_runs').get()).toEqual({ n: 0 }) +}) + +it('rechecks the database identity on every host identity probe and reports the live piezo clock', async () => { + const { open, handle } = setup(), input = submissionFixture() + const identity = vi.fn(() => ({ probed: true })) + vi.mocked(installedShadowIdentity).mockReturnValue(identity as unknown as ReturnType) + open.mockImplementation(async (_plan: unknown, options: { identity: () => unknown, piezoReceivedAt: () => number | null }) => { + expect(options.identity()).toEqual({ probed: true }) + expect(options.piezoReceivedAt()).toBe(submissionTime) + return { host: { admit: vi.fn() }, close: vi.fn() } + }) + await submitInstalledShadow(input, paths, policy, 50) + expect(identity).toHaveBeenCalledOnce() + expect(handle.assertCurrent.mock.calls.length).toBeGreaterThanOrEqual(3) +}) + +it.each(['admission', 'window', 'cancel'])('does not queue after %s changes while opening admission', async (failure) => { + const { owned, open, handle } = setup(), abort = new AbortController() + open.mockImplementation(async () => { + if (failure === 'window') now += 2000 + if (failure === 'cancel') abort.abort() + if (failure === 'admission') owned.host.admit.mockImplementation(() => { + throw new Error('binding changed') + }) + return owned + }) + await expect(submitInstalledShadow(submissionFixture(), paths, policy, 50, abort.signal)).rejects.toThrow() + expect(db.prepare('SELECT count(*) AS n FROM biometric_shadow_runs').get()).toEqual({ n: 0 }) + expect(owned.close).toHaveBeenCalledOnce() + expect(handle.close).toHaveBeenCalledOnce() +}) diff --git a/src/biometrics/shadow/tests/worker-child.test.ts b/src/biometrics/shadow/tests/worker-child.test.ts new file mode 100644 index 000000000..f85a976fa --- /dev/null +++ b/src/biometrics/shadow/tests/worker-child.test.ts @@ -0,0 +1,322 @@ +// @vitest-environment node +import type * as ChildProcess from 'node:child_process' +import { randomUUID } from 'node:crypto' +import { existsSync } from 'node:fs' +import { resolve } from 'node:path' +import { setTimeout as delay } from 'node:timers/promises' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { defaultConfig } from '../../algorithms/registry' +import { configHash } from '../../algorithms/canonical' +import { shadowPartialReceiptSchema } from '../receipt' +import { runShadowChild } from '../worker-child' + +const state = vi.hoisted(() => ({ mode: 'normal', pid: 0, descendantPid: 0, rss: 1000 as number | null, hangRss: false, delayClose: false, missingAfterFinish: false, launches: [] as unknown[][] })) +vi.mock('../../backtest/process-group-memory', () => ({ sampleProcessGroupRss: () => state.hangRss ? new Promise(() => {}) : Promise.resolve(state.rss) })) +vi.mock('node:child_process', async (original) => { + const actual = await original() + return { ...actual, spawn: (command: string, args: string[], options: ChildProcess.SpawnOptions) => { + state.launches.push([command, args, options]) + const script = ` +const readline = require('node:readline'); +const mode = ${JSON.stringify(state.mode)}; +let init, records = 0, outputs = 0; +const health = state => ({state, reason:null, records, outputs, cpu_ms:1, input_hash:'a'.repeat(64), limitations:[], + counts:{records,accepted_frames:records,empty_channel_frames:0,pump_gated_frames:0,preroll_outputs:0,scored_outputs:outputs}}); +function emit(row) { process.stdout.write(JSON.stringify({runId:init.runId,attemptId:init.attemptId,...row})+'\\n'); } +if(mode==='ignore') { setInterval(()=>{},1000); } +else readline.createInterface({input:process.stdin}).on('line', line => { + const message = JSON.parse(line); + if(!init) { + init = message; + if(mode==='descendant') { + const grandchild = require('node:child_process').spawn(process.execPath, ['-e', 'setInterval(()=>{},1000)'], {stdio:'inherit'}); + process.stderr.write('descendant:'+grandchild.pid+'\\n'); grandchild.unref(); + } + if(mode==='bad') return process.stdout.write('{}\\n'); + if(mode==='flood') return process.stdout.write('x'.repeat(140000)); + if(mode==='stderr') return process.stderr.write('x'.repeat(17000)); + if(mode==='duplicate') return process.stdout.write((JSON.stringify({runId:init.runId,attemptId:init.attemptId,type:'ready',configHashes:init.configHashes})+'\\n').repeat(2)); + return emit({type:'ready', configHashes:init.configHashes}); + } + if(message.type==='frame') { + records++; + const batch = mode==='output' ? [{sequence:outputs++,inputHash:'a'.repeat(64),side:init.side,sourceSide:init.side, + timestampMs:init.startMs+records*1000,configHash:init.configHashes[init.side],flags:null,limitations:[],readings:[ + {metric:'heart_rate',unit:'bpm',windowMs:null,value:60,quality:1}, + {metric:'breathing_rate',unit:'breaths/min',windowMs:null,value:12,quality:1}, + {metric:'window_ibi_variability_v1',unit:'ms',windowMs:300000,value:null,quality:0}]}] : []; + return emit({type:'frame_result',sequence:message.sequence,outputs:batch,health:health('running')}); + } + if(mode==='noexit') { emit({type:'finished',outputs:[],health:health('completed')}); setInterval(()=>{},1000); return; } + if(mode==='truncated') { process.stdout.write('{'); return process.exit(0); } + emit({type:'finished',outputs:[],health:health('completed')}); + process.exitCode = mode==='badexit' ? 1 : 0; +});` + const child = state.mode === 'python' + ? actual.spawn(resolve('modules/piezo-processor/.venv/bin/python'), ['-B', '-m', 'common.biometrics.shadow_worker'], options) + : state.mode === 'missing' ? actual.spawn('/nonexistent/shadow-test-python', [], options) : actual.spawn(process.execPath, ['-e', script], options) + child.stderr?.on('data', (chunk: Buffer) => { + const match = /descendant:(\d+)/.exec(chunk.toString()) + if (match) state.descendantPid = Number(match[1]) + }) + child.stdout?.on('data', (chunk: Buffer) => { + if (state.missingAfterFinish && chunk.toString().includes('"type":"finished"')) state.rss = null + }) + if (state.delayClose) { + // procfs can lose the address space before Node drains/closes its pipes. + child.once('exit', () => { + state.rss = null + }) + const emit = child.emit.bind(child) + child.emit = ((event: string, ...args: unknown[]) => { + if (event === 'close') { + setTimeout(() => emit(event, ...args), 150) + return true + } + return emit(event, ...args) + }) as typeof child.emit + } + state.pid = child.pid ?? 0 + return child + } } +}) +const hash = configHash(defaultConfig) +function init() { + return { schema: 'biometric-shadow-worker-v1', runId: randomUUID(), attemptId: randomUUID(), side: 'left', + startMs: 1700000000000, endMs: 1700000150000, configs: { left: defaultConfig, right: defaultConfig }, + configHashes: { left: hash, right: hash }, maximumRecords: 10, maximumOutputs: 10, maximumCpuMs: 1000, + maximumInputBytes: 1024 * 1024, maximumOutputBytes: 1024 * 1024 } +} +const budgets = { cpuSeconds: 2, addressSpaceBytes: 100000000, maximumRssBytes: 2000, wallMs: 2000, + acknowledgmentMs: 500, idleMs: 300, rssIntervalMs: 50, rssTimeoutMs: 100 } +async function* source() { + yield { type: 'frame', sequence: 0 } +} +const run = (frames: AsyncIterable = source(), options: Partial[4]> = {}) => + runShadowChild({ pythonPath: '/trusted/python', modulesPath: '/trusted/modules' }, init(), frames, () => {}, { budgets, ...options }) +function reaped() { + expect(() => process.kill(state.pid, 0)).toThrow() +} +beforeEach(() => { + state.mode = 'normal' + state.rss = 1000 + state.hangRss = false + state.delayClose = false + state.missingAfterFinish = false + state.pid = 0 + state.descendantPid = 0 + state.launches = [] +}) +describe('shadow child supervision', () => { + it('uses fixed shipped entrypoint, sanitized environment and bounded sequential replies', async () => { + const result = await run() + expect(result).toMatchObject({ health: { state: 'completed', records: 1 }, peakObservedRssBytes: 1000, exactKernelPeak: false }) + expect(state.launches[0]).toMatchObject(['/trusted/python', ['-B', '-m', 'common.biometrics.shadow_process', '--parent-pid', String(process.pid), '--wall-ms', '2000', '--cpu-seconds', '2', '--address-space-bytes', '100000000'], { detached: true }]) + const env = (state.launches[0][2] as { env: Record }).env + expect(Object.keys(env).sort()).toEqual(['LANG', 'MKL_NUM_THREADS', 'NODE_ENV', 'NUMEXPR_NUM_THREADS', 'OMP_NUM_THREADS', 'OPENBLAS_NUM_THREADS', 'PYTHONPATH', 'TZ']) + reaped() + }) + it.each(['bad', 'duplicate', 'flood', 'stderr', 'truncated', 'badexit'])('rejects %s worker output and reaps it', async (mode) => { + state.mode = mode + const observed = vi.fn() + await expect(run(source(), { onResources: observed })).rejects.toThrow(/Shadow/) + expect(observed).toHaveBeenCalledOnce() + expect(shadowPartialReceiptSchema.parse(observed.mock.calls[0][0]).cleanupConfirmed).toBe(true) + reaped() + }) + it.each(['ignore', 'noexit'])('enforces acknowledgment/exit deadlines for %s', async (mode) => { + state.mode = mode + await expect(run()).rejects.toThrow(/deadline/) + reaped() + }) + it('kills an idle source and invokes source cleanup without waiting for it', async () => { + const cleanup = vi.fn(() => new Promise>(() => {})) + const frames = { [Symbol.asyncIterator]: () => ({ next: () => new Promise>(() => {}), return: cleanup }) } + await expect(run(frames)).rejects.toThrow(/idle deadline/) + expect(cleanup).toHaveBeenCalledOnce() + reaped() + }) + it('honors cancellation while awaiting a source', async () => { + const controller = new AbortController() + const frames = { [Symbol.asyncIterator]: () => ({ next: () => new Promise>(() => {}) }) } + const task = run(frames, { signal: controller.signal }) + await delay(100) + controller.abort() + await expect(task).rejects.toThrow(/cancelled/) + reaped() + }) + it.each([null, 3000])('fails closed for unavailable/over-budget RSS %s', async (rss) => { + state.rss = rss + const observed = vi.fn() + await expect(run(source(), { onResources: observed })).rejects.toThrow(/RSS/) + expect(observed).toHaveBeenCalledOnce() + const receipt = shadowPartialReceiptSchema.parse(observed.mock.calls[0][0]) + expect(receipt.peakObservedRssBytes).toBe(rss) + expect(receipt.cleanupConfirmed).toBe(true) + if (rss === null) expect(receipt.rssSamples).toBe(0) + reaped() + }) + it('accepts disappearing RSS after completion only when clean pipe closure follows', async () => { + state.delayClose = true + const result = await run() + expect(result.health.state).toBe('completed') + expect(result.rssSamples).toBeGreaterThan(0) + reaped() + }) + it('still rejects a failed exit after completion when RSS disappears', async () => { + state.delayClose = true + state.mode = 'badexit' + await expect(run()).rejects.toThrow(/failed or returned an incomplete result/) + reaped() + }) + it('bounds missing RSS after a terminal response when the worker never exits', async () => { + state.mode = 'noexit' + state.missingAfterFinish = true + await expect(run()).rejects.toThrow(/exit deadline/) + reaped() + }) + it('bounds a stalled RSS observation', async () => { + state.hangRss = true + state.mode = 'ignore' + await expect(run()).rejects.toThrow(/RSS observation timed out/) + reaped() + }) + it('enforces wall time independently of a longer per-command deadline', async () => { + state.mode = 'ignore' + await expect(run(source(), { budgets: { ...budgets, wallMs: 100 } })).rejects.toThrow(/wall deadline/) + reaped() + }) + it('preserves source errors and still reaps the worker', async () => { + async function* broken() { + yield { type: 'frame', sequence: 0 } + throw new Error('source disconnected') + } + const observed = vi.fn() + await expect(run(broken(), { onResources: observed })).rejects.toThrow('source disconnected') + expect(observed).toHaveBeenCalledOnce() + expect(observed.mock.calls[0][0]).toMatchObject({ health: { state: 'running', records: 1 }, cleanupConfirmed: true }) + reaped() + }) + it('rejects input quota and out-of-order frames before sending them', async () => { + const task = runShadowChild({ pythonPath: '/trusted/python', modulesPath: '/trusted/modules' }, { ...init(), maximumInputBytes: 1 }, source(), () => {}, { budgets }) + await expect(task).rejects.toThrow(/input exceeds/) + reaped() + async function* wrong() { + yield { type: 'frame', sequence: 7 } + } + await expect(run(wrong())).rejects.toThrow() + reaped() + }) + it('does not launch for invalid config or cancelled admission', async () => { + await expect(runShadowChild({ pythonPath: '/trusted/python', modulesPath: '/trusted/modules' }, { ...init(), configHashes: { left: 'b'.repeat(64), right: hash } }, source(), () => {}, { budgets })).rejects.toThrow(/identity/) + await expect(run(source(), { signal: AbortSignal.abort() })).rejects.toThrow(/cancelled/) + expect(state.launches).toHaveLength(0) + }) +}) + +it('waits for the fenced output sink before requesting another frame', async () => { + state.mode = 'output' + let acknowledged = false + async function* frames() { + yield { type: 'frame', sequence: 0 } + expect(acknowledged).toBe(true) + } + const sink = vi.fn(async () => { + await delay(30) + acknowledged = true + }) + await runShadowChild({ pythonPath: '/trusted/python', modulesPath: '/trusted/modules' }, init(), frames(), sink, { budgets }) + expect(sink).toHaveBeenCalledOnce() + expect(sink.mock.calls[0]).toMatchObject([{ side: 'left', timestampMs: 1700000001000 }]) + reaped() +}) +it('bounds a hung sink by the independent wall deadline', async () => { + state.mode = 'output' + await expect(runShadowChild({ pythonPath: '/trusted/python', modulesPath: '/trusted/modules' }, init(), source(), + () => new Promise(() => {}), { budgets: { ...budgets, wallMs: 200 } })).rejects.toThrow(/wall deadline/) + reaped() +}) +it('preserves storage errors while terminating the worker', async () => { + state.mode = 'output' + await expect(runShadowChild({ pythonPath: '/trusted/python', modulesPath: '/trusted/modules' }, init(), source(), + () => { throw new Error('lease replaced') }, { budgets })).rejects.toThrow('lease replaced') + reaped() +}) +it('kills descendants that retain output pipes after the worker exits', async () => { + state.mode = 'descendant' + await expect(run()).rejects.toThrow(/exit deadline/) + reaped() + expect(state.descendantPid).toBeGreaterThan(0) + let exists = true + for (let attempt = 0; exists && attempt < 50; attempt++) { + try { + process.kill(state.descendantPid, 0) + } + catch { exists = false } + if (exists) await delay(20) + } + expect(exists).toBe(false) +}) +it('handles failed process creation without an unhandled event or cleanup hang', async () => { + state.mode = 'missing' + const observed = vi.fn() + await expect(run(source(), { onResources: observed })).rejects.toThrow(/could not start/) + expect(observed.mock.calls[0][0]).toMatchObject({ health: null, rssSamples: 0, peakObservedRssBytes: null, cleanupConfirmed: true }) + expect(state.pid).toBe(0) +}) +it.skipIf(!existsSync(resolve('modules/piezo-processor/.venv/bin/python')))('supervises the actual locked Python transport through clean completion', async () => { + state.mode = 'python' + async function* frames() { + yield { type: 'frame', sequence: 0, observedAtMs: 1700000000000, record: { type: 'frzHealth' }, + context: { activeSides: ['left'], leftOccupied: true, rightOccupied: false, pumpContextKnown: true } } + } + // Kernel bootstrap is verified separately on Linux; this host test exercises + // real Python pipe behavior and protocol validation through the supervisor. + const result = await runShadowChild({ pythonPath: '/trusted/python', modulesPath: resolve('modules') }, init(), frames(), () => {}, + { budgets: { ...budgets, acknowledgmentMs: 5000, wallMs: 10000 } }) + expect(result.health).toMatchObject({ state: 'completed', records: 1 }) + reaped() +}) +it('enforces cumulative output and frame input limits', async () => { + await expect(runShadowChild({ pythonPath: '/trusted/python', modulesPath: '/trusted/modules' }, { ...init(), maximumOutputBytes: 10 }, source(), + () => {}, { budgets })).rejects.toThrow(/output exceeds byte quota/) + reaped() + async function* oversized() { + yield { type: 'frame', sequence: 0, payload: 'x'.repeat(128 * 1024) } + } + await expect(run(oversized())).rejects.toThrow(/input exceeds byte quota/) + reaped() +}) +it('does not send frames beyond the frozen record quota', async () => { + async function* two() { + yield { type: 'frame', sequence: 0 } + yield { type: 'frame', sequence: 1 } + } + await expect(runShadowChild({ pythonPath: '/trusted/python', modulesPath: '/trusted/modules' }, { ...init(), maximumRecords: 1 }, two(), + () => {}, { budgets })).rejects.toThrow(/record quota/) + reaped() +}) + +it('does not hide failed process-group cleanup behind the original protocol error', async () => { + state.mode = 'bad' + const kill = process.kill.bind(process) + const spy = vi.spyOn(process, 'kill').mockImplementation((pid, signal) => { + if (pid < 0 && signal === 'SIGKILL') throw Object.assign(new Error('denied'), { code: 'EPERM' }) + return kill(pid, signal) + }) + try { + const observed = vi.fn() + await expect(run(source(), { onResources: observed })).rejects.toMatchObject({ code: 'SHADOW_CLEANUP_UNCONFIRMED', + message: 'shadow_worker_cleanup_unconfirmed', cause: expect.objectContaining({ message: 'Shadow worker emitted an invalid response' }) }) + expect(observed.mock.calls[0][0]).toMatchObject({ cleanupConfirmed: false }) + } + finally { + spy.mockRestore() + if (state.pid) { + try { + kill(-state.pid, 'SIGKILL') + } + catch { /* The direct child may have exited when its pipes closed. */ } + } + } +}) diff --git a/src/biometrics/shadow/tests/worker-protocol.test.ts b/src/biometrics/shadow/tests/worker-protocol.test.ts new file mode 100644 index 000000000..4284e7a36 --- /dev/null +++ b/src/biometrics/shadow/tests/worker-protocol.test.ts @@ -0,0 +1,84 @@ +// @vitest-environment node +import { randomUUID } from 'node:crypto' +import { existsSync } from 'node:fs' +import { spawnSync } from 'node:child_process' +import { resolve } from 'node:path' +import { describe, expect, it } from 'vitest' +import { defaultConfig } from '../../algorithms/registry' +import { configHash } from '../../algorithms/canonical' +import { ShadowWorkerProtocol } from '../worker-protocol' + +const hash = 'a'.repeat(64), start = 1700000000000 +const expected = { runId: randomUUID(), attemptId: randomUUID(), side: 'left', startMs: start, endMs: start + 150000, + configHashes: { left: hash, right: hash }, maximumRecords: 200, maximumOutputs: 10, maximumCpuMs: 30000 } +function fixture() { + const protocol = new ShadowWorkerProtocol(expected) + const ready = { type: 'ready', runId: expected.runId, attemptId: expected.attemptId, configHashes: expected.configHashes } + const output = { sequence: 0, inputHash: hash, side: 'left', sourceSide: 'right', timestampMs: start, configHash: hash, + readings: [{ metric: 'heart_rate', unit: 'bpm', windowMs: null, value: 60, quality: 1 }, + { metric: 'breathing_rate', unit: 'breaths/min', windowMs: null, value: null, quality: 0 }, + { metric: 'window_ibi_variability_v1', unit: 'ms', windowMs: 300000, value: null, quality: 0 }], flags: null, limitations: ['cold_start_without_serialized_state'] } + const frame = { type: 'frame_result', runId: expected.runId, attemptId: expected.attemptId, sequence: 0, outputs: [output], + health: { state: 'running', reason: null, records: 1, outputs: 1, cpu_ms: 1, input_hash: hash, limitations: [], + counts: { records: 1, accepted_frames: 1, empty_channel_frames: 0, pump_gated_frames: 0, preroll_outputs: 0, scored_outputs: 1 } } } + return { protocol, ready, frame } +} +describe('shadow worker response validation', () => { + it('requires matching initialization and validates a complete typed batch', () => { + const { protocol, ready, frame } = fixture() + expect(() => protocol.accept(frame, 'frame_result')).toThrow('initialization') + protocol.accept(ready, 'ready') + expect(protocol.accept(frame, 'frame_result')).toMatchObject({ outputs: [{ side: 'left', sourceSide: 'right' }] }) + expect(() => protocol.accept(frame, 'frame_result')).toThrow('sequence') + }) + it('rejects stale identities, side/config/window changes, metric substitutions and forged health without advancing', () => { + const { protocol, ready, frame } = fixture() + protocol.accept(ready, 'ready') + for (const changed of [{ ...frame, attemptId: randomUUID() }, { ...frame, sequence: 1 }, + { ...frame, health: { ...frame.health, outputs: 2 } }, { ...frame, health: { ...frame.health, cpu_ms: 30001 } }]) { + expect(() => protocol.accept(changed, 'frame_result')).toThrow() + } + for (const changes of [{ side: 'right' }, { configHash: 'b'.repeat(64) }, { timestampMs: start - 1 }, { timestampMs: expected.endMs }, + { readings: frame.outputs[0].readings.map(row => row.metric === 'window_ibi_variability_v1' ? { ...row, metric: 'hrv_rmssd' } : row) }]) { + expect(() => protocol.accept({ ...frame, outputs: [{ ...frame.outputs[0], ...changes }] }, 'frame_result')).toThrow() + } + expect(protocol.accept(frame, 'frame_result').type).toBe('frame_result') + }) + it('requires the requested terminal command and closes after acknowledgment', () => { + const { protocol, ready, frame } = fixture() + protocol.accept(ready, 'ready') + protocol.accept(frame, 'frame_result') + const finish = { type: 'finished', runId: expected.runId, attemptId: expected.attemptId, outputs: [], health: { ...frame.health, state: 'completed' } } + expect(() => protocol.accept(finish, 'frame_result')).toThrow('command') + protocol.accept(finish, 'finished') + expect(() => protocol.accept(frame, 'frame_result')).toThrow('completion') + }) +}) +const python = resolve('modules/piezo-processor/.venv/bin/python') +it.skipIf(!existsSync(python))('validates actual locked-Python worker output across the process boundary', () => { + const configs = { left: defaultConfig, right: defaultConfig } + const configHashes = { left: configHash(defaultConfig), right: configHash(defaultConfig) } + const init = { ...expected, configHashes, schema: 'biometric-shadow-worker-v1', configs, + maximumInputBytes: 2 * 1024 * 1024, maximumOutputBytes: 512 * 1024 } + const messages: unknown[] = [init] + for (let second = 0; second < 150; second++) { + const left = Buffer.alloc(2000) + for (let index = 0; index < 500; index++) left.writeInt32LE(Math.trunc(Math.sin((second * 500 + index) / 500 * 2 * Math.PI * 1.2) * 1000000), index * 4) + messages.push({ type: 'frame', sequence: second, observedAtMs: start + second * 1000, + record: { type: 'piezo-dual', left1: left.toString('base64'), right1: Buffer.alloc(2000).toString('base64') }, + context: { activeSides: ['left'], leftOccupied: true, rightOccupied: false, pumpContextKnown: true } }) + } + messages.push({ type: 'finish' }) + const result = spawnSync(python, ['-m', 'common.biometrics.shadow_worker'], { env: { ...process.env, PYTHONPATH: resolve('modules') }, + input: messages.map(message => JSON.stringify(message)).join('\n') + '\n', encoding: 'utf8', timeout: 30000, maxBuffer: 2 * 1024 * 1024 }) + expect(result.status, result.stderr).toBe(0) + const replies = result.stdout.trim().split('\n').map(line => JSON.parse(line)) + const protocol = new ShadowWorkerProtocol({ ...expected, configHashes }) + let outputs = 0 + replies.forEach((reply, index) => { + const result = protocol.accept(reply, index === 0 ? 'ready' : index === replies.length - 1 ? 'finished' : 'frame_result') + if ('outputs' in result) outputs += result.outputs.length + }) + expect(replies).toHaveLength(152) + expect(outputs).toBeGreaterThan(0) +}, 30000) diff --git a/src/biometrics/shadow/worker-child.ts b/src/biometrics/shadow/worker-child.ts new file mode 100644 index 000000000..e86bd0ad1 --- /dev/null +++ b/src/biometrics/shadow/worker-child.ts @@ -0,0 +1,223 @@ +import type { ShadowPartialReceipt } from './receipt' +import { shadowChildBudgetsSchema as budgetsSchema } from './execution-policy' +import { confirmShadowProcessCleanup } from './process-cleanup' +import { ShadowCleanupError } from './cleanup-error' +import { spawn } from 'node:child_process' +import { tmpdir } from 'node:os' +import { z } from 'zod' +import { configHash } from '../algorithms/canonical' +import { validateConfig } from '../algorithms/registry' +import { sampleProcessGroupRss } from '../backtest/process-group-memory' +import { ShadowWorkerProtocol, type shadowWorkerOutputSchema } from './worker-protocol' + +const integer = z.number().int().positive().max(Number.MAX_SAFE_INTEGER) +const hash = z.string().regex(/^[a-f0-9]{64}$/) +const initSchema = z.object({ schema: z.literal('biometric-shadow-worker-v1'), + runId: z.string().uuid(), attemptId: z.string().uuid(), side: z.enum(['left', 'right']), + startMs: integer, endMs: integer, configs: z.object({ left: z.unknown(), right: z.unknown() }).strict(), + configHashes: z.object({ left: hash, right: hash }).strict(), maximumRecords: integer, + maximumOutputs: integer, maximumCpuMs: integer, maximumInputBytes: integer, maximumOutputBytes: integer, +}).strict() +export type ShadowChildBudgets = z.infer +type Output = z.infer +type Health = Extract, { type: 'finished' }>['health'] +const LINE_BYTES = 128 * 1024 + +/** Trusted host paths and shipped code only. One frame in flight; the source + * must itself bound buffering. Callers fence every output through the store. + * This supervisor never opens a database, source, network or hardware handle. */ +export async function runShadowChild(location: { pythonPath: string, modulesPath: string }, rawInit: unknown, + frames: AsyncIterable, onOutput: (output: Output) => void | Promise, + options: { budgets: ShadowChildBudgets, signal?: AbortSignal, onResources?: (receipt: ShadowPartialReceipt) => void }) { + const init = initSchema.parse(structuredClone(rawInit)) + const budgets = budgetsSchema.parse(options.budgets) + for (const side of ['left', 'right'] as const) { + validateConfig(init.configs[side]) + if (configHash(init.configs[side]) !== init.configHashes[side]) throw new Error('Shadow configuration identity mismatch') + } + const protocol = new ShadowWorkerProtocol({ runId: init.runId, attemptId: init.attemptId, side: init.side, + startMs: init.startMs, endMs: init.endMs, configHashes: init.configHashes, + maximumRecords: init.maximumRecords, maximumOutputs: init.maximumOutputs, maximumCpuMs: init.maximumCpuMs }) + if (options.signal?.aborted) throw new Error('Shadow worker cancelled') + const iterator = frames[Symbol.asyncIterator]() + const child = spawn(location.pythonPath, ['-B', '-m', 'common.biometrics.shadow_process', + '--parent-pid', String(process.pid), '--wall-ms', String(budgets.wallMs), '--cpu-seconds', String(budgets.cpuSeconds), '--address-space-bytes', String(budgets.addressSpaceBytes)], { + detached: true, cwd: tmpdir(), stdio: ['pipe', 'pipe', 'pipe'], + env: { NODE_ENV: 'production', PYTHONPATH: location.modulesPath, LANG: 'C.UTF-8', TZ: 'UTC', + OPENBLAS_NUM_THREADS: '1', OMP_NUM_THREADS: '1', MKL_NUM_THREADS: '1', NUMEXPR_NUM_THREADS: '1' }, + }) + let lastHealth: Health | null = null + let cleanupUnconfirmed = false + let failure: Error | undefined, closed = false, terminal = false, stopping = false + let inputBytes = 0, outputBytes = 0, diagnosticBytes = 0, peakRssBytes = 0, rssSamples = 0 + let lastRssAt = performance.now(), maximumObservationGapMs = 0 + let pending = Buffer.alloc(0) + let reply: { type: Parameters[1], resolve: (value: ReturnType) => void } | undefined + let rejectFailure!: (error: Error) => void + const failed = new Promise((_resolve, reject) => { + rejectFailure = reject + }) + // A source/sink can be awaiting its own I/O when the process fails. + void failed.catch(() => {}) + const fail = (message: string) => { + if (!failure) { + failure = new Error(message) + rejectFailure(failure) + } + } + const abort = () => fail('Shadow worker cancelled') + options.signal?.addEventListener('abort', abort, { once: true }) + if (options.signal?.aborted) abort() + const wall = setTimeout(() => fail('Shadow worker wall deadline exceeded'), budgets.wallMs) + const exited = new Promise((resolve) => { + child.once('close', (code) => { + closed = true + if (!stopping && (code !== 0 || !terminal || pending.length)) fail('Shadow worker failed or returned an incomplete result') + resolve() + }) + }) + child.on('error', () => fail('Shadow worker could not start')) + child.stdin.on('error', () => fail('Shadow worker input pipe closed')) + child.stdout.on('error', () => fail('Shadow worker output pipe failed')) + child.stderr.on('error', () => fail('Shadow worker diagnostic pipe failed')) + child.stderr.on('data', (chunk: Buffer) => { + diagnosticBytes += chunk.length + if (diagnosticBytes > 16384) fail('Shadow diagnostics exceed byte quota') + }) + child.stdout.on('data', (chunk: Buffer) => { + if (failure) return + outputBytes += chunk.length + if (outputBytes > init.maximumOutputBytes) return fail('Shadow output exceeds byte quota') + // Only one reply is permitted per command; never queue unsolicited output. + const newline = chunk.indexOf(10) + if (pending.length + (newline < 0 ? chunk.length : newline + 1) > LINE_BYTES) return fail('Shadow response line exceeds byte quota') + pending = Buffer.concat([pending, chunk]) + const end = pending.indexOf(10) + if (end < 0) return + if (!reply || end !== pending.length - 1) return fail('Shadow worker emitted unsolicited output') + try { + const line = new TextDecoder('utf-8', { fatal: true }).decode(pending.subarray(0, end)) + const response = protocol.accept(JSON.parse(line), reply.type) + if ('health' in response) lastHealth = structuredClone(response.health) + const acknowledge = reply.resolve + reply = undefined + pending = Buffer.alloc(0) + if (response.type === 'finished') terminal = true + acknowledge(response) + } + catch { fail('Shadow worker emitted an invalid response') } + }) + async function bounded(operation: Promise, ms: number, reason: string): Promise { + const timer = setTimeout(() => fail(reason), ms) + try { + return await Promise.race([operation, failed]) + } + finally { clearTimeout(timer) } + } + let monitoring = false + const sample = async () => { + if (monitoring || closed || failure) return + monitoring = true + try { + const rss = await bounded(Promise.race([sampleProcessGroupRss(child.pid ?? 0), exited.then(() => null)]), budgets.rssTimeoutMs, 'Shadow RSS observation timed out') + if (closed || stopping || failure) return + // procfs can lose a finished worker before Node delivers pipe closure. + // Accept that disappearance only after a validated terminal response and + // a bounded, successful close; the close handler still rejects bad exits. + if (rss === null && terminal) { + await bounded(exited, budgets.acknowledgmentMs, 'Shadow worker exit deadline exceeded') + return + } + if (rss === null || !Number.isSafeInteger(rss) || rss <= 0) return fail('Shadow RSS observation unavailable') + const observedAt = performance.now() + maximumObservationGapMs = Math.max(maximumObservationGapMs, observedAt - lastRssAt) + lastRssAt = observedAt + rssSamples++ + peakRssBytes = Math.max(peakRssBytes, rss) + if (rss > budgets.maximumRssBytes) fail('Shadow RSS quota exceeded') + } + catch { + if (!closed && !stopping) fail('Shadow RSS observation unavailable') + } + finally { monitoring = false } + } + const memoryTimer = setInterval(() => { + void sample() + }, budgets.rssIntervalMs) + child.once('spawn', () => { + void sample() + }) + async function request(message: unknown, type: Parameters[1]) { + if (failure) throw failure + if (closed || terminal || reply) throw new Error('Shadow worker is not accepting commands') + const line = JSON.stringify(message) + '\n' + const bytes = Buffer.byteLength(line) + if (bytes > LINE_BYTES || inputBytes + bytes > init.maximumInputBytes) throw new Error('Shadow input exceeds byte quota') + inputBytes += bytes + const response = new Promise>((resolve) => { + reply = { type, resolve } + }) + child.stdin.write(line) + return bounded(response, budgets.acknowledgmentMs, 'Shadow acknowledgment deadline exceeded') + } + let result: { health: Health, inputBytes: number, outputBytes: number, peakObservedRssBytes: number, rssSamples: number, maximumObservationGapMs: number, budgets: ShadowChildBudgets, memoryScope: 'sum_rss_of_shadow_process_group', exactKernelPeak: false } | undefined + try { + await request(init, 'ready') + let records = 0 + while (true) { + const next = await bounded(Promise.resolve().then(() => iterator.next()), budgets.idleMs, 'Shadow source idle deadline exceeded') + if (next.done) break + if (records >= init.maximumRecords) throw new Error('Shadow record quota exceeded') + const frame = z.object({ type: z.literal('frame'), sequence: z.literal(records) }).passthrough().parse(next.value) + const response = await request(frame, 'frame_result') + if (response.type !== 'frame_result') throw new Error('Shadow frame acknowledgment missing') + for (const output of response.outputs) await Promise.race([Promise.resolve().then(() => onOutput(output)), failed]) + records++ + } + const response = await request({ type: 'finish' }, 'finished') + if (response.type !== 'finished') throw new Error('Shadow completion missing') + for (const output of response.outputs) await Promise.race([Promise.resolve().then(() => onOutput(output)), failed]) + child.stdin.end() + await bounded(exited, budgets.acknowledgmentMs, 'Shadow worker exit deadline exceeded') + if (!rssSamples) throw new Error('Shadow worker has no RSS observations') + if (failure) throw failure + result = { health: response.health, inputBytes, outputBytes, peakObservedRssBytes: peakRssBytes, rssSamples, maximumObservationGapMs: Math.max(maximumObservationGapMs, performance.now() - lastRssAt), + budgets, memoryScope: 'sum_rss_of_shadow_process_group', exactKernelPeak: false } + } + catch (error) { + fail(error instanceof Error ? error.message : 'Shadow source or sink failed') + } + finally { + stopping = true + clearTimeout(wall) + clearInterval(memoryTimer) + options.signal?.removeEventListener('abort', abort) + // Terminate the entire detached group, including a child retaining pipes. + // SIGKILL is deliberate: no canonical state exists in this worker to flush. + if (child.pid) { + try { + process.kill(-child.pid, 'SIGKILL') + } + catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ESRCH') cleanupUnconfirmed = true + } + } + child.stdin.destroy() + child.stdout.destroy() + child.stderr.destroy() + // Iterator cleanup may itself be stalled; never let it keep a worker alive. + if (iterator.return) void Promise.resolve().then(() => iterator.return?.()).catch(() => {}) + if (!await confirmShadowProcessCleanup(child.pid, () => closed)) cleanupUnconfirmed = true + } + try { + options.onResources?.({ kind: 'partial_supervisor_observation', health: lastHealth, + inputBytes, outputBytes, diagnosticBytes, peakObservedRssBytes: rssSamples ? peakRssBytes : null, rssSamples, + maximumObservationGapMs: Math.max(maximumObservationGapMs, performance.now() - lastRssAt), budgets, + cleanupConfirmed: !cleanupUnconfirmed, memoryScope: 'sum_rss_of_shadow_process_group', exactKernelPeak: false }) + } + catch (error) { fail(error instanceof Error ? error.message : 'Shadow resource observation failed') } + if (cleanupUnconfirmed) throw new ShadowCleanupError(failure) + if (failure) throw failure + if (!result) throw new Error('Shadow completion missing') + return result +} diff --git a/src/biometrics/shadow/worker-protocol.ts b/src/biometrics/shadow/worker-protocol.ts new file mode 100644 index 000000000..fb8fc6887 --- /dev/null +++ b/src/biometrics/shadow/worker-protocol.ts @@ -0,0 +1,87 @@ +import { z } from 'zod' +import { configHash } from '../algorithms/canonical' + +const integer = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) +const hash = z.string().regex(/^[a-f0-9]{64}$/) +const side = z.enum(['left', 'right']) +const reading = { value: z.number().finite().nullable(), quality: z.number().min(0).max(1) } +const metricReading = z.discriminatedUnion('metric', [ + z.object({ metric: z.literal('heart_rate'), unit: z.literal('bpm'), windowMs: z.null(), ...reading }).strict(), + z.object({ metric: z.literal('breathing_rate'), unit: z.literal('breaths/min'), windowMs: z.null(), ...reading }).strict(), + z.object({ metric: z.literal('window_ibi_variability_v1'), unit: z.literal('ms'), windowMs: z.literal(300000), ...reading }).strict(), +]) +export const shadowWorkerOutputSchema = z.object({ sequence: integer, inputHash: hash, side, sourceSide: side, + timestampMs: integer, configHash: hash, readings: z.array(metricReading).length(3), + flags: z.array(z.string().max(200)).max(32).nullable(), limitations: z.array(z.string().max(200)).max(32), +}).strict().refine(output => new Set(output.readings.map(row => row.metric)).size === 3, 'All typed metrics required') +export const shadowWorkerHealthSchema = z.object({ state: z.enum(['running', 'completed', 'stopped']), reason: z.string().max(500).nullable(), + records: integer, outputs: integer, cpu_ms: z.number().finite().nonnegative(), input_hash: hash, + counts: z.object({ records: integer, accepted_frames: integer, empty_channel_frames: integer, + pump_gated_frames: integer, preroll_outputs: integer, scored_outputs: integer }).strict(), + limitations: z.array(z.string().max(200)).max(32), +}).strict() +const identity = { runId: z.string().uuid(), attemptId: z.string().uuid() } +const hashes = z.object({ left: hash, right: hash }).strict() +const responseSchema = z.discriminatedUnion('type', [ + z.object({ ...identity, type: z.literal('ready'), configHashes: hashes }).strict(), + z.object({ ...identity, type: z.literal('frame_result'), sequence: integer, outputs: z.array(shadowWorkerOutputSchema).max(4), health: shadowWorkerHealthSchema }).strict(), + z.object({ ...identity, type: z.literal('finished'), outputs: z.array(shadowWorkerOutputSchema).max(4), health: shadowWorkerHealthSchema }).strict(), + z.object({ ...identity, type: z.literal('cancelled'), health: shadowWorkerHealthSchema }).strict(), +]) +const expectedSchema = z.object({ ...identity, side, startMs: integer, endMs: integer, configHashes: hashes, + maximumRecords: integer.positive(), maximumOutputs: integer.positive(), maximumCpuMs: integer.positive(), +}).strict().refine(value => value.startMs < value.endMs, 'Invalid shadow response window') + +/** Stateful parent-side validation. A rejected batch changes no counters, so + * callers cannot accidentally accept partial unvalidated output from a reply. */ +export class ShadowWorkerProtocol { + private readonly expected: z.infer + private ready = false + private closed = false + private records = 0 + private outputs = 0 + private lastTimestamp = -1 + private cpuMs = 0 + + constructor(expected: unknown) { + this.expected = expectedSchema.parse(structuredClone(expected)) + } + + accept(raw: unknown, expectedType: 'ready' | 'frame_result' | 'finished' | 'cancelled') { + if (this.closed) throw new Error('Shadow response after completion') + const response = responseSchema.parse(raw) + const expected = this.expected + if (response.runId !== expected.runId || response.attemptId !== expected.attemptId || response.type !== expectedType) throw new Error('Shadow response identity or command mismatch') + if (response.type === 'ready') { + if (this.ready || configHash(response.configHashes) !== configHash(expected.configHashes)) throw new Error('Shadow ready identity mismatch') + this.ready = true + return response + } + if (!this.ready) throw new Error('Shadow worker has not acknowledged initialization') + const records = this.records + (response.type === 'frame_result' ? 1 : 0) + if (response.type === 'frame_result' && response.sequence !== this.records) throw new Error('Shadow acknowledgment sequence mismatch') + const batch = response.type === 'cancelled' ? [] : response.outputs + const outputCount = this.outputs + batch.length + if (records > expected.maximumRecords || outputCount > expected.maximumOutputs) throw new Error('Shadow response exceeds run quota') + let timestamp = this.lastTimestamp + for (const [index, output] of batch.entries()) { + if (output.sequence !== this.outputs + index || output.side !== expected.side || output.configHash !== expected.configHashes[expected.side] + || output.timestampMs < expected.startMs || output.timestampMs >= expected.endMs || output.timestampMs <= timestamp) throw new Error('Shadow output identity, order or window mismatch') + timestamp = output.timestampMs + } + const health = response.health + const state = response.type === 'frame_result' ? 'running' : response.type === 'finished' ? 'completed' : 'stopped' + if (health.state !== state || health.records !== records || health.counts.records !== records || health.outputs !== outputCount + || health.cpu_ms < this.cpuMs || health.cpu_ms > expected.maximumCpuMs + || health.counts.accepted_frames + health.counts.empty_channel_frames + health.counts.pump_gated_frames > records + || health.counts.scored_outputs < outputCount + || health.counts.scored_outputs + health.counts.preroll_outputs > 2 * health.counts.accepted_frames + || batch.some(output => output.inputHash !== health.input_hash)) throw new Error('Shadow health receipt inconsistent') + this.records = records + this.outputs = outputCount + this.lastTimestamp = timestamp + this.cpuMs = health.cpu_ms + this.closed = response.type !== 'frame_result' + return response + } +} diff --git a/src/biometrics/shadow/worker-sink.ts b/src/biometrics/shadow/worker-sink.ts new file mode 100644 index 000000000..25a36a2ac --- /dev/null +++ b/src/biometrics/shadow/worker-sink.ts @@ -0,0 +1,48 @@ +import { z } from 'zod' +import type { ShadowStore } from './store' +import { type ShadowLease, shadowActiveProvenanceSchema } from './store' +import { shadowWorkerOutputSchema } from './worker-protocol' + +export const shadowActiveReadingSchema = z.object({ side: z.enum(['left', 'right']), timestampMs: z.number().int().nonnegative(), + provenance: shadowActiveProvenanceSchema.optional(), + configHash: z.string().regex(/^[a-f0-9]{64}$/), inputHash: z.string().regex(/^[a-f0-9]{64}$/).nullable().optional(), readings: shadowWorkerOutputSchema.shape.readings, +}).strict().refine(value => new Set(value.readings.map(row => row.metric)).size === 3, 'All active metric identities required') + +/** Fenced candidate-output sink. The supervisor validates response batches with + * ShadowWorkerProtocol first and supplies actual contemporaneous active data. + * Missing active data stays missing; no nearest-time substitution is allowed. */ +export class ShadowWorkerSink { + private readonly baseSequence: number + private nextOutput = 0 + private readonly plan: ReturnType['plan'] + + constructor(private readonly store: ShadowStore, private readonly lease: ShadowLease) { + const run = store.get(lease.runId) + if (run.state !== 'running' || run.attempt_id !== lease.attemptId) throw new Error('Shadow sink attempt mismatch') + this.baseSequence = run.point_count + this.plan = structuredClone(run.plan) + } + + append(rawOutput: unknown, rawActive: unknown | null) { + const output = shadowWorkerOutputSchema.parse(rawOutput) + const active = rawActive === null ? null : shadowActiveReadingSchema.parse(rawActive) + if (output.sequence > this.nextOutput || output.side !== this.plan.side || output.configHash !== this.plan.candidateConfigHash) throw new Error('Shadow sink output identity mismatch') + if (active && (active.side !== this.plan.side || active.configHash !== this.plan.activeConfigHash || active.timestampMs !== output.timestampMs)) throw new Error('Active reading is not contemporaneous with shadow output') + const readings = this.plan.metrics.map((metric) => { + const candidate = output.readings.find(row => row.metric === metric) + const baseline = active?.readings.find(row => row.metric === metric) + if (!candidate || (active && !baseline)) throw new Error('Missing sink metric') + const expectedWindow = metric === 'window_ibi_variability_v1' ? this.plan.windowIbiWindowMs : null + if (candidate.windowMs !== expectedWindow || (baseline && baseline.windowMs !== expectedWindow)) throw new Error('Shadow sink metric window mismatch') + return { metric, candidate: { value: candidate.value, quality: candidate.quality }, + active: baseline ? { value: baseline.value, quality: baseline.quality } : { value: null, quality: 0 } } + }) + const result = this.store.append(this.lease, { sequence: this.baseSequence + output.sequence, side: output.side, timestampMs: output.timestampMs, + inputHash: output.inputHash, candidateConfigHash: output.configHash, activeConfigHash: this.plan.activeConfigHash, readings, + ...(active?.provenance ? { activeProvenance: active.provenance } : {}), + candidateProvenance: { sourceSide: output.sourceSide, outputSequence: output.sequence, flags: output.flags, + limitations: active ? output.limitations : [...output.limitations, 'missing_contemporaneous_active_reading'] } }) + if (result.state !== 'quota_reached') this.nextOutput = Math.max(this.nextOutput, output.sequence + 1) + return result + } +} diff --git a/src/biometrics/tests/background-admission.test.ts b/src/biometrics/tests/background-admission.test.ts new file mode 100644 index 000000000..d29c87980 --- /dev/null +++ b/src/biometrics/tests/background-admission.test.ts @@ -0,0 +1,45 @@ +// @vitest-environment node +import { afterEach, expect, it, vi } from 'vitest' +import { withBiometricBackgroundSlot } from '../background-admission' +const shared = globalThis as typeof globalThis & { __biometricBackgroundSlotsV1?: unknown } +afterEach(() => { + delete shared.__biometricBackgroundSlotsV1 +}) +it('reserves work synchronously while leaving one report read available', async () => { + let finish!: () => void + const work = withBiometricBackgroundSlot('work', () => new Promise((resolve) => { + finish = resolve + })) + const other = vi.fn(async () => null) + await expect(withBiometricBackgroundSlot('work', other)).rejects.toMatchObject({ code: 'RESOURCE_LIMIT' }) + expect(other).not.toHaveBeenCalled() + expect(await withBiometricBackgroundSlot('read', async () => 'report')).toBe('report') + finish() + await work + expect(await withBiometricBackgroundSlot('work', other)).toBeNull() +}) +it('shares ownership across module reloads and releases ordinary failures', async () => { + let finish!: () => void + const work = withBiometricBackgroundSlot('work', () => new Promise((resolve) => { + finish = resolve + })) + vi.resetModules() + const copy = await import('../background-admission') + await expect(copy.withBiometricBackgroundSlot('work', async () => null)).rejects.toMatchObject({ code: 'RESOURCE_LIMIT' }) + finish() + await work + await expect(copy.withBiometricBackgroundSlot('work', async () => { + throw new Error('failed') + })).rejects.toThrow('failed') + expect(() => withBiometricBackgroundSlot('work', () => { + throw new Error('synchronous') + })).toThrow('synchronous') + expect(await withBiometricBackgroundSlot('work', async () => 'available')).toBe('available') +}) +it.each(['SHADOW_CLEANUP_UNCONFIRMED', 'BIOMETRIC_COORDINATOR_CLEANUP_UNCONFIRMED'])('quarantines uncertain cleanup: %s', async (code) => { + await expect(withBiometricBackgroundSlot('work', async () => { + throw Object.assign(new Error('cleanup'), { code }) + })).rejects.toMatchObject({ code }) + await expect(withBiometricBackgroundSlot('work', async () => null)).rejects.toThrow('cleanup is unconfirmed') + expect(await withBiometricBackgroundSlot('read', async () => 'diagnostics')).toBe('diagnostics') +}) diff --git a/src/components/Autopilot/builderModel.ts b/src/components/Autopilot/builderModel.ts index 5cbdad052..0759a839f 100644 --- a/src/components/Autopilot/builderModel.ts +++ b/src/components/Autopilot/builderModel.ts @@ -62,6 +62,10 @@ export const SIGNALS: SignalDef[] = [ // Capacitive presence sensing (not temperature): per-zone body-contact load. // Back-testable from the downsampled cap_sense_frames history (see automations // loadSeries); the live zone matrix is replayable too but not a scalar signal. + // Virtual occupancy sensor: 1 = a person is in bed, 0 = empty or a light load + // (pet). Live-only — the classified signal has no history table. + { id: '{side}.presence', label: 'Presence (person)', unit: '', kind: 'num', group: 'Presence', icon: 'Bed', perSide: true, max: 1, liveOnly: true }, + { id: '{side}.presence.zones', label: 'Occupied zones', unit: '', kind: 'num', group: 'Presence', icon: 'Bed', perSide: true, max: 3, liveOnly: true }, { id: '{side}.cap.max', label: 'Bed pressure (peak)', unit: '', kind: 'num', group: 'Presence', icon: 'Activity', perSide: true }, { id: '{side}.cap.mean', label: 'Bed pressure (avg)', unit: '', kind: 'num', group: 'Presence', icon: 'Activity', perSide: true }, { id: '{side}.cap.spread', label: 'Pressure spread', unit: '', kind: 'num', group: 'Presence', icon: 'Activity', perSide: true }, @@ -507,14 +511,16 @@ export function buildSentence(r: BuilderRule, timeFormat: TimeFormat = '12h'): S // --------------------------------------------------------------------------- /** - * Signals the engine reads live today (ADR 0023): per-side temperature/level - * and water.low. Biometric, ambient and enum signals are backtest only, so a - * rule reading them can never fire on the bed yet. + * Signals the engine reads live today (ADR 0023): per-side temperature/level, + * presence and water.low. Other biometric, ambient and enum signals are + * backtest only, so a rule reading them can never fire on the bed yet. */ export const LIVE_SIGNALS: ReadonlySet = new Set([ '{side}.currentTemperature', '{side}.targetTemperature', '{side}.currentLevel', + '{side}.presence', + '{side}.presence.zones', 'water.low', ]) diff --git a/src/components/Autopilot/tests/automationsLogic.test.ts b/src/components/Autopilot/tests/automationsLogic.test.ts index 210aaf582..e6830e96c 100644 --- a/src/components/Autopilot/tests/automationsLogic.test.ts +++ b/src/components/Autopilot/tests/automationsLogic.test.ts @@ -3,7 +3,7 @@ import { backtestLine, currentNightStart, entryTime, matchesFilter, nightHeader, nightStarts, ownerView, reasonText, ruleMode, ruleWindow, scheduleBands, scheduleBlocks, statusLine, type SideTonight, } from '../automationsLogic' -import { missingLiveSignals, templateRule, TEMPLATES } from '../builderModel' +import { blankRule, missingLiveSignals, templateRule, TEMPLATES } from '../builderModel' const t = (d: number, h: number, m = 0) => new Date(2026, 8, d, h, m).getTime() const fmt = (f: number) => `${f}°` @@ -157,6 +157,9 @@ describe('rule rows', () => { expect(missing('restless')).toEqual(['movement']) expect(missing('water-low')).toEqual([]) expect(missing('hold-room')).toEqual([]) + const presenceRule = { ...blankRule(), when: { type: 'agg' as const, agg: 'avg' as const, signal: '{side}.presence', window: 5, op: '>' as const, value: 0 } } + expect(missingLiveSignals(presenceRule)).toEqual([]) + expect(missingLiveSignals({ ...presenceRule, when: { ...presenceRule.when, signal: '{side}.presence.zones' } })).toEqual([]) expect(templateRule('nope')).toBeNull() expect(TEMPLATES.map(x => x.title)).toEqual(['Hold room +3°F', 'Cool when restless', 'Tell me when water is low']) }) diff --git a/src/components/Schedule/AICurveWizard.tsx b/src/components/Schedule/AICurveWizard.tsx index e232bfe42..2f0af619d 100644 --- a/src/components/Schedule/AICurveWizard.tsx +++ b/src/components/Schedule/AICurveWizard.tsx @@ -4,7 +4,6 @@ import { useTimeFormat } from '@/src/providers/TimeFormatProvider' import { TimeInput } from './TimeInput' import { useCallback, useEffect, useState } from 'react' import { - X, Sparkles, Copy, Check, @@ -19,16 +18,14 @@ import { import { cn } from '@/lib/utils' import { Alert, Button, GhostIcon, Modal, SectionLabel, Stepper } from '@/src/components/ds' import { useTemperatureUnit } from '@/src/hooks/useTemperatureUnit' +import { useCurveTemplates } from '@/src/hooks/useCurveTemplates' import { displayToSetpointF, setpointFToDisplay } from '@/src/lib/tempUtils' import { generatePrompt, parseAIResponse, EXAMPLE_SUGGESTIONS, - loadTemplates, - saveTemplate, - deleteTemplate, } from '@/src/lib/sleepCurve/curvePrompt' -import type { GeneratedCurve, CurveTemplate, ParseResult } from '@/src/lib/sleepCurve/curvePrompt' +import type { GeneratedCurve, ParseResult } from '@/src/lib/sleepCurve/curvePrompt' import { CurveChart } from './CurveChart' import { TONE_VAR, tempTone } from './scheduleFormat' @@ -70,15 +67,10 @@ export function AICurveWizard({ open, onClose, onApply }: AICurveWizardProps) { // Step 4: Preview const [curve, setCurve] = useState(null) const [editablePoints, setEditablePoints] = useState>([]) - const [savedTemplates, setSavedTemplates] = useState([]) + const { save: saveTemplate, isSaving } = useCurveTemplates(open) + const [templateError, setTemplateError] = useState(null) /* eslint-disable react-hooks/set-state-in-effect */ - useEffect(() => { - if (open) { - setSavedTemplates(loadTemplates()) - } - }, [open]) - useEffect(() => { if (!open) { setStep(0) @@ -90,6 +82,7 @@ export function AICurveWizard({ open, onClose, onApply }: AICurveWizardProps) { setParseResult(null) setCurve(null) setEditablePoints([]) + setTemplateError(null) } }, [open]) @@ -190,30 +183,19 @@ export function AICurveWizard({ open, onClose, onApply }: AICurveWizardProps) { } }, []) - const handleLoadTemplate = useCallback((template: CurveTemplate) => { - setCurve(template) - setEditablePoints( - Object.entries(template.points) - .map(([time, tempF]) => ({ time, tempF })) - .sort((a, b) => a.time.localeCompare(b.time)), - ) - setStep(3) - setHighestStep(3) - }, []) - - const handleDeleteTemplate = useCallback((name: string) => { - deleteTemplate(name) - setSavedTemplates(loadTemplates()) - }, []) - - const handleSaveTemplate = useCallback(() => { + const handleSaveTemplate = useCallback(async () => { if (!curve) return const pointsObj: Record = {} for (const p of editablePoints) pointsObj[p.time] = p.tempF const updated: GeneratedCurve = { ...curve, points: pointsObj } - saveTemplate(updated) - setSavedTemplates(loadTemplates()) - }, [curve, editablePoints]) + setTemplateError(null) + try { + await saveTemplate(updated) + } + catch (err) { + setTemplateError(err instanceof Error ? err.message : 'Save failed') + } + }, [curve, editablePoints, saveTemplate]) const updatePoint = useCallback((idx: number, field: 'time' | 'tempF', value: string | number) => { setEditablePoints((prev) => { @@ -280,7 +262,7 @@ export function AICurveWizard({ open, onClose, onApply }: AICurveWizardProps) { )} {step === 3 && ( <> - + @@ -312,9 +294,6 @@ export function AICurveWizard({ open, onClose, onApply }: AICurveWizardProps) { )} {step === 1 && ( @@ -343,6 +322,7 @@ export function AICurveWizard({ open, onClose, onApply }: AICurveWizardProps) { onRemovePoint={removePoint} /> )} + {step === 3 && templateError && {`Could not save the curve: ${templateError}`}} ) } @@ -352,15 +332,9 @@ const FIELD = 'w-full rounded-ctl border border-line-2 bg-field px-3 py-[9px] te function StepDescribe({ preferences, onPreferencesChange, - templates, - onLoadTemplate, - onDeleteTemplate, }: { preferences: string onPreferencesChange: (v: string) => void - templates: CurveTemplate[] - onLoadTemplate: (t: CurveTemplate) => void - onDeleteTemplate: (name: string) => void }) { return (
@@ -392,24 +366,6 @@ function StepDescribe({ ))}
- {templates.length > 0 && ( -
- Saved curves - {templates.map(t => ( -
- - onDeleteTemplate(t.name)} /> -
- ))} -
- )} ) } diff --git a/src/components/Schedule/CurveCard.tsx b/src/components/Schedule/CurveCard.tsx index 7f11b0f3b..599e7a0fb 100644 --- a/src/components/Schedule/CurveCard.tsx +++ b/src/components/Schedule/CurveCard.tsx @@ -27,6 +27,8 @@ interface CurveCardProps { featured?: boolean /** Featured only: measured bed temperature on the chart's minute axis, with its legend label ("last night"). */ bed?: { samples: BedSample[], label: string } + /** The curve's optional note. */ + notes?: string } /** "79–84°F" / "79–84°" — set-point range in the user's unit. */ @@ -145,7 +147,7 @@ function stop(fn: () => void) { } } -export function CurveCard({ group, onEdit, onDelete, isActive = false, nextEvent = null, featured = false, bed }: CurveCardProps) { +export function CurveCard({ group, onEdit, onDelete, isActive = false, nextEvent = null, featured = false, bed, notes }: CurveCardProps) { const { timeFormat } = useTimeFormatter() const { unit } = useTemperatureUnit() const hasSetPoints = group.setPoints.length > 0 @@ -188,12 +190,14 @@ export function CurveCard({ group, onEdit, onDelete, isActive = false, nextEvent
+ {notes &&

{notes}

}
{/* Phone: sparkline + next set point */}
{meta} + {notes &&

{notes}

} {active && nextEvent && ( Next @@ -226,6 +230,7 @@ export function CurveCard({ group, onEdit, onDelete, isActive = false, nextEvent
No set points active
) : hasSetPoints && } + {notes &&

{notes}

} {paused ? 'Schedule paused' diff --git a/src/components/Schedule/CurveEditor.tsx b/src/components/Schedule/CurveEditor.tsx index ef877353d..299a0bd83 100644 --- a/src/components/Schedule/CurveEditor.tsx +++ b/src/components/Schedule/CurveEditor.tsx @@ -23,6 +23,7 @@ import { timeStringToMinutes, } from '@/src/lib/sleepCurve/generate' import { rebaseSetPoints } from '@/src/lib/sleepCurve/rebase' +import { CURVE_NOTES_MAX } from '@/src/lib/sleepCurve/savePlan' import { type ScheduleEndAction, simplifySetPoints, sortChronological } from '@/src/lib/scheduleGrouping' import { useTemperatureUnit } from '@/src/hooks/useTemperatureUnit' import { displayToSetpointF, setpointFToDisplay } from '@/src/lib/tempUtils' @@ -34,6 +35,8 @@ interface CurveEditorProps { initialEndAction?: ScheduleEndAction /** Initial set points. Empty array = create mode with empty list. */ initialSetPoints?: Array<{ time: string, temperature: number }> + /** The curve's optional note. */ + initialNotes?: string } interface PresetDef { @@ -81,6 +84,7 @@ export function CurveEditor({ initialDays = [], initialSetPoints = [], initialEndAction = 'turn_off', + initialNotes = '', }: CurveEditorProps) { const { saveCurve, detectCurveConflicts, isMutating, allSchedules } = useSchedule() const { unit } = useTemperatureUnit() @@ -108,6 +112,7 @@ export function CurveEditor({ simplifySetPoints(initialSetPoints).map((p, i) => ({ id: -(i + 1), time: p.time, temperature: p.temperature })), ) const [endAction, setEndAction] = useState(initialEndAction) + const [notes, setNotes] = useState(initialNotes) const [bedtime, setBedtime] = useState(initial.bedtime) const [wakeTime, setWakeTime] = useState(initial.wake) const [minTemp, setMinTemp] = useState(initial.min) @@ -242,6 +247,7 @@ export function CurveEditor({ setPoints: points.map(p => ({ time: p.time, temperature: p.temperature })), originalDays: initialDays, endAction, + notes, }) setPendingConflict(null) onClose() @@ -250,7 +256,7 @@ export function CurveEditor({ setPendingConflict(null) setSaveError(err instanceof Error ? err.message : 'Save failed') } - }, [days, points, initialDays, endAction, detectCurveConflicts, saveCurve, onClose]) + }, [days, points, initialDays, endAction, notes, detectCurveConflicts, saveCurve, onClose]) const editingPoint = editingId !== null ? points.find(p => p.id === editingId) ?? null : null const title = isEdit ? 'Edit curve' : 'New curve' @@ -436,6 +442,22 @@ export function CurveEditor({ )} + + +