diff --git a/.github/workflows/mutation.yml b/.github/workflows/mutation.yml index f54ec7a6d..de4bd4669 100644 --- a/.github/workflows/mutation.yml +++ b/.github/workflows/mutation.yml @@ -59,6 +59,8 @@ jobs: mutate: "src/streaming/**/*.ts,!src/**/tests/**,!src/**/*.test.ts,!src/**/*.test.tsx,!src/**/*.d.ts" - name: hooks mutate: "src/hooks/**/*.ts,src/hooks/**/*.tsx,!src/**/tests/**,!src/**/*.test.ts,!src/**/*.test.tsx,!src/**/*.d.ts" + - name: biometrics + mutate: "src/biometrics/**/*.ts,!src/**/tests/**,!src/**/*.test.ts,!src/**/*.test.tsx,!src/**/*.d.ts" - name: db mutate: "src/db/**/*.ts,!src/**/tests/**,!src/**/*.test.ts,!src/**/*.test.tsx,!src/**/*.d.ts" name: mutate (${{ matrix.shard.name }}) diff --git a/.github/workflows/python-modules.yml b/.github/workflows/python-modules.yml index 1704d5073..d27c5d54c 100644 --- a/.github/workflows/python-modules.yml +++ b/.github/workflows/python-modules.yml @@ -5,6 +5,8 @@ on: branches: [dev] paths: - 'modules/**/*.py' + - 'modules/common/biometrics/*.json' + - 'modules/piezo-processor/fixtures/*.json' - 'modules/**/pyproject.toml' - 'modules/**/uv.lock' - 'modules/.coveragerc' @@ -14,6 +16,8 @@ on: types: [opened, synchronize, reopened] paths: - 'modules/**/*.py' + - 'modules/common/biometrics/*.json' + - 'modules/piezo-processor/fixtures/*.json' - 'modules/**/pyproject.toml' - 'modules/**/uv.lock' - 'modules/.coveragerc' @@ -50,9 +54,9 @@ jobs: workdir: modules target: common cov_source: common - # cbor2 is needed to decode the real NATS capture fixtures through - # NatsFollower's decode path (test_nats_follower). - deps: "cbor2" + # Build-identity tests inspect installed NumPy/SciPy metadata; + # use the piezo module's locked environment, including cbor2. + deps: "" - name: cover-buttons workdir: modules/cover-buttons target: . @@ -72,6 +76,7 @@ jobs: workdir: modules/piezo-processor target: . cov_source: . + cov_extra: "--cov=../common/biometrics" deps: "numpy scipy" - name: sleep-detector workdir: modules/sleep-detector @@ -95,7 +100,7 @@ jobs: with: # Pod runtime is pinned to 3.10 by every module's pyproject # (`requires-python = ">=3.9,<3.11"`). Match it in CI. - python-version: '3.14' + python-version: '3.10' - name: Install uv uses: astral-sh/setup-uv@v4 @@ -105,7 +110,10 @@ jobs: - name: Install test deps working-directory: ${{ matrix.module.workdir }} run: | - if [ -f uv.lock ]; then + if [ "${{ matrix.module.name }}" = "common" ]; then + # Keep common's cwd/import layout while using the numerical lock. + UV_PROJECT_ENVIRONMENT="$PWD/.venv" uv sync --project piezo-processor --locked --no-install-project + elif [ -f uv.lock ]; then uv sync --locked --no-install-project else uv venv .venv @@ -125,6 +133,7 @@ jobs: pytest ${{ matrix.module.target }} \ --cov-config=${{ github.workspace }}/modules/.coveragerc \ --cov=${{ matrix.module.cov_source }} \ + ${{ matrix.module.cov_extra }} \ --cov-report=xml:coverage.xml \ --junit-xml=junit.xml \ -o junit_family=legacy diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index eba7cbba0..89ea07cd5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -29,6 +29,7 @@ jobs: - { name: "Lint", cmd: "CHANGED=$(git diff --name-only --diff-filter=d origin/${BASE_REF#refs/heads/}...HEAD -- '*.ts' '*.tsx' | head -200); if [ -n \"$CHANGED\" ]; then echo \"$CHANGED\" | xargs -d '\\n' pnpm eslint --; else echo 'No TS/TSX files changed'; fi" } - { name: "Typecheck", cmd: "pnpm tsc" } - { name: "Unit Tests", cmd: "pnpm test run --coverage --passWithNoTests" } + - { name: "Browser Tests", cmd: "NODE_ENV=production CI=1 pnpm build && pnpm test:browser" } steps: - name: Checkout @@ -51,6 +52,27 @@ jobs: - name: Install Dependencies run: pnpm install --frozen-lockfile + - name: Setup Python for runtime integration + if: matrix.task.name == 'Unit Tests' || matrix.task.name == 'Browser Tests' + uses: actions/setup-python@v6 + with: + python-version: '3.10' + + - name: Setup uv for runtime integration + if: matrix.task.name == 'Unit Tests' || matrix.task.name == 'Browser Tests' + uses: astral-sh/setup-uv@v4 + with: + enable-cache: true + + - name: Install locked runtime integration dependencies + if: matrix.task.name == 'Unit Tests' || matrix.task.name == 'Browser Tests' + working-directory: modules/piezo-processor + run: uv sync --locked --no-install-project + + - name: Install browser + if: matrix.task.name == 'Browser Tests' + run: pnpm exec playwright install --with-deps chromium + - name: Cache Assets (Vitest) id: asset-cache uses: actions/cache@v5 @@ -89,4 +111,12 @@ jobs: fail_ci_if_error: false slug: sleepypod/core files: test-results/junit.xml - flags: javascript \ No newline at end of file + flags: javascript + + - name: Upload browser evidence + if: matrix.task.name == 'Browser Tests' && !cancelled() + uses: actions/upload-artifact@v6 + with: + name: calibration-browser-results + path: test-results/browser + retention-days: 7 diff --git a/app/api/raw/[filename]/route.ts b/app/api/raw/[filename]/route.ts index 0eb1011ed..12b481cf5 100644 --- a/app/api/raw/[filename]/route.ts +++ b/app/api/raw/[filename]/route.ts @@ -20,26 +20,26 @@ export async function GET( } const resolved = path.resolve(RAW_DIR, filename) - if (!resolved.startsWith(path.resolve(RAW_DIR))) { + if (!resolved.startsWith(path.resolve(/* turbopackIgnore: true */ RAW_DIR))) { return NextResponse.json({ error: 'Path traversal detected' }, { status: 400 }) } try { // Reject symlinks to prevent escaping RAW_DIR - const lstats = await lstat(resolved) + const lstats = await lstat(/* turbopackIgnore: true */ resolved) if (lstats.isSymbolicLink()) { return NextResponse.json({ error: 'Path traversal detected' }, { status: 400 }) } // Verify canonical path is still inside RAW_DIR - const canonicalFile = await realpath(resolved) - const canonicalDir = await realpath(RAW_DIR) + const canonicalFile = await realpath(/* turbopackIgnore: true */ resolved) + const canonicalDir = await realpath(/* turbopackIgnore: true */ RAW_DIR) if (!canonicalFile.startsWith(canonicalDir)) { return NextResponse.json({ error: 'Path traversal detected' }, { status: 400 }) } - const s = await stat(resolved) - const stream = createReadStream(resolved) + const s = await stat(/* turbopackIgnore: true */ resolved) + const stream = createReadStream(/* turbopackIgnore: true */ resolved) const webStream = Readable.toWeb(stream) as ReadableStream return new Response(webStream, { diff --git a/app/api/raw/[filename]/tests/route.test.ts b/app/api/raw/[filename]/tests/route.test.ts new file mode 100644 index 000000000..2a57dd7ac --- /dev/null +++ b/app/api/raw/[filename]/tests/route.test.ts @@ -0,0 +1,43 @@ +// @vitest-environment node +import { Readable } from 'node:stream' +import { beforeEach, expect, it, vi } from 'vitest' +import { GET } from '../route' +const fs = vi.hoisted(() => ({ lstat: vi.fn(), realpath: vi.fn(), stat: vi.fn(), createReadStream: vi.fn() })) +vi.mock('node:fs/promises', () => ({ lstat: fs.lstat, realpath: fs.realpath, stat: fs.stat })) +vi.mock('node:fs', () => ({ createReadStream: fs.createReadStream })) +const request = (filename = 'capture.RAW') => GET(new Request('http://localhost/api/raw/' + filename), { params: Promise.resolve({ filename }) }) +beforeEach(() => { + vi.resetAllMocks() + fs.lstat.mockResolvedValue({ isSymbolicLink: () => false }) + fs.realpath.mockImplementation(async (path: string) => path) + fs.stat.mockResolvedValue({ size: 3 }) + fs.createReadStream.mockImplementation(() => Readable.from([Buffer.from([1, 2, 3])])) +}) +it('streams the original bytes with explicit download metadata', async () => { + const response = await request() + expect(response.status).toBe(200) + expect(response.headers.get('Content-Type')).toBe('application/octet-stream') + expect(response.headers.get('Content-Disposition')).toBe('attachment; filename="capture.RAW"') + expect(response.headers.get('Content-Length')).toBe('3') + expect(new Uint8Array(await response.arrayBuffer())).toEqual(new Uint8Array([1, 2, 3])) +}) +it.each(['../private.RAW', 'nested/capture.RAW', 'capture.txt', 'capture.RAW\n'])('rejects invalid filename %s before filesystem access', async (filename) => { + expect((await request(filename)).status).toBe(400) + expect(fs.lstat).not.toHaveBeenCalled() + expect(fs.createReadStream).not.toHaveBeenCalled() +}) +it('refuses symbolic links and canonical paths outside the configured directory', async () => { + fs.lstat.mockResolvedValueOnce({ isSymbolicLink: () => true }) + expect((await request()).status).toBe(400) + expect(fs.realpath).not.toHaveBeenCalled() + fs.realpath.mockResolvedValueOnce('/outside/capture.RAW').mockResolvedValueOnce('/owned') + expect((await request()).status).toBe(400) + expect(fs.createReadStream).not.toHaveBeenCalled() +}) +it.each([['ENOENT', 404, 'File not found'], ['EACCES', 500, 'Failed to read file']] as const)('sanitizes %s filesystem failures', async (code, status, error) => { + fs.lstat.mockRejectedValue(Object.assign(new Error('/private/source'), { code })) + const response = await request() + expect(response.status).toBe(status) + expect(await response.json()).toEqual({ error }) + expect(fs.createReadStream).not.toHaveBeenCalled() +}) diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index f369dec14..91c924c68 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -293,3 +293,19 @@ sudo bash /tmp/sleepypod-install --purge-free-sleep It also flushes the running iptables and ip6tables rules; the install then reconfigures iptables itself. Without the flag, run the cleanup block the installer prints, reboot, and re-run the installer. free-sleep's `unblock_internet_access.sh` is not a substitute: it flushes runtime rules only and leaves the persisted files in place. Run `sp-uninstall` if you want to revert (the script flushes iptables and removes sleepypod's units); the purge is not undone, so reinstall free-sleep from its own installer. + + +### Runtime filesystem reads and standalone tracing + +Pod data (RAW captures, saved alarm state, HomeKit identity seeds, archive-push +configuration, procfs and storage-accounting paths) is read on the destination +pod. These dynamic filesystem reads use `turbopackIgnore` call annotations so +Next does not infer that the build machine's whole checkout belongs in the +standalone output. The annotations affect packaging only; existing validation, +file access and error handling still run on the pod. + +The biometrics worker location probes are also excluded from automatic tracing: +`scripts/build/biometrics-worker.mjs` explicitly generates `worker.cjs`, +`coordinator.cjs` and `shadow-host.cjs` in the standalone bundle after Next builds. +Keep that packaging step and its dependency-graph checks when changing tracing. +Do not apply these annotations to shipped assets that need inclusion in a release. diff --git a/docs/biometrics-backtest-jobs.md b/docs/biometrics-backtest-jobs.md new file mode 100644 index 000000000..4bfd7c57d --- /dev/null +++ b/docs/biometrics-backtest-jobs.md @@ -0,0 +1,262 @@ +# Durable biometric backtest jobs + +`BacktestJobStore` persists experiments, immutable evaluation inputs, leases, +progress, cancellation, results, and holdout exposure in `biometrics.db`. +It is the job repository; execution, API/UI integration, and production worker +resource validation are separate parts of the calibration implementation. + +Enqueue pins the retained dataset hash, protocol hash, exact resolved +configurations, engine manifest hash, and selected split. Duplicate idempotency +keys return the original job, while changed requests fail. Hash or engine +mismatches are rejected. Configuration snapshots are limited to 64 KiB. + +Only one unexpired running or cancellation-pending job can hold the worker +slot. Claim increments both the attempt and fencing token. Every heartbeat, +completion, failure, and cancellation acknowledgment verifies owner, token, +state, and expiry under a short SQLite transaction. An expired worker cannot +publish after replacement. Recovery marks abandoned work interrupted and +allows at most three attempts; abandoned cancellations become cancelled. + +Cancellation of queued work is immediate. Running work exposes a cancellation +request through heartbeat, and cannot publish a successful result afterward. +The worker supervisor must terminate unresponsive execution after its grace +period. The repository itself does not execute code or control processes. + +A result and its terminal job state commit atomically. Summaries are bounded +to 16 MiB each and 64 MiB retained total; job and experiment histories each stop at 1000 +entries. These repository bounds do not replace worker CPU, memory, wall-time, +free-space, or live-ingest safeguards. They require target-pod measurement +before enabling production replay. + +Holdout exposure is recorded on enqueue, conservatively including jobs later +cancelled or failed. It follows physical side and overlapping recorded time +ranges across datasets and experiments. Creating another dataset or changing +its asserted subject name does not erase exposure. Repeated access is labeled +`reused_holdout_is_validation`. Idempotent retries of the same enqueue do not +count twice. Future promotion must recheck current exposure, not rely solely +on the count captured in an older report. + +The current evaluation protocol is explicitly descriptive. This repository +does not award validated improvement; that requires the complete evaluator, +reference support, coverage-aware policy, independent nights, and operational +validation. No backtest state activates a profile. + +## Shipped processed-data worker + +`evaluator.ts` executes comparison and stage replay over the frozen selected +split. Overlapping session rows are deduplicated by original ID; overlapping +windows merge and disconnected sessions are classified separately. It calls +the existing retrospective classifier with the exact resolved rules and the +protocol's explicitly assumed calibration quality. Default/current snapshots +with identical hashes execute once. Fixed clock offsets also apply to stage +scoring. Original unshifted stage traces remain available in the report. + +Numeric comparisons use original stored readings. They do not evaluate a +candidate's waveform-processing parameters. Stage comparisons report +interval-duration agreement, not the chart's older midpoint agreement. +Reports remain descriptive/insufficient-evidence until the promotion policy, +reference support, common-support checks and operational qualification are +complete. Merged replay windows are currently limited to one day, and complete +reports up to 2 MiB remain inline. Larger reports use checksummed files beside +the biometrics database, with a small descriptor in SQLite. The complete report +is available through the same result reader, with both byte and canonical-data +hash verification; it is never truncated to fit the inline limit. + +`pnpm build` now bundles the worker into +`.next/standalone/biometrics-worker/worker.cjs`; `pnpm build:biometrics-worker` +builds it independently. Packaging rejects database, hardware, automation, +scheduler, HomeKit and streaming dependencies. Shared schemas live in the pure +`contracts.ts` module so the worker does not import repositories. Enqueue pins +the SHA-256 of the actual worker bundle, and the runner checks it before launch. + +`executeNextBacktest` runs one leased attempt after an injected host admission +check. It does not start itself or enable production replay. The host must wire +measured resource/ingest admission and lifecycle handling. `runBacktestChild` +uses a separate low-priority Node process, a clean environment without pod +credentials, a 128 MiB JavaScript heap limit, bounded stdin/stdout/stderr, a +wall-time watchdog, and cancellation with SIGTERM followed by SIGKILL. This is +isolation for shipped trusted code; it is not the external sandbox required +for user-proposed algorithm packages. The heap limit is not a total RSS cap. +The child records evaluation runtime and peak RSS for operational inspection. + +The runner heartbeats independently of the child's progress. It rechecks resource +admission every five seconds and before publication; denied or unavailable health +evidence aborts the child and records a resource interruption under the same +bounded retry policy. Checks never overlap. Shutdown leaves +owned unfinished jobs interrupted for bounded retry. Cancellation is +acknowledged only after the child has stopped, and old attempts cannot publish. +Result reads verify their hash and winning fencing token and return the current +holdout exposure count, so later reuse remains visible. + +## Large report storage + +`ReportArtifactStore` stages a uniquely named file, fsyncs it, atomically renames +it and fsyncs the directory before a fenced database transaction publishes the +descriptor and successful job state. File paths derive from the server-owned +database directory, job UUID, attempt token and content hash; request filenames +never select a path. An in-memory database needs an explicitly injected artifact +store for large-report tests. Existing inline results remain readable. + +A failed database commit can leave an unpublished file, but cannot expose a +partial result. Staged files count toward storage checks. The collector removes +only owned files that are unreferenced and older than one hour; it preserves +committed descriptors and conservatively retains future-version named references +during downgrade. A separate SQLite publication lock serializes quota checks, +file publication, and collection across processes. File writes hold only this +sidecar lock; the live database transaction remains limited to fenced publication. +A terminated process releases the lock automatically. Identical staged retries +reuse the verified file without counting its bytes twice. + +`BacktestHost` provides a serialized, recoverable five-second polling loop and +orphan collection every thirty minutes. Shutdown aborts and awaits the current +attempt before database closure. Its global singleton requires an explicit +admission callback. Production startup and measured resource admission remain +pending; merely importing the host does not start replay. + +## Admission evidence + +`admission.ts` defines a deployment-policy contract with a qualification ID and +explicit limits for memory reserve, free disk, one-minute load per available CPU, +current event-loop delay, piezo age, temperature and sample age. It has no shipped +threshold defaults yet. Missing or malformed policy yields `unqualified`; missing +or malformed health evidence yields `health_unavailable`. Fresh evidence must +meet every limit. Future timestamps and stale samples also defer replay, and the +decision returns all applicable pressure reasons for later API/UI exposure. + +The Linux collector uses `MemAvailable`, filesystem `bavail`, normalized load, +thermal zone zero and a twenty-millisecond scheduling probe in the Node host. +The probe measures current scheduling delay rather than using the lifetime +performance histogram. Its supplied timestamp callback must read specifically +`getSensorFrameTimes()['piezo-dual']`, never the newest arbitrary sensor frame. +Unknown piezo freshness, missing thermal data, read failures or unsupported +platforms yield unavailable evidence. Collection makes no hardware commands or +database queries. These helpers are not yet connected to production startup; +measured thresholds and full publication-load qualification remain required. + +## Explicit report release + +Trusted deployment configuration may set `SLEEPYPOD_BIOMETRICS_REPLAY_BUDGET` +to JSON containing `jobWallMs`, `waveformWallMs`, and `waveformCpuSeconds`. +These are operational ceilings, not candidate parameters or RPC inputs. The +coordinator validates the complete budget before spawning: the job wall limit +is at most 48 hours, each waveform process at most one hour, and CPU time at +most 3,600 seconds. CPU time must fit within waveform wall time, which must fit +within job wall time. No production values are inferred from those ceilings. + +Only execution tasks receive the extended job deadline. Read, publication, +inspection, and cleanup deadlines remain unchanged, as does the Node scorer's +separate deadline. Python applies the selected CPU limit before importing the +numerical implementation. Without explicit configuration, existing short +deadlines and the 120-second CPU limit remain. Changing this environment setting +requires restarting the coordinator service; it does not start the worker host, +grant resource admission, or establish hardware qualification. The synthetic +waveform benchmark accepts the same setting for qualification runs. + +The coordinator reserves a separate process slot for read-only report, trace, +dataset-descriptor, and raw-catalog requests. At most one reader and one work +process run at once. Replay, scanning, publication, release, and cleanup share +the serialized work slot. Each lane accepts at most seven active-plus-pending +requests, with a combined limit of eight, reserving capacity for the other lane. +Pending requests expire after 30 seconds; shutdown cancels both active children +and rejects pending work. This prevents a long replay from consuming report-read +capacity. Resource qualification must include the additional reader's memory +and CPU, and execution time budgets remain independently enforced. + +Completed results support an exact-hash, idempotent release. The release +transaction replaces payload bytes with an empty stored object and records +`released_at_ms`, preserving the original content hash and completion/attempt +metadata. Readers return a null payload with explicit unavailability. Report +reads and releases share the publication lock, preventing file deletion between +descriptor lookup and byte verification. After commit, an unreferenced artifact +is unlinked; failed file deletion leaves an orphan counted against disk quota +until collection can remove it. A failed database transaction never deletes the +file. Referenced artifacts are retained conservatively, including future-format +named references. There is no automatic age-based expiry of committed reports; +release is an explicit user action through the shared API. + +## Paired stage support + +Each stage-replay night now includes comparisons of current versus default and, +when supplied, candidate versus default and current. Each window preserves the +configuration hashes and reports agreement, macro F1 and coverage on each +configuration's own support and on the exact intersection of their predicted +intervals. Conflicting or unknown reference intervals remain excluded by the +shared duration scorer. Common-support confusion matrices are retained; absent +paired support yields null metrics, never zero error. Fixed clock offsets apply +to both outputs for scoring without modifying the original displayed traces. + +A separate coverage-reduction flag and duration/coverage differences expose +candidates that omit difficult intervals. For example, omitting the incorrect +half of a prediction may raise its own-support agreement from 50% to 100%, but +cannot produce an agreement gain on the remaining common support when its +labels are unchanged. This is descriptive reporting only. It does not yet +implement the frozen promotion policy, per-night uncertainty aggregation, or +validated activation. Numeric waveform candidates still require raw replay. + +This comparison adds evaluator work and report bytes. Earlier target benchmark +numbers apply only to their recorded worker hash; repeat qualification with the +final worker before enabling production admission defaults. + +## Whole-night stage summaries + +`stageNightSummary` groups comparisons by baseline/candidate labels and exact +configuration hashes. It first sums confusion durations across each cohort's +nonoverlapping windows, then computes nightly agreement and per-stage +precision/recall/F1. It does not average window F1 values. Both full-window +output coverage and eligible-reference coverage remain available per night. +Duplicate cohorts/comparisons and overlapping windows are rejected. + +Each comparison reports scored and unscored night counts, nights with positive, +negative or zero agreement differences, and nights with any coverage reduction. +The equal-night mean agreement difference gives every scored whole-night cohort +one vote. Pooled duration confusion and agreement are reported separately and +can differ substantially when night lengths differ. Missing reference support +remains null and is counted as unscored, not converted to a zero difference. +The observed nightly minimum/maximum is explicitly a descriptive range, not a +confidence interval. The optional seeded interval is reported separately below. The frozen promotion +policy remains required before a report can establish validated improvement. + +## Reproducible exploratory uncertainty + +Experiment protocols may now include `nightUncertainty` with version +`paired-night-percentile-v1`, an explicit seed (1–64 characters), resample count +(200–5,000), and minimum scored nights (3–60). These fields participate in the +immutable protocol hash. Older protocols remain readable and report +`not_requested`; no parameters are silently added to their hashes. + +The statistic is the equal-night mean of paired common-support agreement +differences. Whole-night cohorts are sorted by ID, then resampled with +replacement, preserving the baseline/candidate pairing. A SHA-256 counter +stream over JSON `[version, seed, counter]` supplies big-endian 32-bit draws; +rejection sampling removes modulo bias. The two-sided 95% percentile interval +uses linear interpolation at ranks `(resamples - 1) * 0.025` and +`(resamples - 1) * 0.975`. This is a versioned local implementation, not a call +to SciPy. The general resampling procedure and pairing are described in +[the SciPy bootstrap reference](https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html). + +Missing-reference nights are counted and omitted from the statistic. Too few +scored nights return `insufficient_nights`, and identical observed differences +return `degenerate`, both with a null interval. The requested minimum is a +reporting threshold, not a claim of adequate statistical power. Intervals assume +exchangeable nights; serial dependence and unobserved nights can make the +estimate misleading. Reports retain these limitations and remain +`insufficient_evidence` for promotion. Objective/coverage/operational policy and +held-out qualification are still required for validated activation. + +## Shared support across all submitted configurations + +Saved stage replays compute one intersection across default, current and any +candidate for each evaluation window, after applying the fixed clock offset. +Every pairwise comparison uses that same mask and identifies its +`commonSupportScope` as `all_configurations`. If any configuration has no output, +the shared mask is empty and common-support agreement is null. The configuration +is never silently omitted from the intersection. Each output's own coverage and +agreement remain separate, preserving visibility into lost output and references. + +The interval sweep clips to the evaluation bounds, preserves gaps and labels, +and rejects overlapping or malformed input intervals. The standalone two-output +comparison helper still supports explicitly labelled `pair` scope, but saved +replay reports always supply the all-configuration mask. Nightly summaries and +uncertainty therefore use the same scored intervals for every comparison in a +run. The output-availability policy and validated promotion gate remain separate +requirements; common support alone does not establish improvement. diff --git a/docs/biometrics-calibration-api.md b/docs/biometrics-calibration-api.md new file mode 100644 index 000000000..c6909e213 --- /dev/null +++ b/docs/biometrics-calibration-api.md @@ -0,0 +1,530 @@ +# Biometric calibration API + +The `biometricTuning` tRPC router and `/api/biometric-tuning/*` OpenAPI routes +share the same services. They use the existing LAN trust model. Supplied +requester names are attribution, not authenticated identities. + +## Inspect and create profiles + +- `getManifest` returns the versioned default, parameter schema and current + manifest hash. +- `getProfileOverview` separates desired configuration, the existing stage + view, sensor-calibration freshness, and observed runtime receipts. No receipt + means unreported, not applied. A legacy stage customization awaiting migration + remains visible. +- `resolveProfile` validates explicit leaf/ordered-array overrides and returns + the resolved snapshot, hash and override provenance without saving anything. + Omit a path to restore inheritance; null does not mean remove. +- `createProfileRevision`, `getProfileRevision`, and `listProfileRevisions` + create and inspect immutable candidates. Creating one does not bind it or + write an activation outbox event. Public origins are manual or agent; + migration/system origins are reserved for server workflows. + +Manual activation is available. Desired profile state must not be presented as +actual runtime adoption; availability means requests can be queued, not that a +processor is currently healthy or has adopted a particular revision. + +## Retain data and submit work + +`listSessions` is cursor-paginated and side-specific. `getSessionCapabilities` +reports actual processed timestamp coverage, gaps, reference metric counts, +and unknown processing provenance. Raw waveforms are marked not inventoried; +the presence of processed readings does not imply signal replay capability. +Reference counts include revisions, so dataset creation requires explicit +import IDs and accepts only one revision per family. + +`createDataset` takes side, asserted sleeper ID, session IDs, reference import +IDs, and an idempotency key. The server pins the configured device timezone. +`getDataset` and `listDatasets` return metadata/manifests, not the retained raw +rows. Snapshot integrity is verified on a dataset read. Reference deletion and +observation retention do not change an existing snapshot. + +`createExperiment` pins a dataset and descriptive protocol: evaluation mode, +explicit assumed stage-calibration quality, and fixed alignment tolerance and +clock offset. `enqueueBacktest` takes the experiment ID, exact candidate +revision ID, expected binding generation, split, and idempotency key. Callers +cannot supply default/current snapshots, evaluator hashes, or executable paths. +The server resolves them. The current stage baseline comes from the actual +retrospective view, not a pending desired vitals profile. + +The current evaluation scopes are original-reading comparison and stage rules +on original readings. Waveform parameter changes are not evaluated by these +scopes. The job descriptor states this explicitly. + +`listBacktests`, `getBacktest`, `cancelBacktest`, and `getBacktestResult` expose +bounded job history, progress, cancellation and verified results. Enqueue +persists work; an admitted worker host must execute it. A queued response is +not a completed result or an activation. Result reads include the current +holdout exposure count. Existing successful retries return the original job +when the same public submission key and payload are supplied. + +`getBacktestSummary` (`GET /biometric-tuning/backtests/{id}/summary`) is the +interactive review path. It verifies the complete report in the coordinator, +then omits stage-trace rows before IPC and HTTP serialization. All scores, +per-night comparisons, support, provenance, and operational measurements remain. +Its `projection` is `scores_without_stage_trace_rows_v1`; `contentHashScope` is +`complete_retained_report`. The hash identifies the retained full artifact, +not the summary's serialized bytes. Summaries exceeding 2 MiB fail explicitly +rather than silently truncating nights or scores. `getBacktestResult` continues +to return the full verified report, and `getBacktestTrace` pages exact trace +rows using that same retained hash. The UI and MCP report reader use the summary +path; release and holdout exposure semantics are unchanged. + +The backtest host loop, measured resource admission, richer promotion reports, +and full client UI/MCP integration remain separate implementation steps. The API does not claim that these are enabled merely because job +submission exists. + +Profile diffs report both changed resolved values and changed override intent, +including explicit-default overrides whose numerical output is unchanged. +`validateProfileRevision` checks schema/engine compatibility only; it explicitly +leaves evaluation evidence unchecked. + +Experiment creation requires an idempotency key. `getExperiment` and +`listExperiments` preserve access to its frozen protocol and audit identity. + +`releaseDataset` is explicit and idempotent. It refuses release while queued, +running, cancellation-pending, or interrupted jobs still need the inputs. +After cancellation or completion, release removes the retained input snapshot, +keeps its manifest/hash and holdout audit metadata, and marks it unavailable. +Existing reports remain separate retained results. New runs cannot use a +released dataset; a fresh snapshot requires a fresh creation key. Storage +accounting continues to include the retained manifest. Active configuration +lives in the configuration database and is unaffected. + +Canonical dataset/report hashing uses the same finite-number encoding as +configuration hashing, with a separate bounded budget for realistic multi-night +rows. Configuration limits remain unchanged. Dataset hashes are checked when +reading retained input data, and metadata reads verify available snapshots; +released metadata remains readable without pretending inputs still exist. + +Activation history is available through `listActivationHistory` (side and +optional `beforeGeneration` cursor). Pages contain at most fifty events and +include fallback/default-update evidence. Profile overview separates its +historical receipts from `runtimeHealth`, which reports current numerical +heartbeat freshness and configuration consistency. General and legacy stage +settings use the same versioned intent and acknowledgment protocol. + +## Manual activation and observation + +- `activateProfile` / `POST /biometric-tuning/activate` selects an exact revision. +- `resetProfile` / `POST /biometric-tuning/reset` selects current shipped defaults + without deleting revisions or audit history. Malformed unmigratable legacy + stage settings are retained in the legacy row and reset audit evidence; valid + unmigrated customizations must first be preserved as a revision. +- `rollbackProfile` / `POST /biometric-tuning/rollback` selects the previous + binding, including its exact revision identity (null means the default binding). + +All three require side, expected binding generation, expected resolved hash, +immediate/next-session boundary, requester, idempotency key, and explicit +`allowUnvalidated: true`. These are deliberate manual changes, recorded as +`manual_unvalidated`. They do not assert that a backtest established improvement. +A stale binding, changed idempotent request or mismatched hash is a conflict. +Creating or evaluating a candidate never calls these mutations implicitly. + +`getActivationStatus` / `GET /biometric-tuning/activation-status` accepts a side +and optional event ID. It returns the persisted audit event, observed delivery +and matching-receipt state, receipt freshness and whether the event still +belongs to the desired generation. Polling is read-only and cannot acknowledge +the stage consumer. The persisted event can lag observed receipts until the +five-second reconciliation pass. An applied old event is history, not proof it +is still selected; check `isCurrent` and the overview's current runtime health. + +The cross-runtime integration test uses temporary fixture databases and the +locked piezo environment. Prepare it with +`cd modules/piezo-processor && uv sync --locked --no-install-project`, then run +`pnpm exec vitest run src/biometrics/profiles/tests/runtime-integration.test.ts` +from the repository root. The JS unit-test CI job installs that same environment. +The test runs the shipped Python adapter with a synthetic frame, verifies the +next-session boundary, TypeScript stage receipt, restart restoration and rollback; +it neither starts a sensor follower nor accesses hardware. + +## Releasing completed reports + +`releaseBacktestResult` (`POST /biometric-tuning/backtests/{id}/result/release`) +requires the job ID and `expectedContentHash`. It applies only to successful +jobs and rejects a mismatched hash. Repeating the same release is idempotent. +The result endpoint now includes `available` and nullable `releasedAtMs`. +After release it returns `available: false`, `report: null` and the original +content hash, completion time, winning fencing token and current holdout count. +A released report is distinct from a job that has not completed (which still +returns null as the whole result). + +Release removes inline payload bytes or detaches an artifact descriptor in a +transaction before deleting the file. It does not erase experiments, jobs, +holdout exposure, profile revisions or activation history. A retried enqueue +with the same idempotency key still resolves to the original successful job; +it cannot recreate released bytes or erase previous holdout use. A new replay +requires a new request key and available retained inputs. + +The additive biometrics migration `0027_release_backtest_reports` preserves +existing reports with a null release timestamp. Older readers cannot validate +a released payload against the original hash, so they report an unavailable or +corrupt result rather than silently displaying different data. UI/MCP clients +must use the explicit availability fields when presenting released evidence. + +## Paginated stage traces + +`getBacktestTrace` (`GET /biometric-tuning/backtests/{id}/trace`) requires +`expectedContentHash` and `configuration` (`default`, `current`, or `candidate`). +It accepts an optional `cohortId`, an `offset` (default zero), and `limit` +(default 200, maximum 500). The hash must match the verified retained result. +Unfinished jobs reject trace requests, changed hashes return a conflict, released +reports return not found, and comparison-only runs report that no stage replay +was produced. Missing cohorts/configurations and cursors beyond the selected +trace are errors rather than fabricated empty results. + +The response includes exact intervals, configuration hashes/revisions, cohort +and source-window identities, the total selected interval count, and +`nextCursor: { offset, contentHash }` or null. Reuse that offset and hash as +`offset` and `expectedContentHash`, keeping the selection unchanged. No +intervals are downsampled or interpolated. Trace timestamps precede the fixed +reference-alignment offset, which is returned separately as `clockOffsetMs`. +The provenance is explicitly `versioned_stage_replay`, not original observed +stages. Filtering/pagination never changes evaluation scores. + +This bounds response detail, not artifact-read cost: the service currently +verifies the entire retained report before selecting a page. Parent-process +artifact parsing and hashing remain part of the target performance qualification. + + +## Retained waveform discovery and publication + +`getReplayAdmission` (`GET /biometric-tuning/replay-admission`) reports whether +server policy is configured, its operator-supplied qualification ID, and current +admission reasons. `SLEEPYPOD_BIOMETRICS_ADMISSION_POLICY` is server-owned JSON +matching `admissionPolicySchema`; missing or malformed policy denies admission. +Supplying an ID does not itself qualify hardware. Each check uses fresh process, +storage, thermal, and live piezo receipt evidence; no caller can supply health +samples or override admission through an RPC parameter. + +`inspectRawSource` (`POST /biometric-tuning/raw-sources/inspect`) accepts only +`afterName`, initially null, and inspects at most one server-owned archive. +Persisted evidence is a mutation even though scanning does not modify the +archive. Reuse `nextCursor` until `complete`; `source_unavailable` advances past +an unreadable source, and `archive_unavailable` reports a missing archive root. +`deferred` preserves the cursor and means no new scan was admitted. This API +accepts neither filesystem paths nor executable code. + +`listRawSources` (`GET /biometric-tuning/raw-sources`) accepts `startMs`, `endMs`, +and optional `afterId`, returning at most fifty descriptors and a cursor. Query +all selected cohort windows including five minutes of pre-roll. Catalog event +bounds indicate possible overlap, not continuous support. Descriptors explicitly +say `requires_reverification`; inventory alone has not retained any waveforms. + +`createWaveformDataset` (`POST /biometric-tuning/waveform-datasets`) accepts a +parent dataset ID, its exact `expectedContentHash`, an idempotency key, and +`cohorts: [{ cohortId, sourceIds }]`. The server derives context from that frozen +parent, re-verifies selected archives, and atomically retains inputs and publishes +the immutable dataset. New publication requires ongoing admission; a `deferred` +response contains no dataset. A retry of already committed publication returns +the original descriptor even when current admission is unavailable or original +archives have disappeared. Changed selections under the same key conflict. +Released descriptors remain released; retries never recreate their bytes. + +Create a `signal_replay` experiment using the published dataset, then submit and +poll a durable backtest. Publication does not start a worker host or activate a +profile. Host qualification and scheduling remain separate operational gates. + + +The retained-dataset UI exposes **Select recorded waveforms** on available +processed datasets. It inspects one archive per explicit request, keeps scan +cursors on deferral, and preserves source selections across pages and nights. +Every parent cohort requires a selection, with at most 64 source IDs per cohort. +Publication pins the parent content hash and reuses its request key after a lost +reply or deferral. Released publication retries are shown as unavailable. + +Published waveform datasets offer **Replay retained waveforms** in the protocol +editor. Sleep-stage replay is optional and requires an explicit calibration +quality assumption; numerical-only replay does not. Protocol creation and queued +submission still depend on the separate worker host for execution. The browser +demo contains no RAW archive fixture or qualified pod telemetry, so its waveform +operations remain explicitly deferred. + +### Numeric objective policies + +Signal-replay protocols may pin an `objectivePolicy` with version +`numeric-objective-v1`. Its metric, minimum supported nights and paired references, +maximum lost-observation fraction, minimum MAE improvement, night and subgroup +regression tolerances, required subgroup names, and operational limits are included +in the protocol hash. Processed-reading protocols reject numeric objectives. +Reports retain the thresholds and per-baseline `objectiveAssessments`. + +Caller-supplied thresholds are not a designated promotion policy. The current +production evaluator has no designated policy and does not yet attach measured +operational or subgroup evidence, so its assessments remain insufficient evidence. +The pure assessor checks an independent first-use holdout, supported whole nights, +uncertainty, coverage, subgroup regressions, and measured operational limits before +it can award improvement under a designated policy. Test-only thresholds exercise +boundaries; they are not release defaults or evidence of clinical validation. + +### Replay host startup + +After migrations and hardware initialization, production startup reads +`SLEEPYPOD_BIOMETRICS_ADMISSION_POLICY`. A missing or invalid policy leaves the +background host stopped. A valid policy starts the existing singleton host; +startup does not assert that hardware has been qualified. Its execution and +cleanup operations still obtain fresh admission, including live piezo receipt +age. CI skips this startup with the other hardware-dependent services. Shutdown +aborts and awaits the host before closing the coordinator. + +Deploy only a policy backed by recorded pod measurements. Configure the matching +`SLEEPYPOD_BIOMETRICS_REPLAY_BUDGET` before process startup; the coordinator captures +that budget when its singleton is created. Changing process environment requires +a service restart to start a previously disabled host or replace that budget. + +New waveform dataset manifests advertise retained-input replay capability. +Execution availability remains a separate, current admission query. Older +immutable manifests and their hashes are not rewritten by this change. + +### Exact numeric trace pages + +`GET /api/biometric-tuning/backtests/{id}/numeric-trace` (tRPC +`getBacktestNumericTrace`, MCP `backtest_biometrics` action `numeric_trace`) +returns up to 500 exact recomputed readings per page. Supply the immutable report +`expectedContentHash`, configuration label, optional cohort, and offset/limit. +The next cursor carries the same hash. Released results and stale hashes fail; +paging does not rerun an experiment or change its scores. + +Pages contain only the report dataset's side, within its scoring windows. They +preserve null readings, quality, flags, routing provenance, and the original +replay timestamp; `clockOffsetMs` is metadata for reference alignment and is not +applied to the returned timestamp. Opposite-side readings and internal `hrRaw` +values are excluded. No downsampling is performed. Historical reports without +explicit side provenance cannot supply this projection. Numeric trace UI and +reference overlays remain separate consumers of this API. + +Waveform reports expose these pages through **Inspect numeric traces**. Select a +night, configuration, and metric; the chart shows only that page's individual +readings. The exact-value table includes missing values, quality, flags, source +channel, and limitations. Changing night/configuration resets the cursor. The +processed-only demo does not fabricate waveform readings for this view. + +### Frozen reference pages + +`getBacktestReferenceTrace` / `GET .../backtests/{id}/reference-trace` and MCP +`backtest_biometrics` action `reference_trace` require a report hash, cohort, +metric, and bounded offset/limit. They read the immutable dataset selected by +that report, verify its hash, and expose the frozen import revision, source, +normalized measurement, compatibility reason, and original reference times. +They never resolve the current revision of an import. Original submitted payloads, +annotations, and subject fields are excluded. + +These are display records, not newly matched pairs or independent evidence. +Point/interval and aggregation semantics remain explicit. HRV-family records +with an incompatible definition remain inspectable with their compatibility +reason. Scoring-window boundaries are preserved. A released report or dataset +makes this detail unavailable; it does not silently fall back to live data. + +In a waveform report, open **Inspect numeric traces**, then **Compare frozen +references**. The overlay pages references independently from recomputed readings +(200 records per page), preserves point and interval shapes, and applies the +experiment's fixed clock offset only to recomputed timestamps. The source table +keeps reference timestamps in UTC. Incompatible definitions remain visible in the +table but are not plotted. Display overlap does not establish a new scored pair; +use the report's frozen alignment and common-support scores for comparisons. + +### Numeric subgroup evidence + +Waveform experiments with a numeric objective additionally compute +`reference-defined-numeric-subgroups-v1` for that objective's metric. Fixed IDs +are `reference_points`, `reference_intervals`, `window_first_half`, and +`window_second_half`. Membership uses native reference timestamps and temporal +kind, never candidate values, errors, or missingness. Each disconnected scoring +window has its own midpoint. A point at the midpoint belongs to the second half; +an interval ending there belongs to the first. Intervals spanning the midpoint +are excluded from both half-window groups without splitting or changing their +aggregation; the report retains their excluded count. + +Each group uses the same all-configuration common timestamp support as the main +numeric comparison. References are weighted within a night, then nights receive +equal weight. Objective evidence requires the configured minimum paired records +in every included night; an unsupported night makes that group's mean unavailable +for promotion rather than silently dropping it. A required group with inadequate +night support blocks improvement. Per-window observation coverage counters still +refer to complete scoring windows, not inferred continuous coverage within a +reference subgroup. The report UI shows subgroup means, night counts and boundary +exclusions. Groups overlap and are not independent statistical experiments. +They do not establish motion, pump, empty-bed, or artifact-specific performance; +those require separately qualified context/fixture evidence. Unrecognized required +group names remain unsupported and cannot award improvement. + +### Stage objective evidence + +`stageObjectivePolicy` freezes a `stage-objective-v1` policy alongside the +experiment protocol. It requires `stage_replay`, or `signal_replay` with +`waveformStages: true`; plain comparison cannot evaluate a stage objective. +Numeric and stage policies are separate, and a joint replay may retain both. + +The stage policy sets minimum independent nights and paired reference duration, +minimum candidate output coverage, maximum loss of baseline output from shared +support, minimum equal-night agreement improvement, tolerated night regressions, +macro-F1 regression, and required-stage F1 regression/support. Common-support +confusion durations are combined within each whole night before scoring. The +uncertainty gate uses the lower bound of the frozen whole-night agreement +bootstrap; epochs are not treated as independent nights. Missing required stages +on any evaluated night remain insufficient evidence, never fabricated zero error. + +`stageObjectiveAssessments` preserve the exact policy and evidence for candidate +versus both default and current configurations. The coordinator replaces any +scorer-supplied operational evidence after the final admission check, using the +same explicit sampling-policy rules as numeric objectives. Only a separately +designated qualified policy can award improvement. The UI presents thresholds, +missing evidence and regression reasons. These are agreement results against the +named stage reference, not claims of clinical stage accuracy or live activation. + +### Evidence verification before activation + +The internal profile evidence verifier checks retained report bytes and exact +candidate revision, current configuration, shipped default, dataset/side, frozen +holdout cohorts, protocol, evaluator build, and waveform dependencies. Reused +holdout data or released artifacts cannot pass. These expectations must be +loaded by the service; they are not client assertions. + +The verifier recomputes objective assessments using supervised operational +measurements and separately designated policies. It requires both baseline +comparisons and every frozen holdout night. A changed stage configuration must +improve stage agreement; a numeric change must improve its metric and pass +downstream stage coverage/regression checks, which may report no material change. +An assessment of one numeric metric cannot validate changes to another metric. +A signal replay can freeze `objectivePolicies`, a list of one to three numeric +policies with unique metric identities. It is mutually exclusive with legacy +`objectivePolicy`; legacy snapshots and hashes remain unchanged. Evaluation, +supervisor finalization, and activation verification require candidate comparisons +against both default and current for every policy. Each changed numeric component +must have its own qualified improvement evidence; a passing heart-rate objective +cannot validate breathing or HRV changes. Stage objectives remain separately +required for downstream effects. Report displays group thresholds and reasons by +metric. These policy lists do not themselves designate a production policy. + +The internal validated activation service loads the retained job, experiment, +dataset, report, profile revision and current runtime baseline. It verifies +the report before acquiring write reservations, then rechecks report/dataset +availability, fencing identity, holdout exposure, runtime baseline and binding +generation before committing. A short biometrics write reservation prevents +concurrent exposure, release or runtime writes until the configuration commit +finishes. Full report parsing and hashing happen before this reservation; the +final artifact checks read only metadata. Only the configuration database is +written: binding, evidence audit and outbox intent commit together. + +The audit keeps the evaluation ID, report/configuration/build hashes, policies +and assessment summary even if biometric artifacts are later released. An exact +idempotent retry returns the original activation event without requiring those +artifacts again. Changing the request under the same key is rejected; manual +and validated requests cannot alias one another. A successful request still +returns `pending_delivery`, not proof of processor adoption. + +Stage replay now captures the entire acknowledged current configuration, including +numeric parameters, instead of substituting defaults for those parameters. Pending +or incompatible personalization blocks validated activation. Qualified production +policy designation remains unfinished. The shared +`activateValidatedProfile` procedure (`POST /biometric-tuning/activate-validated`) +accepts the exact revision, resolved hash, binding generation, evaluation ID, +report hash, processing boundary, requester, and idempotency key. Build hashes and +policy designations are resolved on the server. New requests currently fail closed +because no production policy is designated. Exact retries precede build probing +and artifact lookup. MCP `apply_biometric_profile` exposes the same procedure with +`action: "activate_validated"`; it does not accept `allowUnvalidated`. Explicit +manual activation retains its existing unvalidated semantics. + + +The holdout report UI offers application of its exact candidate and report hash. +`reviewValidatedActivation` (`GET /biometric-tuning/validated-activation-review`) +performs a read-only eligibility check using the same stored evidence and installed +build authority. A review creates no activation event or outbox item, and cannot +reserve evidence: application repeats verification and its transactional guards. +The apply control remains disabled when review fails or is refreshing. Once sent, +an uncertain request retains its exact identity and boundary for retry, including +when detailed report data is subsequently released. Processor adoption is polled +separately. The demo explicitly rejects validated activation of synthetic evidence. + + +Promotion designation also pins the complete experiment protocol hash. Matching +objective thresholds alone is insufficient: changes to clock offset, alignment +tolerance, assumed stage calibration quality, uncertainty settings, replay mode, +or objective lists require qualification of the resulting protocol. This server +value cannot be supplied through activation or review requests. The audit already +retains that exact protocol hash. This identity gate does not establish that missing +historical pump/occupancy context is acceptable; replay-context qualification is +still a separate release requirement. + + +Waveform promotion also requires a server-designated list of accepted replay +limitations. The verifier reads both per-run and per-reading diagnostics across +every held-out cohort and distinct configuration, verifies their retained payload +hash and build/configuration identities, and rejects missing or duplicate outputs. +Missing pump/occupancy/sleeper-mode context, short pre-roll, cold starts and event +clock substitution cannot pass merely because numeric scores improved. Unknown +limitation codes cannot be designated until the contract is reviewed. Activation +audit evidence records both the accepted list and the limitations actually observed. +No production replay-context allowance is installed yet; this gate does not by +itself qualify any approximation. + + +The publisher and activation service share a release-installed qualification +registry, `promotion-qualifications.json`, beside the packaged replay worker. +Its version is `biometric-promotion-qualifications-v1`; each entry pins the full +protocol, manifest hash, evaluator build hash, optional Python build hash, +qualification ID, fixture and pod evidence hashes, and replay-context allowance. +Every objective policy must use the entry's qualification ID. Exactly one matching +entry is required. Missing, malformed, oversized, duplicate or mismatched entries +grant no designation. Reads are capped at 256 KiB; no public endpoint writes this +registry. Evidence hashes identify release-reviewed artifacts; loading a registry +is not itself a qualification experiment. The publisher additionally rejects +context outside the registered allowance before awarding metric-level statuses. +No production registry has been generated or installed yet. + + +The supervisor now derives the overall retained report status with the same +assessment logic used for activation. `improved` requires complete qualified +independent evidence for every changed domain and acceptable downstream stages. +Supported failures produce `regressed`; insufficient support remains +`insufficient_evidence`, incompatible metric evidence is `incompatible`, and +supported outcomes below the improvement threshold are `no_material_change`. +An identical candidate/current configuration is also reported as no material +change. The supervisor replaces worker status claims. The UI displays these as +historical evaluation outcomes; an improved report never substitutes for current +activation review, holdout-exposure checks, or processor acknowledgement. + +## Original vitals provenance and CSV compatibility + +`biometrics.getVitals` retains its array and existing numerical fields. It now +adds nullable `processingRunId` and `provenance` metadata for each original +observation: `interpretation: original_observation`, a status, the recorded +configuration hash, and the recorded HRV metric identifier. The run must belong +to the same side and be a live run. Missing associations are `unavailable`; rows +without a run remain `legacy_unknown`. A missing, malformed, or unavailable HRV +definition stays null. The current shipped manifest is never used to infer an +old row's metric identity. This metadata describes the recorded association; +it is not validation evidence or a current processor-health receipt. + +Only bounded scalar metadata is projected from the processing bundle. The raw +bundle, input identity, and unrelated fields are not returned. A future recorded +HRV metric identifier is preserved rather than relabelled as the current window +IBI variability index. Consumers must keep different/unknown definitions +separate when computing comparable baselines. + +Summary and baseline responses now include exact SQL aggregate provenance counts +across the full selected range, independent of chart pagination: distinct recorded +configurations (manifest plus resolved configuration hash), unknown configuration +rows, HRV definitions and unknown HRV rows. Mixed known definitions, or known +and unknown HRV values together, produce null HRV aggregates. Entirely legacy +ranges retain their descriptive values with `legacy_unverified`; no definition +is inferred. All baseline means and deviations are null when multiple recorded +configurations or recorded/unknown configurations coexist. A summary may still +describe original HR/breathing values across configurations, with explicit counts. + +The chart separates recorded processing runs and HRV definitions before smoothing, +omits bands/statistics across those boundaries, and explains original/unknown +provenance. Its raw log exposes run IDs and HRV definitions. Unknown legacy rows +remain unverified: missing history cannot prove that their processing was uniform. +A run boundary is deliberately conservative even when two restarts used the same +configuration. This is distinct from the manifest/configuration counts in the +whole-range API summary. + +The vitals CSV keeps its original five columns first and appends interpretation, +processing-run ID, provenance status, configuration hash, and HRV metric ID. +Unavailable definitions are written as `unknown`, never inferred as clinical +RMSSD. Missing numerical values remain empty. Metadata cells are CSV-escaped +and spreadsheet-formula prefixes are neutralized. Older consumers can continue +using the original named columns; positional consumers must accept appended +columns. Demo/older responses without metadata export unknown provenance. diff --git a/docs/biometrics-calibration-readiness.md b/docs/biometrics-calibration-readiness.md new file mode 100644 index 000000000..60f309ec4 --- /dev/null +++ b/docs/biometrics-calibration-readiness.md @@ -0,0 +1,72 @@ +# Biometric calibration release evidence + +This is the acceptance evidence map for the calibration implementation, including +prospective monitoring and external algorithm experiments. A passing synthetic +benchmark is resource evidence for its exact build and input; it is not evidence +of improved accuracy or authorization to promote an algorithm. + +## Validation recorded on October 13, 2026 + +- Full CI Vitest run at `cb39444b`: 6,856 passed, 18 skipped; 480 files passed, + one skipped. Skips are not counted as verified behavior. +- Locked piezo environment: 203 piezo/shared numerical tests passed; the separate + common-module suite passed 415 tests with five skipped. +- TypeScript and the production build passed after the MCP workflow addition. +- Browser checks at `216dab46`: five passed, including the real database manual + calibration/receipt/rollback flow and English/Spanish provenance views. +- Deployment archive regression at `b1e05db3`: 26 tests passed. Local Python + environments/caches are excluded while source and locked dependencies remain. +- The shadow benchmark harness at `4ee74c25` passes TypeScript/lint and bundles; + those checks do not establish target-pod execution. + +## Acceptance map + +These links locate implementation contracts and behavioral evidence. Target-pod +acceptance is separate from fixture and browser results. + +| Requirement | Implementation and evidence | +| --- | --- | +| Defaults without user configuration | [Registry tests](../src/biometrics/algorithms/tests/registry.test.ts), [numerical golden fixtures](../modules/piezo-processor/test_default_golden.py) | +| Independent compatible side overrides | [Profile activation tests](../src/biometrics/profiles/tests/activation.test.ts), [runtime routing](../modules/common/test_biometric_routing.py) | +| Sensor calibration distinct from personalization | [Runtime ownership and lifecycle](biometrics-runtime-activation.md), [sensor calibration architecture](hardware/calibration-architecture.md) | +| Preserve existing stage overrides | [Runtime integration](../src/biometrics/profiles/tests/runtime-integration.test.ts), [migration/provenance tests](../src/biometrics/profiles/tests/provenance-migration.test.ts) | +| Reference source, metric, interval, timezone and import provenance | [Reference normalization](../src/biometrics/references/tests/normalize.test.ts), [reference storage](../src/biometrics/references/tests/store.test.ts), [import contracts](biometrics-reference-imports.md) | +| HRV definition across storage, reports, UI, exports and prompts | [API/provenance contracts](biometrics-calibration-api.md), [router tests](../src/server/routers/tests/biometrics.provenance.test.ts), [CSV tests](../src/components/biometrics/tests/vitals-csv.test.ts), [MCP contracts](../src/mcp/tests/server.test.ts) | +| Identical retained inputs, whole-night separation, coverage-aware evaluation | [Frozen datasets](biometrics-datasets.md), [dataset tests](../src/biometrics/backtest/tests/datasets.test.ts), [evaluation assessment](../src/biometrics/profiles/tests/evaluation-assessment.test.ts) | +| Missing raw/context limits claims | [Waveform dataset tests](../src/biometrics/backtest/tests/waveform-dataset.test.ts), [retained replay tests](../modules/piezo-processor/test_retained_replay.py) | +| Shared processing, durable jobs, bounded resources, no canonical/hardware effects | [Backtest lifecycle](biometrics-backtest-jobs.md), [waveform publication integration](../src/biometrics/backtest/tests/waveform-publication.integration.test.ts), [shared live/replay tests](../modules/piezo-processor/test_waveform_replay.py) | +| Shared UI/MCP workflow | [Browser workflow](../tests/browser/workflow.e2e.ts), [failed validation](../tests/browser/calibration.e2e.ts), [MCP session tests](../src/mcp/tests/server.test.ts) | +| Exact revision, concurrency and actual acknowledgment | [Runtime integration](../src/biometrics/profiles/tests/runtime-integration.test.ts), [daemon adapter tests](../modules/piezo-processor/test_runtime_adapter.py) | +| Distinct audited reset and rollback | [Activation tests](../src/biometrics/profiles/tests/activation.test.ts), [runtime protocol](biometrics-runtime-activation.md) | +| Original observations distinct from recomputation | [Historical API semantics](biometrics-calibration-api.md), [provenance browser checks](../tests/browser/provenance.e2e.ts) | +| Coherent updates, resets, cleanup, failures and restarts | [Runtime host tests](../src/biometrics/profiles/tests/runtime-host.test.ts), [backtest lifecycle](biometrics-backtest-jobs.md), [deployment tests](../tests/updateDeploy.test.ts) | +| Isolated generated code and reviewed promotion | [External experiment contracts and real-Docker checks](../scripts/algorithm-experiments/README.md), [shadow admission tests](../src/biometrics/shadow/tests/admission.test.ts), [promotion registry tests](../src/biometrics/profiles/tests/qualification-registry.test.ts) | +| Target-pod operational behavior | [Measurements and limitations](biometrics-replay-benchmark.md); deployment and live measurement status below | + +## Delivery and qualification status + +- The current eight-hour replay and 30-minute synthetic shadow supervisor runs + completed successfully; their receipts are linked in the measurement document. + The separate 180-second live source/isolated sink assessment also completed; + its exit-race regression, build identities and limitations are recorded there. + Synthetic output cannot supply held-out accuracy. +- The measured replay policy is installed and the pod reports admitted. + Reviewed promotion/shadow registries still need their own matching fixture and + held-out evidence; absence must remain visible and deny qualified promotion. +- Build `4c3a6a6e` is deployed, including the live-shadow exit-race fix. Migrations succeeded; the calibration API, live + data path and core/numerical services are healthy, and new vitals carry processing + provenance. [Deployment receipt](performance/biometrics-replay-2026-10-13/deployment-verification.json). + Unpersonalized defaults have no requested activation generation, so this check + does not claim a new profile acknowledgment or change the user's profile. +- [PR #824](https://github.com/sleepypod/core/pull/824) is pushed against `dev`. + CodeRabbit full review was requested, but the provider skipped it because + the PR exceeds its 100-file review limit. No completed CodeRabbit review is claimed. +- Build, lint, typecheck, unit, browser and all Python jobs passed at `8909f305`. + The patch-coverage gate passed at 86.47%, within the repository's existing + eight-point margin below its 88.93% target. No coverage threshold was changed. + Direct coordinator and host tests cover admission, cancellation, retention, + error sanitization and resource ownership. + +Legacy reference availability alone does not establish independent nights, metric +compatibility, reference reliability or sufficient held-out support. No accuracy +qualification is inferred from an inventory of existing references. diff --git a/docs/biometrics-datasets.md b/docs/biometrics-datasets.md new file mode 100644 index 000000000..b52e093ae --- /dev/null +++ b/docs/biometrics-datasets.md @@ -0,0 +1,251 @@ +# Frozen biometric datasets + +`DatasetStore` retains bounded processed-data snapshots in `biometrics.db`. +It is the persistence layer for the calibration backtest workflow; it is not +yet a waveform replay service. Creating a dataset does not modify live vitals, +active settings, or hardware. + +A snapshot contains the selected full recorded sessions, original vitals and +movement rows, exact reference import revisions and normalized measurements, +and available processing-run bundles. Unknown legacy processing provenance +stays null. Measured calibration bytes are not inferred from today's profile. +The manifest explicitly records that limitation and disables signal replay. +Stage replay must state its calibration-quality assumption when the original +quality context is unavailable. + +Each snapshot owns its input bytes. Deleting or correcting a reference import, +or pruning original readings, leaves a frozen dataset unchanged. Dataset reads +verify the hash of both manifest and snapshot. Reusing an idempotency key +returns the original dataset; a different selection under that key fails. +Only one revision from a reference family may appear in a dataset. Both side +and asserted sleeper mapping must match its references. + +The initial processed-data limits are 60 selected sessions, 20 reference +imports, 20,000 retained rows, 8 MiB per snapshot including its manifest, and +64 MiB across these snapshots. These are conservative service bounds, not a +claim about target-pod replay performance. Raw artifacts need separate retention +and free-space safeguards before signal replay can be enabled. + +Whole-night grouping uses the selected timezone's local noon as the night +boundary. Overnight sessions and naps touching the same night stay together; +a session spanning noon connects adjacent night cohorts. Cohorts are sorted +chronologically before assigning tuning, validation, and holdout. Fewer than +three cohorts yields descriptive tuning only. More cohorts establish a split, +not validated improvement or adequate statistical power. ICU/timezone runtime +versions and the actual assignment are retained; later timezone updates cannot +silently change an existing dataset. + +Coverage includes actual timestamp counts, duplicates, and leading/internal/ +trailing gaps under an explicit nominal cadence. This estimates recording +availability, not measured signal support duration. Overlapping selected +sessions retain original row IDs; evaluators must deduplicate shared rows and +score one independent night cohort rather than treating sessions as independent +nights. Retrospective classification remains retrospective. + +## Shared waveform replay driver + +`modules/common/biometrics/replay.py` provides the internal streaming driver for +retained waveform experiments. Callers supply immutable per-logical-side +configurations, integer Unix-millisecond record times, recorded sleeper mode +and occupancy snapshots, pump-context availability, and an experiment output +sink. The driver does not read current bindings, occupancy files, or databases. +It processes both physical channels in left-before-right order using the same +sample repair, pump/presence gates, stateful estimator, and single-sleeper +arbitration as the live adapter. + +Records must be ordered within the retained half-open time window. Each channel +is capped at 5,000 int32 samples per record; processor buffers retain only their +fixed windows. Pre-roll is processed, but outputs are scored only inside the +requested half-open scoring window. The output records preserve computed +logical side, physical source side, configuration hash, event timestamp, +quality, and flags, including when a held candidate is flushed after a mode +change. Context and outputs are immutable; outputs stream to the supplied sink. + +Cold-start state and replacing live arrival time with recorded event time are +explicit limitations. Missing mode, occupancy, or pump context adds separate +limitations; unknown mode uses a disclosed two-side fallback. Less than five +minutes of pre-roll is also flagged. These labels do not claim exact historical +parity. The shared driver does not itself discover or retain RAW files, capture +historical context, or authorize a public waveform job. Those integration and +resource checks are required before dataset capabilities may advertise raw +signal replay. + +`modules/common/biometrics/raw_artifacts.py` supplies retained-byte storage for +that integration. It accepts a selected `.RAW.gz` filename under a server-owned +archive directory, rejects path traversal, symlinks and non-regular files, and +copies into a private temporary file. It detects source changes, verifies gzip +expansion within a byte limit, hashes the exact compressed bytes, fsyncs the +copy, and atomically publishes a content-addressed artifact. Repeated retention +of identical bytes reuses the verified artifact. Deleting the rolling archive +thereafter does not remove the retained input. + +A process lock serializes publication/quota checks and verified reads. Crash +staging files are reclaimed only while holding that lock. Published artifacts +are immutable and private (0600 files, 0700 directory); readers check their +size and SHA-256 before exposing a bounded decompression stream. A changed or +missing artifact fails rather than substituting an archive file with the same +original name. Gzip validation is not CBOR/frame or timestamp validation: the +record inventory must still establish those properties before advertising +waveform coverage. + +The injectable retention limits default to 256 MiB compressed per artifact, +512 MiB expanded, 1 GiB and 1,024 published artifacts total, and a 512 MiB free-space +reserve. One additional compressed staging copy can exist while publication +is serialized, including during an idempotent retry at the published quota. +These are provisional storage limits, not a measured production replay policy. +Dataset references, release/garbage collection, record inventory, and job +admission must be integrated before this storage helper is exposed publicly. + +`raw_records.py` adds a bounded firmware-CBOR decoder and retained-file inventory. +It supports the definite and indefinite maps/arrays/strings used by captured +firmware, with explicit record-byte, string-byte, node-count and nesting limits. +Container lengths are checked before allocation; duplicate/non-text map keys, +semantic tags, malformed UTF-8 and trailing values are rejected. The retained +iterator distinguishes clean end-of-file from a truncated outer RAW record and +counts empty sequence placeholders toward its record limit. + +The inventory reports exact retained artifact identity, event-time bounds, +relevant record counts, usable dual-channel frames, lost-marker counts, +truncated sample tails, timestamp reversals and oversized channels. Its +`ordered_waveform_available` flag means the retained records meet this driver's +ordering and frame bounds; it does not establish continuous recording coverage, +complete pump context, or availability of occupancy/calibration snapshots. +Equal event timestamps are reported and preserved. Other sensor records are +counted without being misrepresented as waveform support. Numerical sensor +values retain CBOR float semantics; nonfinite or invalid event timestamps are +reported as unavailable for ordered replay. + +`raw_sources.py` inspects server-owned cold archives without retaining them. +Discovery is bounded and excludes symlinks; inspection streams checksum, +expanded-size validation and record inventory. Its result explicitly states +`retained: false`. Selection passes the inspected descriptor as `expected` to +`RawArtifactStore.retain`; changed compressed bytes or sizes fail before +publication. A source name or successful inspection alone is never evidence +that a dataset has frozen input bytes. Persisted inventory caching and dataset +references remain responsible for tracking inspected versus retained state. + +Migration `0028_retained_raw_artifacts` adds separate tables for inspected +sources, content-addressed retained artifacts, and ordered dataset references. +`RawSourceCatalog` validates the Python inspection contract, including matching +hashes, event bounds, record counts and ordered-waveform eligibility. Repeating +an inspection refreshes its timestamp; different evidence for identical bytes +and inventory version fails. Replacing a filename preserves the old identity +and marks it unavailable. A late failure for an old identity cannot mark newer +bytes unavailable. + +Catalog queries return at most 50 entries per page for potential event-time +overlap. Unexpired inspection evidence is labelled `requires_reverification`, +not available or retained: selected copies still require the expected-descriptor +check. Source scans do not create datasets or retained-artifact rows. Shared +artifact references have unique dataset positions and cannot dangle; release +and file garbage collection must coordinate those references before deletion. +The catalog and schema do not yet enable public waveform jobs. + +Dataset release now atomically tombstones its RAW references with the dataset +snapshot. Queued, running, cancellation-pending and interrupted jobs prevent +release. An artifact becomes eligible for collection only when its last live +reference is released; releasing another dataset never changes that timestamp. +This eligibility timestamp does not itself mean the file has been deleted. + +`RawArtifactStore.collect_released` holds the same process lock as verified +readers and publication, then takes a SQLite immediate transaction while it +checks eligibility and unlinks files. The default one-hour grace period is +injectable. Directory traversal is bounded before any deletion; the collector +preserves unknown files/metadata, rejects symlinks and mismatched identities, +and keeps database tombstones. It scans present files, so previously collected +rows cannot starve later eligible files. Unregistered crash orphans are not yet +collected automatically. + +The internal `retain(..., expected=descriptor, publish=callback)` hook allows a +trusted publisher to commit references before releasing the file lock, including +when reusing existing bytes. Publishers must lock files before beginning their +SQLite transaction, commit before returning, and never reacquire the file lock +inside the callback. Publication failure leaves complete bytes for recovery. +The public waveform dataset creation path and scheduled collector still need +to be wired through this protocol before waveform replay is enabled. + +`retained_replay.py` connects immutable artifact descriptors to the shared +waveform engine. Its versioned `retained-waveform-input-v1` contract contains a +retained/pre-roll/scoring window, ordered artifact descriptors with inspected +event bounds, and frozen context intervals. It rejects duplicate or overlapping +artifact windows instead of guessing which boundary frames to discard. Missing +context intervals resolve to unknown; it never reads the current sleeper mode, +occupancy or calibration to fill historical gaps. + +The adapter verifies retained bytes and consumes every selected file to EOF, +including events outside the scoring window. It checks actual event bounds and +ordering against the frozen input, then passes selected frames to the same +numerical driver as direct replay. Initial bounds are 64 artifacts, 1 GiB +compressed input, one million RAW records across all artifacts (including +empty placeholders), 20,000 context intervals and 20,000 output readings. +These are explicit implementation bounds, not a qualified pod resource policy. +The caller must also enforce a process deadline; cooperative cancellation does +not interrupt a file checksum operation already in progress. + +Output sink rows are provisional until the adapter returns successfully. A +late archive integrity failure must fail the job and discard its staged result. +The returned run metadata includes consumed/decoded counts, exact artifact and +configuration hashes, processing counters and limitations. The public durable +worker still needs to capture this input contract, retain its references and +publish the completed result through its fenced job lifecycle. + +The fixed Python entry point is `python -B -m common.biometrics.replay_worker +--retained-directory `. Its input is one bounded JSON +`retained-waveform-work-v1` envelope containing `input` and both logical-side +`configs`. It opens existing retained storage in read-only mode: missing storage +or a missing lock fails, and it neither creates directories nor changes their +permissions. RAW retention/collection methods reject calls on that reader. + +The worker emits bounded JSON-lines `reading` events and exactly one final +`result` on success. Readings explicitly identify HRV as +`window_ibi_variability_v1`; nonfinite output fails serialization. Input rejects +duplicate JSON fields and nonfinite constants. Errors emit one fixed diagnostic +without exporting inputs, paths or stack traces. CPU limits are 120 seconds +(121 hard), file descriptors are capped at 64, core dumps are disabled, numerical +thread counts are one, and process priority is reduced. These provisional caps +are not a qualified memory or pod admission policy and do not sandbox uploaded +code. + +`runWaveformChild` provides the coordinator-side transport with a clean child +environment, 8 MiB input, 16 MiB output, 64 KiB event and 16 KiB diagnostic +limits, a 20,000-reading bound, and timeout/cancellation with TERM/KILL escalation. +It verifies event order, schema, final row counts and per-side configuration +hashes, returning readings only after a final result and zero exit. A missing or +failed worker never returns its provisional rows. Executable, module and retained +storage locations are internal server configuration and must not be accepted +from public requests. The transport is tested against the actual Python worker; +the coordinator still needs to invoke the frozen dataset publisher and durable +waveform jobs before public waveform capability can be enabled. + +Waveform dataset publication now has an internal complete path. The TypeScript +preparer verifies a processed parent dataset, detaches its historical context, +and creates a new `waveform-dataset-v1` snapshot with one frozen input per +whole-night cohort. It preserves original observations and references, requires +matching scoring windows, normalizes cohort order, and hashes the complete new +snapshot. Shared artifacts must have identical descriptors and inspected event +bounds across cohorts. The parent dataset remains independently readable. + +The fixed `common.biometrics.publication_worker` accepts a server-prepared row +and inspected source selections, with database/archive/retained paths supplied +only by server configuration. It opens an existing database, enables foreign +keys, and uses `retain_batch` to hold the file lifecycle lock through all copies +and reference publication. Copies and checksum/decompression verification finish +before the SQLite immediate transaction. That transaction inserts the dataset, +artifact metadata, and all ordered references together, enforcing the aggregate +snapshot quota. A failed transaction rolls back every database write. Complete +unreferenced files can remain after failure and still require orphan recovery. + +Identical retries return the original dataset, including a released tombstone; +they do not resurrect it or require the rolling archives. A new dataset may +reuse exact retained bytes after rechecking compressed identity and expanded +size, even when the original archive directory has expired. Reusing an artifact +clears its collection eligibility only in the transaction adding its new live +references. Existing reference history and artifact creation time remain intact. + +The shared reader understands both processed and waveform snapshot versions and +verifies their full content hashes. A committed integration test covers actual +archive inspection → TypeScript preparation → Python publication → shared +reader verification → Python numerical replay → release → collection, while +checking that canonical observations and the parent dataset remain unchanged. +Public `signalReplay` remains false until coordinator/API source selection and +fenced waveform-job/report integration are wired to this publisher. diff --git a/docs/biometrics-reference-imports.md b/docs/biometrics-reference-imports.md new file mode 100644 index 000000000..5413a8c94 --- /dev/null +++ b/docs/biometrics-reference-imports.md @@ -0,0 +1,175 @@ +# Biometric reference import contract + +Reference imports are comparison data. They never write the pod's `vitals`, +change a profile, or control hardware. Agreement with a named reference source +is not clinical accuracy. APIs use the existing LAN trust model. + +## Preview and commit + +Call `biometricTuning.previewImport` with an explicit physical `side`, a +`subjectId` identifying the sleeper, a named `source`, and bounded `rows`. +Neither sleeper nor side is inferred from a filename. The preview returns +accepted/rejected/duplicate/conflict counts, bounded rejection examples, +metric identities, time interpretation, coverage endpoints and a content hash. +It writes nothing. + +```json +{ + "side": "left", + "subjectId": "sleeper-A", + "source": "wearable-export", + "timePolicy": { "timezone": "America/Los_Angeles", "disambiguation": "reject" }, + "rows": [ + { + "recordKey": "sample-123", + "metric": "heart_rate", + "value": 62, + "unit": "bpm", + "timestamp": "2026-10-10T23:00:00-07:00" + } + ] +} +``` + +Commit with `biometricTuning.commitImport`, passing the same payload as `import` +and the returned hash as `previewHash`. Changes to data, mapping, units, source +or time interpretation require another preview. When some rows were rejected, +commit additionally requires `acceptRejectedRows: true`. An import with no +accepted rows cannot be committed. Repeating the same content is idempotent. + +For a corrected full export, include `supersedesImportId` naming the latest +revision in the family. Corrections preserve source, sleeper and side; they +create a new immutable revision rather than editing old measurements. A stale +parent is rejected. Selecting an import ID always selects that exact revision. + +`listImports` pages at 50 imports using both `beforeCreatedAtMs` and `beforeId`. +`getImportMeasurements` pages at up to 500 rows using `afterId`. Follow +`nextCursor` to read the complete import. Preview summaries never contain the +full export. `deleteImport` explicitly removes its measurement rows and any +linked legacy chart rows, leaving a compact unavailable-import tombstone; +retrying the deletion is a no-op. A deleted import ID cannot supply readings. + +REST equivalents are under `/api/biometric-tuning/references`, with POST +`/preview`, POST `/commit`, GET `/`, GET `/{id}`, GET `/{id}/measurements` and +DELETE `/{id}`. All use the same service as tRPC. + +## Records + +Required fields: `recordKey`, `metric`, `value`, `unit`, and a time representation. +`recordKey` is the source's stable record identity. Repeated identical keys in +one batch are deduplicated; different measurements with the same key are all +rejected. Optional row `subjectId` must agree with the batch mapping. + +Use exactly one time representation: + +- Point: `timestampMs` (integer Unix milliseconds) or `timestamp` (ISO text). +- Interval: `startMs` plus `endMs`, or ISO `start` plus `end`. + +Intervals are half-open `[start, end)` and may span at most 24 hours. Unix +milliseconds must fall between 2001 and 2100; Unix seconds in millisecond fields +are rejected. ISO text includes seconds, optional milliseconds, and preferably +an explicit `Z` or `±HH:MM` offset. Invalid calendar dates are rejected. + +Naive local ISO times require a `timePolicy` with an IANA `timezone` or explicit +`utcOffsetMinutes`. Repeated DST times are rejected unless `disambiguation` is +explicitly `earlier` or `later`, or a valid offset is supplied. Nonexistent local +times are rejected rather than shifted. An explicit offset must agree with a +supplied timezone. An absent source timezone stays absent in provenance; it is +never inferred from the pod's current timezone. + +| Metric identifier | Units accepted | Meaning | +| --- | --- | --- | +| `heart_rate` | `bpm`, `beats/min`, `Hz` | Reference heart rate; Hz converts to bpm | +| `breathing_rate` | `breaths/min`, `br/min`, `Hz` | Reference breathing rate | +| `window_ibi_variability_v1` | `ms`, `s` | Pod window-level IBI variability index, not clinical RMSSD | +| `hrv_rmssd` | `ms`, `s` | Reference RMSSD, kept distinct from the pod index | +| `hrv_sdnn` | `ms`, `s` | Reference SDNN, kept distinct from RMSSD and the pod index | +| `hrv_unknown` | `ms`, `s` | Unspecified HRV definition; never directly scored | +| `sleep_stages` | `stage` | Interval label: `wake`, `light`, `deep`, `rem`, `asleep`, `in_bed`, `unknown` | + +Numeric values must be finite, nonnegative and within JavaScript's safe numeric +range after conversion. These are data representation constraints, not medical +normal ranges. Original submitted values and units are retained. + +Optional fields are `aggregation` (`instant`, `mean`, `median`, `window_index`, +`stage`), `windowMs`, `quality` (0–1), and `annotation` (up to 1000 characters). +The pod index requires `aggregation: "window_index"` and `windowMs: 300000` for +direct comparison. Matching HRV definitions with different or unknown windows +are not directly comparable. RMSSD/SDNN must never be scored as the pod index. + +Unknown, generic asleep and in-bed stages remain separate labels and are +unscored. Conflicting overlapping stage labels are preserved, but both entire +intervals are marked unscorable under the persisted `exclude_conflicts` policy. +Back-to-back intervals do not overlap. An annotation is untrusted data, never +an instruction, executable code or a command. + +## Text adapters and limits + +The shared `parseReferenceText` adapter accepts a JSON array of records or CSV +with the field names above as the header. CSV supports quoted commas, escaped +quotes and multiline annotations. Duplicate/unknown headers, malformed quotes +and mismatched column counts are rejected. CSV timestamps must retain their +explicit field units; there is no seconds/milliseconds guessing. + +Each upload is limited to 2 MiB, 2000 records, 12 JSON nesting levels and a +bounded node count. Rejection examples stop at 25, while aggregate counts are +complete. No server path, URL, shell command, archive or automatic cloud upload +is accepted. Hosts may parse a local export and send bounded normalized batches. +Do not log complete health exports. + +## Alignment semantics + +`reference-alignment-v1` uses chronological nearest-unused point matching with +a fixed tolerance (default 30 seconds). Each pod observation and reference is +used in at most one pair. Interval comparisons aggregate actual observations +inside the interval using the declared mean or median. Sparse references are +never forward-filled. Equivalent duplicate references do not count twice. + +Clock offset is explicit and fixed for the comparison; a training-derived +offset must remain frozen for validation/holdout. Reports include MAE, median +and 95th-percentile absolute error, signed bias, matched/unmatched counts, +reference and output coverage, exclusions and limitations. Point samples +contribute counts, not inferred duration. Interval duration counts only known +observation support and does not fill gaps. Incompatible HRV has no error score. +This alignment primitive does not by itself establish validated improvement. + +## Legacy Apple Watch clients + +The existing `biometrics.importReferenceStages` and `importReferenceVitals` +contracts still accept Unix seconds and return `{ written }`. They retain +first-write-wins heart-rate behavior and whole-request stage-overlap rejection. +Newly accepted rows and normalized provenance commit atomically; if provenance +cannot be saved, the legacy insert rolls back too. + +Old clients provide no subject identity or source timezone. Such imports are +visibly mapped to `legacy_unknown:left` or `legacy_unknown:right`, with that +limitation in the original record annotation. Older unlinked rows remain +legacy/unknown instead of receiving invented provenance. New clients should +use explicit sleeper mapping and the preview/commit workflow. + +### Stored-reading comparison + +`biometricTuning.compareReadings` (`POST /api/biometric-tuning/compare`) accepts an +exact `importId`, `side`, `subjectId`, metric, and a half-open `startMs`/`endMs` +interval of at most 24 hours. Optional `pointToleranceMs` (default 30 seconds) +and `clockOffsetMs` (default zero) are fixed, reported settings; this endpoint +does not fit an offset. It reads all stored observations in that interval, +without chart downsampling, and returns bounded summary scores, coverage, +source revision, and processing-run counts. A null processing-run ID means +`legacy_unknown`. `requestedMetric` identifies the export's metric; `metric` +identifies the pod metric actually available. Clinical HRV cannot be scored +against the pod's window IBI variability index. + +This is a read-only comparison of original readings. Its input hash identifies +the read snapshot but does not retain bytes or constitute a saved evaluation. +Only reference intervals fully inside the requested range are used, preserving +the original aggregation window. Stored readings do not record their support +duration, so comparisons report point counts and unknown duration (zero), +without filling gaps. A changed reference revision must be selected explicitly. + +The shared stage scorer uses intersecting interval duration, reports a confusion +matrix (rows reference, columns predicted), per-stage precision/recall/F1, +macro F1, coverage, and stage-duration/transition differences on common support. +Absent classes have null scores. Transitions across unobserved gaps are not +inferred. Conflicting intervals from separate sources remain unscored. This +scorer is a primitive for replay reports; it is not a new live stage classifier. diff --git a/docs/biometrics-replay-benchmark.md b/docs/biometrics-replay-benchmark.md new file mode 100644 index 000000000..d6684c9c8 --- /dev/null +++ b/docs/biometrics-replay-benchmark.md @@ -0,0 +1,666 @@ +# Initial synthetic replay measurement + +Measured on the requested pod (`192.168.1.88`, SSH port 8822) on October 12, +2026. This is an initial worker-cost measurement, not the final admission-policy +or end-to-end live-system qualification. + +| Property | Observation | +| --- | --- | +| Runtime | ARM64, four CPUs, Node 24.16.0 | +| Physical memory | 2,014,796 KiB | +| Available memory before / after | 1,440,260 / 1,425,340 KiB | +| `/persistent` available before | 3,741,108 KiB | +| One-minute load before / after | 0.54 / 0.36 | +| Input | 27 separate 12-hour nights, 720 minute readings each; 19,440 readings total | +| Input bytes | 2,374,215 | +| Configurations | Identical default/current; one unique numerical evaluation after deduplication | +| Child settings | nice 19, 128 MiB Node heap limit, 30-second external timeout | +| Evaluator elapsed time | 2,519.355 ms | +| Child peak RSS | 132,436 KiB (about 129 MiB) | +| Complete report size | 2,450,542 bytes | +| Core / piezo service status | Both active before and after | + +The shipped worker SHA-256 was +`16bab5135fcf0ba4d4a1624660f8565ba0b073cf39e7d13760df17347cd24d6a`. +Input was generated from the repository's processed-backtest fixture, expanded +to daily-separated nights with synthetic `60 + sin(minute / 10) * 8` BPM, +HRV index 30 and breathing rate 14. No personal recordings were transferred. +The worker and synthetic input/output existed only in a unique temporary pod +directory, which was removed after measurement. No live database, service +configuration or hardware controls were changed. + +The report exceeded the old 2 MiB inline limit. That concrete failure motivated +the large-report artifact path rather than reducing dataset size or dropping +traces. The worker was invoked directly for measurement; the old repository +would have rejected publication of this report. + +Still required before choosing final admission limits: three distinct candidate +configurations, repeated and worst-case inputs, wall-clock startup/serialization, +artifact I/O, CPU utilization, SQLite contention, ingest age, temperatures and +control/API responsiveness while replay runs. Service `active` status alone does +not prove those properties. This measurement does not cover waveform replay or +untrusted algorithm execution, and does not justify a universal speed claim. + +## Three-configuration follow-up + +On October 12, the same pod and worker hash completed three consecutive runs +with distinct default/current/candidate configuration hashes. Input remained +27 synthetic twelve-hour nights (19,440 readings). The deep-stage HR-ratio +thresholds were 0.92, 0.90, and 0.94 respectively. Other input and protocol +settings were unchanged, including assumed calibration quality zero. These +are distinct evaluations, but this fixture does not exercise every stage branch +or establish a worst-case runtime. + +| Run | Wall time, including child startup/output | Evaluator time | Peak worker RSS | JSONL output bytes | +| --- | --- | --- | --- | --- | +| 1 | 4,295.804 ms | 3,397.305 ms | 156,036 KiB | 3,675,631 | +| 2 | 4,163.752 ms | 3,252.410 ms | 141,956 KiB | 3,675,638 | +| 3 | 4,247.537 ms | 3,330.915 ms | 145,188 KiB | 3,675,631 | + +A Python harness sampled local read-only performance, sensor-source, and device +status HTTP endpoints approximately every half second (requests are sequential, +so request time extends the interval). It recorded only latency, frame age, +resource counters and thermal-zone-zero temperature. It used four seconds of +baseline and three seconds afterward. There were seven baseline samples, +twenty-three replay samples and four afterward; all requests succeeded. + +| Measurement | Baseline | During replay | After | +| --- | --- | --- | --- | +| Minimum available memory | 1,358,520 KiB | 1,215,752 KiB | 1,328,848 KiB | +| Median aggregate CPU busy | 23.78% | 56.65% | 22.22% | +| Maximum aggregate CPU busy | 28.76% | 76.68% | 29.92% | +| Maximum zone-zero temperature | 33.503 °C | 37.356 °C | 30.934 °C | +| Maximum performance GET latency | 34.92 ms | 328.04 ms | 26.11 ms | +| Maximum sensor-source GET latency | 103.29 ms | 45.32 ms | 27.32 ms | +| Maximum device-status GET latency | 51.58 ms | 108.24 ms | 74.00 ms | +| Maximum reported live-frame age | 457 ms | 514 ms | 418 ms | + +CPU busy is the change in `/proc/stat` non-idle counters across adjacent samples +in the same phase, aggregated across all four CPUs. It includes the running +services and measurement harness. The server's lifetime event-loop p95 remained +21.14 ms; that lifetime histogram cannot establish the replay interval's p95. +The sensor endpoint reports the newest frame of any sensor type, so this is not +proof of piezo-specific freshness. Temperature zones one through three were +unreadable; only zone zero was sampled. Status GET latency is not proof of +hardware-write or closed-loop control responsiveness. + +The harness read the worker's complete output file and extracted operational +counters; it did not run the production Node result parser, artifact publisher +or database transactions. Thus parent-process event-loop cost, persistent +artifact I/O, SQLite contention, larger/reference-heavy fixtures, waveform replay +and sustained-load qualification remain outstanding before final admission limits +are selected. Node ran at nice 19 with a 128 MiB heap and a 45-second external +timeout. Both services remained active afterward. Temporary synthetic input, +worker, harness and output were removed from the pod; no live configuration, +observations or hardware settings were changed. + +## Production runner and reference-heavy qualification + +A follow-up on October 12 used the production `DatasetStore`, `BacktestJobStore`, +child driver, result publication, and integrity-checked result reader against a +new synthetic SQLite database on the pod's persistent storage. The application +service and its databases were left untouched. The reusable harness is +`scripts/benchmarks/biometrics-runner.ts`; bundle it with esbuild for Node, leaving +`better-sqlite3` external, and pass a **new** directory, copied migration +folder, packaged worker path, and `mixed` or `dense` fixture name. Directory +creation is exclusive so an existing database cannot be selected accidentally. + +The tested worker SHA-256 was +`45547553cd456cba5051fe6e60e8562b5510ae32bbf8b92cfeb62c59cdeb59d1`. +The coordinator used Node's 384 MiB heap limit; its evaluator child retained +nice 19 and the 128 MiB heap limit. These are measurement settings, not yet an +approved deployment policy. + +The mixed fixture retained 20 twelve-hour nights, 14,400 vital readings, +3,600 movement rows, and 1,600 heart-rate/stage reference records. It includes +periodic missing HR readings, motion, and varying HR/HRV/breathing. Chronological +splitting evaluated 12 tuning nights. The three configurations have distinct +deep-stage HR-ratio thresholds; assumed calibration quality is explicitly one. +This synthetic fixture measures execution and does not validate biometrics. + +| Measurement | Run 1 | Run 2 | Run 3 | +| --- | --- | --- | --- | +| Coordinator execution through publication | 12,007.73 ms | 11,543.85 ms | 11,673.61 ms | +| Evaluator runtime | 5,202.12 ms | 4,962.23 ms | 5,048.52 ms | +| Evaluator peak RSS | 168,392 KiB | 167,180 KiB | 174,336 KiB | +| Integrity-checked result read | 2,286.37 ms | 2,256.85 ms | 2,267.35 ms | +| Coordinator event-loop maximum | 2,386.56 ms | 2,300.58 ms | 2,365.59 ms | +| Coordinator event-loop p95 | 22.64 ms | 22.95 ms | 20.92 ms | +| Coordinator process peak RSS (cumulative) | 257,564 KiB | 261,296 KiB | 262,292 KiB | + +All three attempts succeeded, produced 1,813,618-byte reports, and left the +synthetic canonical vitals unchanged. Dataset creation took 4,322.13 ms. These +reports remain below the 2 MiB inline threshold, so this workload does not +qualify the artifact path. Minimum available system memory during the replay +phase was 1,140,300 KiB and maximum zone-zero temperature was 39.457 °C. + +**Admission is not qualified by these results.** The coordinator's multi-second +synchronous work would block the application if wired into its main event loop. +Dataset construction and result reads have the same issue. Move these operations +into a bounded coordinator worker before enabling production execution, then +repeat the benchmark. A favorable p95 conceals the long individual stalls. + +The harness's own HTTP requests and piezo receipt callbacks run on that same +coordinator event loop. Their delays therefore cannot be attributed solely to +the application server or sensor ingestion. The separate standard-library probe +`scripts/benchmarks/biometrics-probe.py` measures loopback API latency from another +process. Neither probe retains API bodies or raw sensor samples. The Node +probe retains only sensor timing and resource measurements. This remains +synthetic, non-production-database testing: it does not establish live SQLite +contention, hardware-write responsiveness, waveform replay, or untrusted-code +isolation. + +### Large artifacts and independent API probe + +The dense fixture used the same worker with 19,000 synthetic vitals, no movement +rows, and 800 reference records across 20 nights. Twelve tuning nights produced +reports of 2,301,040, 2,301,041, and 2,301,034 bytes. All three stored result +descriptors were verified as `biometric-report-artifact-v1`, all result reads +passed integrity checks, and canonical synthetic vitals were unchanged. + +| Measurement | Run 1 | Run 2 | Run 3 | +| --- | --- | --- | --- | +| Coordinator execution through publication | 13,236.50 ms | 12,901.88 ms | 13,064.22 ms | +| Evaluator runtime | 5,501.95 ms | 5,455.62 ms | 5,499.04 ms | +| Integrity-checked artifact result read | 2,803.76 ms | 2,933.55 ms | 2,973.02 ms | +| Coordinator event-loop maximum | 2,961.18 ms | 2,780.82 ms | 2,885.68 ms | +| Evaluator peak RSS | 166,692 KiB | 161,240 KiB | 162,472 KiB | + +Dataset creation took 4,477.81 ms. Coordinator peak RSS reached 262,456 KiB; +minimum available system memory during replay was 1,105,904 KiB; zone-zero +maximum temperature was 40.274 °C. Piezo source-timestamp age seen by the +coordinator peaked at 4,728 ms during replay versus 2,584 ms in its baseline. +Because this callback shares the blocked coordinator loop, that measurement +still combines actual frame age with callback delay. + +The independent Python probe made 12 baseline samples, 251 samples spanning +fixture setup and all three attempts, and 9 afterward. Every GET returned 200. + +| Maximum HTTP latency | Baseline | Entire benchmark | After | +| --- | --- | --- | --- | +| Performance | 36.88 ms | 57.52 ms | 22.48 ms | +| Sensor source | 93.53 ms | 72.37 ms | 22.72 ms | +| Device status | 66.74 ms | 80.61 ms | 52.32 ms | + +The production service remained active after the benchmark. These observations +support keeping heavy coordination separate from the application event loop; +they do not approve running it on that loop. The harness's configuration labels +are synthetic comparison roles (thresholds 0.90, 0.92, 0.94), not a promotion +comparison against an asserted shipped-default profile. Repeat qualification +after coordinator isolation and again for the final deployed worker. The +measurements do not yet select universal admission thresholds or qualify live +SQLite contention and closed-loop control responsiveness. + +## Isolated coordinator follow-up + +The coordinator implementation at `0e5c87c0` was measured on the same pod on +October 12 using the dense fixture above. The optional final `COORDINATOR` +argument to `scripts/benchmarks/biometrics-runner.ts` routes dataset creation, +experiment creation, execution, and report reads through the shipped coordinator. +The caller uses the same 384 MiB heap cap as the preceding benchmark; each +coordinator has a 384 MiB cap and nice 19, and its evaluator retains 128 MiB/nice +19. Each operation gets a fresh process. The worker hash remains +`45547553cd456cba5051fe6e60e8562b5510ae32bbf8b92cfeb62c59cdeb59d1`. + +An initial isolated run succeeded three times. The repeat below adds a timer +turn after report serialization, so its event-loop maximum includes final IPC +deserialization and JSON serialization as well as execution/publication. This +is a calling-process measurement, not the production application's event-loop +histogram. Both runs used exclusively created synthetic databases and retained +no personal recordings or API response bodies. + +| Measurement | Run 1 | Run 2 | Run 3 | +| --- | --- | --- | --- | +| Execution through publication | 14,955.19 ms | 15,390.62 ms | 15,141.36 ms | +| Integrity-checked full report read | 4,178.71 ms | 4,418.67 ms | 4,237.17 ms | +| Calling-process event-loop maximum | 144.83 ms | 187.83 ms | 174.59 ms | +| Calling-process event-loop p95 | 21.22 ms | 21.04 ms | 21.12 ms | +| Calling-process cumulative peak RSS | 153,428 KiB | 156,940 KiB | 156,940 KiB | +| Evaluator peak RSS | 164,460 KiB | 171,136 KiB | 162,032 KiB | + +All three attempts succeeded; full reports remained approximately 2.30 MB and +canonical synthetic vitals were unchanged. Dataset construction took 5,693.98 ms +with a calling-process event-loop maximum of 56.30 ms. During replay, minimum +available system memory was 1,046,688 KiB, maximum zone-zero temperature was +41.909 °C, and maximum piezo source timestamp age was 2,756 ms. This does not +measure the coordinator's peak RSS; the memory values above must not be summed +as a complete process-tree maximum. + +The independent probe recorded 12 baseline samples, 291 spanning the complete +benchmark, and nine afterward. Every request returned HTTP 200. + +| Maximum HTTP latency | Baseline | Entire benchmark | After | +| --- | --- | --- | --- | +| Performance | 30.48 ms | 160.12 ms | 38.15 ms | +| Sensor source | 27.47 ms | 109.92 ms | 24.23 ms | +| Device status | 65.74 ms | 159.46 ms | 76.02 ms | + +Isolation removed the multi-second calling-loop stalls observed previously, +but full-report IPC/serialization still transfers large trace arrays. In one +retained synthetic report, `nights` occupied about 2.21 MB; the first evaluated +window's `blocks` alone occupied 56,786 bytes. A bounded summary for interactive +review, with exact traces fetched through the existing paginated endpoint, is +an appropriate next optimization. Keep the complete immutable report and its +hash; a transport projection must not silently redefine the retained artifact. + +These observations still do not qualify live-database write contention, +closed-loop hardware responsiveness, waveform processing, or arbitrary-code +isolation. No production resource policy or automatic background execution was +enabled by this benchmark. Temporary benchmark databases and binaries were +removed after measurements were copied locally; the production service was +left running without a deployment or restart. + +## Retained waveform pipeline on the pod + +The October 12 follow-up used the coordinator and Node evaluator built from +`fe25e71a`, plus the Python process-limit compatibility fix accompanying this +measurement. The pod's Python 3.10.4 installation omits the optional `resource` +extension. Workers now use `/usr/bin/prlimit` on Linux when that extension is +missing, failing startup if the utility fails. A separate child on the pod +confirmed CPU soft/hard limits of 120/121 seconds, core limits of zero, and +open-file limits of 64/64 in `/proc/self/limits`. + +The new `scripts/benchmarks/biometrics-waveform-fixture.py` creates deterministic +500 Hz dual-channel sine-plus-noise CBOR archives in an exclusively created +directory. The runner `biometrics-waveform-runner.ts` creates a separate database, +retains those archives through the production publisher, enqueues three unique +configuration hashes, executes the leased waveform pipeline, and reads its +summary. The first five minutes are retained pre-roll. Configuration labels +are synthetic roles; this fixture does not measure biological accuracy. + +The 30-minute fixture contained 5,976,451 compressed bytes. Dataset creation +took 1.309 seconds and retention took 15.809 seconds. Execution reached its +180-second coordinator limit before publishing a result. Canonical synthetic +observations were unchanged. **This workload is not qualified for automatic +execution.** Full-night execution needs a different scheduling/budget design; +simply enabling the existing host would not make these jobs complete. + +The independent probe sampled the benchmark process and its descendants through +`/proc`, in addition to the existing loopback HTTP measurements. RSS is a sampled +sum, not an exact peak or unique physical-memory accounting. The failed +30-minute run produced 12 baseline, 448 benchmark, and nine post-run samples. +All sampled HTTP responses returned 200. + +| Measurement | During the complete 30-minute-fixture benchmark | +| --- | --- | +| Maximum sampled process-tree RSS | 313,332 KiB | +| Minimum available system memory | 1,181,144 KiB | +| Maximum zone-zero temperature | 40.041 °C | +| Maximum performance GET latency | 49.01 ms | +| Maximum sensor-source GET latency | 79.46 ms | +| Maximum device-status GET latency | 115.39 ms | +| Maximum age of piezo WebSocket receipt | 1,446 ms | + +The receipt age is measured at the benchmark client; it is not the age of the +original sensor timestamp. The calling process's largest sampled scheduling +delay was 179.88 ms, so favorable HTTP timings alone do not establish its event +loop budget. Measurements use synthetic databases and do not qualify live +SQLite write contention or hardware-command latency. No production admission +policy is selected by these results. + +A 10-minute follow-up (five minutes pre-roll and five minutes scored) completed +the entire pipeline with the same three configurations. Its 1,992,541 compressed +bytes took 15.599 seconds to retain. Leased replay and publication took 83.402 +seconds, and integrity-checked summary retrieval took 1.289 seconds. The summary +was 28,798 bytes; synthetic canonical observations again remained unchanged. +The replay caller's event-loop p95/max were 22.00/63.90 ms, including a timer +turn after the coordinator reply. This is a successful short fixture, not a +qualified whole-night workload. + +The independent probe recorded 12 baseline, 247 benchmark, and nine post-run +samples, all with HTTP 200 responses. During the benchmark, process-tree RSS +reached 314,852 KiB, available memory stayed at or above 1,184,736 KiB, and +zone-zero temperature reached 39.924 °C. Maximum performance/sensor-source/device +GET times were 155.93/96.78/75.31 ms. Maximum piezo receipt age was 1,626 ms. + +The successful run pinned compound build +`b91e8552bc6d973922bd38808d04490683e3e05d8c67c9af5ad37d40a64bfe45` +and Python build +`fad13cfff0b361fbec950e0b6ddd892fe3d97f42f3ef9c5031af2e700c579aca`. +Its complete report hash was +`8446b1a8fd4c0f1530c25ed186ac5231c8020c60e4dffc42b1ec248aa2b46291`. +Python used the pod's installed NumPy 2.2.6 and SciPy 1.15.3. The staged +benchmark used fixed production worker bundles, not the temporary diagnostic +coordinator used to investigate the missing Python extension. + +After both runs, no benchmark descendants remained. Synthetic telemetry and +logs were copied locally, and the staging directory was removed. Both installed +services remained active on the original `homekit-0b53800` release. No deployment, +restart, live-database modification, or hardware command was part of this test. + +### Where waveform execution time goes + +A subsequent synthetic 30-minute profile on the same pod separated startup +from processing. The Yocto Python installation also lacks `_lsprof`, so the +diagnostic used `time.monotonic()` and `time.process_time()` around the shipped +functions. These timings include small wrapper overhead and are not a complete +function profiler. It retained the same noisy dual-channel fixture in a new +directory and discarded derived readings after counting them. + +| Operation | Wall time | CPU time | +| --- | --- | --- | +| Import replay and numerical dependencies | 12.726 s | Not measured | +| Retain the synthetic archive | 0.369 s | Not measured | +| Complete one-configuration retained replay | 45.184 s | 44.751 s | +| Inner CBOR decoding (included above) | 0.596 s | 0.583 s | +| Per-side ingestion (included above) | 41.623 s | 41.238 s | +| Numerical estimates (included in ingestion) | 30.365 s | 30.082 s | + +The run consumed 1,860 records including 1,800 piezo frames and produced 50 +scored readings plus ten pre-roll readings. These overlapping timings must not +be summed. Both startup and numerical processing matter; archive decoding was +not the dominant cost. A full-night run still needs direct measurement rather +than extrapolating this short profile into a production runtime guarantee. + +### Interrupted eight-hour qualification (2026-10-13 UTC) + +The three-configuration, eight-hour synthetic run on `192.168.1.88:8822` +was cancelled during its third configuration after systemd explicitly stopped +`sleepypod` at 01:09:44 UTC. Core and piezo services restarted at 01:10:17 UTC; +`NRestarts=0` and the journal showed the stop request and SIGTERM. This task did +not issue that stop. The installed release remained `homekit-0b53800`. +The admission monitor withdrew permission when HTTP health disappeared. This +run is **not a completed qualification**, regardless of the operation timer's +`succeeded` field (which only records that the execution RPC returned). + +Before cancellation, the benchmark recorded 293 concurrent descriptor reads, +zero read failures, read p95 1520 ms / maximum 2077 ms, 5962 HTTP probes with six +failures at shutdown, HTTP p95 42 ms / maximum 937 ms, at least 1,045,213,184 bytes +available memory, maximum temperature 44.594 C, maximum piezo receipt age 1768 ms, +and peak benchmark process-tree RSS 396,940 KiB. Canonical observations remained +unchanged. These observations support cancellation behavior but do not establish +a production resource policy or a successful full-night replay. + +Synthetic measurements and probe metadata were copied locally for analysis. +After confirming all benchmark descendants had exited, its exact staging directory +was removed; both installed services were active. Repeat full qualification in an +uninterrupted window before enabling the production replay host. + +Retained RAW cleanup runs through the coordinator's serialized work lane when +its host performs startup/periodic collection. It uses a fixed Python module, +server-derived database/storage paths, a two-minute deadline, bounded diagnostics, +and a count-only response. Admission is checked before execution and every five +seconds; host shutdown cancels the child. Released artifacts and unpublished +orphans retain their one-hour grace periods and cross-process publication locks. +An absent retained directory is a no-op. Enabling the production host still +requires deployment qualification; adding cleanup does not enable replay. + +### Per-run process measurements + +New retained waveform runs include `resources` with version +`waveform-process-resources-v1`. Monotonic wall time and process CPU time cover +input reading, build verification, replay, and the final build check through +result generation. They exclude interpreter startup before `run`, final result +serialization, and teardown. Peak RSS is the kernel process-lifetime high water +mark: Linux `/proc/self/status` `VmHWM` converted from KiB to bytes (including +Yocto without Python's `resource` extension), or Darwin `ru_maxrss` in bytes. +Unavailable telemetry is null; current RSS is never substituted for a peak. + +Waveform reports summarize the sequential processes as total measured wall/CPU +time and the largest individual process memory peak. Shared configuration hashes +are executed once and are counted once. Missing legacy measurements make the +corresponding aggregate unavailable. Compact report projections retain these +measurements, and the review UI labels their units and scope. They exclude Node +coordination/scoring and do not measure live ingestion health, so they cannot +alone satisfy a whole-job operational promotion policy. Execution measurements +are part of the retained report hash; replay determinism is assessed on derived +readings rather than equality of timing counters between executions. + +Successful jobs also retain `jobOperations` (`backtest-job-operations-v1`), +written by the coordinator after its final admission check rather than supplied +by the scoring child or experiment. Its monotonic duration runs from initial +admission through publication preparation. The supervisor sends ingestion age +from the same fresh health sample used to admit work over the internal process +channel. Only count, maximum observed age, largest monotonic observation gap, +and qualification identity are retained; raw sensor frames are not included. + +`allChecksMeasured` means every admission check supplied evidence under one +qualification identity, with at least initial and final checks. It does not mean +continuous monitoring: the nominal check interval is five seconds, delays are +reported, and intervening health is unobserved. Boolean-only admission callbacks +used by older tools/tests remain supported but produce unavailable health +evidence. A failed or invalid health callback cannot fall back to a permissive +boolean callback. Admission withdrawal still interrupts work without publishing +a result. The waveform report UI shows these observations separately from Python +process costs. Whole-job memory evidence and designated promotion policy remain +separate requirements. + +The supervisor now also samples summed RSS for the owned replay process group +at a nominal one-second interval, plus admission boundaries. Linux procfs scans +are bounded to 4,096 process IDs and 64 group members with at most 16 concurrent +reads. Only group IDs and RSS are read; arguments, environments and sensor data +are excluded. Missing procfs, unsupported platforms, malformed data, or exceeded +bounds yield unavailable samples. Exited processes may disappear during a scan. + +The final admission receipt carries this accumulator into `jobOperations.memory` +(`process-group-rss-observations-v1`): attempts, valid sample count, largest +observed summed RSS and largest observation gap. The coordinator and its worker +children share the owned group; the main application process is outside it. +The sampler stops when the coordinator closes, with no detached polling loop. +This is not an exact kernel peak: short-lived processes or peaks between scans +may be missed, scans are not atomic, and shared pages may be counted more than +once. Reports retain these scope limitations rather than substituting this value +for the per-process lifetime high-water marks. + +### Completed eight-hour synthetic qualification (2026-10-13 UTC) + +The uninterrupted run on `192.168.1.88` completed all three distinct +configurations over the 28,800-second dual-channel 500 Hz synthetic fixture +(95,621,299 compressed bytes). Staged production code was `e1ab3942`, with the +completion-validating benchmark runner from `2ff9bf58`. Completion required a +successful job, retained waveform report, matching dataset/Python identities, +three unique expected configuration hashes, and positive scored outputs for each. +Canonical synthetic observations remained unchanged. + +- Replay: 2,625,664 ms (43.8 minutes); retention: 22,064 ms; summary: 2,849 ms. +- 366 concurrent dataset reads, zero failures; p95 1,516 ms, maximum 1,867 ms. +- Internal probe: 7,456 API requests, zero failures; p95 41.5 ms, maximum 311 ms. +- External probe: 16,905 API requests, zero failures; p95 38.6 ms, maximum 1,429 ms. +- Peak observed benchmark process-tree RSS: 413,163,520 bytes (394 MiB). +- Minimum sampled available memory: 1,132,576,768 bytes; maximum temperature 43.777°C. +- Maximum sampled ingestion age: 1,250 ms. In 68 samples a newer frame arrived + after the collector captured its start timestamp, producing a negative age; + those are collection-timing artifacts, not measured negative latency. +- Replay parent event-loop p95 23.1 ms, maximum 96.7 ms. + +Retained report hash: +`bc1ac53d9168a5f2e6fd521212c69f6ae53939911e7ba17e4f1ccb0fb8e7cdcc`. +Compound build: +`375b2ae0e0e81545b234f1ee3357a2b0cd4cde222cd413591c71c88766b75607`. +Python build: +`23670017f01a37b0281fdea016440261af6f9e6519bff5f294b9056d6f5caf29`. + +Synthetic measurements, probe output, and compact summary were retained locally +under `/tmp/biometrics-waveform-qualification-results/fullnight-success/`. +The owned pod staging directory was removed only after successful termination +and a descendant-process check. Core (`sleepypod.service`) and piezo services +remained active and the health API returned 200 afterward. These results qualify +the stated staged build and fixture; later telemetry and objective changes need +additional target verification. Synthetic signal recovery does not establish +agreement with real reference exports or clinical accuracy. + +A subsequent target smoke test exercised the new process-group sampler against +an isolated Node child holding a touched 32 MiB buffer. Four of four observations +were available; peak observed RSS was 83,582,976 bytes, maximum observation gap +999.8 ms. The child exited and the temporary sampler directory was removed; +both application and piezo services remained active. This verifies Linux +collection and cleanup, not a repeat full-night qualification of newer code. + +### Frozen sampled operational objectives + +Numeric objective policies may explicitly opt into +`operational.observations.version = sampled-process-group-v1`, with maximum +memory and ingestion observation gaps. Without that opt-in, sampled RSS cannot +satisfy the existing peak-memory field. After the final admission check, the +coordinator replaces the scorer's operational evidence with its own measured +job duration, group RSS, and ingestion observations. Missing/partial samples +remain unavailable; excessive observation gaps or a qualification identity +mismatch prevent improvement. Threshold values and sampling semantics remain +part of the frozen experiment protocol. This supplies operational evidence; +a designated qualified policy, sufficient independent reference support, and +subgroup checks are still required for validated promotion. + +### Current-build 30-minute smoke (2026-10-13 UTC) + +An isolated run of `4453c0a5` on the same aarch64 pod (Node 24.16.0, locked +Python 3.10.4 environment) completed three distinct configurations over an +1,800-second dual-channel 500 Hz synthetic fixture, 5,976,451 compressed bytes. +It used the current packaged coordinator/evaluator and common Python sources, +including numerical flock, shared application admission and process-group cleanup +confirmation. Production services continued running; this was not a deployment. + +- Replay 202,760 ms; retention 16,172 ms; summary 1,542 ms. +- 27 concurrent dataset reads, zero failures; p95 1,860 ms. +- 638 internal API requests and 1,545 independent probe requests, zero failures; + respective p95 latencies 65.7 ms and 41.6 ms. +- Peak sampled benchmark process-tree RSS 433,369,088 bytes (413 MiB). +- Minimum available memory 992,690,176 bytes; minimum free disk 3,450,945,536 bytes. +- Maximum temperature 46.696°C; maximum measured ingest age 992 ms. Nine negative + ages reflect collector/frame timing, as in the earlier full-night run. +- Replay parent event-loop p95 28.0 ms, maximum 186.4 ms. + +Completion verifies a retained report, exact dataset/Python/configuration identities, +and positive scored outputs for each configuration. Synthetic canonical rows +remained unchanged. After termination, no process referenced the owned staging +directory; it was removed. Core and piezo services remained active and the health +API returned 200. No production databases, bindings or service configuration changed. + +The [compact measurement summary](performance/biometrics-replay-2026-10-13/smoke-4453c0a5.json) +records exact result/build hashes, explicit test budgets and SHA256 identities of +the inputs and telemetry. Full synthetic telemetry is retained locally at +`/tmp/biometrics-qualification-4453c0a5/`. These are sampled observations, not exact +kernel peaks or continuous guarantees. They verify this short replay workload; +full-night current-build, representative-reference and prospective shadow +qualification remain outstanding. The measured event-loop maximum also prevents +interpreting the admission samples as a guarantee that all delays stayed below +the benchmark's 100 ms admission threshold. No promotion registry or production +budget defaults were derived from this smoke alone. + +### Synthetic shadow supervisor measurement harness + +`scripts/benchmarks/biometrics-shadow-runner.ts` exercises the shipped Node +supervisor and kernel-limited Python shadow child with generated 500 Hz dual +channels. It retains bounded resource receipts, partial failure observations, +output identity, exact Python/runner identities and parent event-loop measurements. +It uses no live waveform subscription, database, credentials or hardware API. + +Bundle with esbuild (`--bundle --platform=node --target=node22 --format=cjs`), +then run on the target Linux pod with arguments `OUTPUT_JSON PYTHON_PATH +MODULES_PATH [SECONDS]`. The default is 1,800 synthetic seconds, processed as +fast as the worker permits; accepted durations are 150–28,800 seconds. Use the +locked installed Python environment and the source modules matching the build. +Wrap it in `biometrics-probe.py` for independent live-service telemetry. Run only +after other replay/shadow measurements finish: the child uses their same kernel +execution lock. The harness's explicit ceilings are measurement stop conditions, +not installed qualification or recommended deployment limits. + +This measures synthetic processing and supervision only. It does not establish +live capture/sink performance, representative artifact robustness, held-out +accuracy or permission to install a shadow qualification registry. + +### Current worker eight-hour run (2026-10-13) + +The current worker/coordinator bytes completed three configurations over 28,800 +synthetic seconds at 500 Hz on the same four-core ARM64, 2 GiB pod. The staged +source was `db52f1da`; subsequent API/UI/prompt changes leave the measured worker +and coordinator hashes unchanged. Exact hashes and artifact identities are in +[the compact evidence](performance/biometrics-replay-2026-10-13/fullnight-db52f1da.json). + +- Replay: 2,756.7 seconds (45.94 minutes); retention: 21.74 seconds; summary: + 2.99 seconds. All operations completed and fixture canonical rows were unchanged. +- 363 concurrent report reads: no failures, p95 1,943.7 ms, maximum 2,183.3 ms. +- 7,390 internal HTTP observations: no failures, p95 57.1 ms, maximum 300.8 ms. + The independent probe recorded 17,568 observations, also without failures, + p95 39.6 ms and maximum 500.1 ms. +- Peak sampled process-tree RSS: 443.1 MiB; minimum available memory: 902.6 MiB; + minimum filesystem space: 3.02 GiB; maximum temperature: 49.965 °C. +- Aggregate CPU busy averaged 68.1%, peaking at 99.0% over a sampling interval. + Maximum one-minute load per CPU was 1.0575. These include live services and + the probe, not just the replay child. +- Maximum observed piezo arrival age: 1,560 ms. There were 71 negative age + differences because the collector records its start before asynchronous reads + and can observe a newer frame afterward; those are not negative ingest latency. +- Parent event-loop p95 during replay: 27.28 ms, maximum 154.01 ms. A 100 ms + admission check is therefore not a continuous event-loop latency guarantee. + +The original process handle exited successfully and no owned worker descendants +remained before the separate shadow measurement started. Core and piezo services +were active afterward. The read-only HTTP probes do not prove actuator-write +latency or clinical performance. Raw measurements remain under +`/tmp/biometrics-qualification-db52f1da/` on the measurement workstation; the +committed evidence records their SHA-256 hashes. No live database or profile was +changed by this benchmark. + +### Shadow supervisor measurement (2026-10-13) + +After the full-night replay released its worker slot, the staged shadow harness +processed 1,800 synthetic seconds at 500 Hz through the shipped Linux process +wrapper and Node supervisor. It completed in 61.34 seconds, producing 30 outputs +and reporting 29.17 seconds of numerical CPU time. The child process-group RSS +peaked at a sampled 113.15 MiB; the independent probe observed 256.5 MiB for the +whole benchmark tree. The completion receipt retains cold-start and insufficient +preroll limitations. + +The probe recorded 456 read-only HTTP observations with no failures (p95 45.41 ms, +maximum 130.39 ms). Minimum available memory was 1,019.5 MiB and maximum temperature +47.513 °C. Parent event-loop p95 was 26.10 ms and maximum 68.62 ms. RSS observation +gaps reached 333.42 ms; this is sampled memory evidence, not an exact kernel peak. + +[The compact receipt](performance/biometrics-replay-2026-10-13/shadow-4ee74c25.json) +retains build identities, completion/cleanup observations, limits and raw-artifact +hashes. The original SSH handle exited zero, no owned descendants remained, and +both live services were active before the owned staging directory was removed. +This run does not exercise the live source/database sink, establish held-out +accuracy or authorize a shadow qualification registry. + +### Target-specific replay deployment policy + +The measured pod uses the [recorded policy](performance/biometrics-replay-2026-10-13/deployment-policy.json). +Its admission floor is 768 MiB available memory and 2 GiB free disk; it refuses +load per CPU above 1.1, temperature above 55 °C, piezo age above five seconds, +stale health samples above two seconds, or a scheduling probe above 100 ms. +These are conservative operating cutoffs around the observed workload, not +universal hardware defaults or guarantees that every transient stays below them. +A temporary refusal is expected when current health exceeds a cutoff. + +The whole-job deadline is two hours; each numerical waveform child has a +30-minute wall deadline and 1,200 CPU-second limit, matching the successful +full-night measurement. Larger jobs can still exceed these ceilings and fail +explicitly. Existing worker serialization, process limits, quotas and health +checks remain enforced. This policy authorizes bounded replay on this measured +pod only; it provides no accuracy designation or shadow qualification. + +Deployment stores the two environment values in +`/etc/sleepypod/biometrics-replay.env`, loaded by +`/etc/systemd/system/sleepypod.service.d/biometrics-replay.conf`. A service restart +is required after changing them. Remove that drop-in and restart to return to +unconfigured admission. No user profile or hardware temperature setting changes +when installing this policy. + +### Live source and isolated sink measurement (2026-10-13) + +The [live operational receipt](performance/biometrics-replay-2026-10-13/live-shadow-operational.json) +records a successful 180-second observation through the loopback sensor stream, +read-only active observations, shared numerical worker and fenced isolated sink. +It retained six comparison points (5,396 bytes), with zero canonical rows in the +isolated database. The worker used 4.95 CPU seconds, sampled peak group RSS of +108.61 MiB and a maximum RSS observation gap of 497.24 ms. The independent probe +made 1,587 read-only HTTP requests with no failures (p95 40.21 ms, maximum +123.67 ms); minimum available memory was 1,082.67 MiB and maximum temperature +44.828 °C. No raw frames or numerical readings were exported in this receipt. + +Two earlier attempts failed: one refused health admission; another reported +unavailable RSS after its worker completed. The latter exposed a process-exit +race fixed in `4c3a6a6e`: disappearance after a validated terminal response now +requires bounded, successful pipe closure. Missing RSS during processing, bad +exit status and a worker that never exits still fail. Regression tests reproduce +all three outcomes. The successful repeat used the fixed supervisor without +relaxing resource limits. Its SSH process exited zero, owned processes were +absent, and all three owned staging directories were removed after saving +metadata receipts locally. + +This trusted measurement harness uses read-only production handles and a fresh +isolated database. It does not install a qualification registry, alter profile +bindings or exercise production registry admission. Cold start, insufficient +preroll, missing occupancy context and missing pump context remain recorded; +this operational check cannot establish accuracy or authorize promotion. diff --git a/docs/biometrics-runtime-activation.md b/docs/biometrics-runtime-activation.md new file mode 100644 index 000000000..77d11fcbb --- /dev/null +++ b/docs/biometrics-runtime-activation.md @@ -0,0 +1,142 @@ +# Biometric runtime activation protocol + +`requestManualActivation` is the configuration transaction for an explicitly +unvalidated manual change. It checks side, exact revision/hash, expected binding +generation and idempotency key before committing the binding, audit event and +immutable outbox payload together. A missing legacy migration prevents a new +binding from discarding existing stage personalization. Reset selects the +current shipped default; rollback selects the previous binding and remains a +distinct audited action. Neither operation deletes revision history. + +An explicit reset can recover malformed legacy stage data that cannot pass the +stage-classifier schema. The original legacy row remains untouched, and the +reset event preserves its value as `evidence.preservedLegacyClassifier` with +`recovery: "invalid_legacy_reset"`. This is audit data, never an executable +profile revision. The binding, evidence and default outbox payload commit +atomically; a publication failure leaves the original state intact. Valid +legacy customizations still require migration before any new binding, including +reset. Recovery does not claim processor adoption. + +For a bound revision whose engine is unavailable, the web recovery view shows +its preserved revision ID and the current default recovery hash. It cannot +produce a valid before/after diff of the unavailable engine and explicitly says +so. The user can request an acknowledged, generation-checked reset to defaults; +the old revision remains the previous binding. Rollback to that revision still +requires its engine to be compatible and never silently substitutes defaults. + +`projectRuntimeConfig` validates the manifest/configuration identity and writes +an immutable side/generation projection into the biometrics database. Duplicate +identical delivery succeeds; a different payload for an existing generation +fails. `deliverRuntimeOutbox` acknowledges delivery only after projection. +Transient failures retry with capped exponential delay; invalid envelopes fail +visibly. These are separate database transactions, so a crash after projection +is handled by idempotent redelivery rather than assumed cross-database atomicity. + +`repairCurrentRuntimeProjections` restores the latest authoritative delivered +payload after biometric history has been cleared. It preserves configuration +ownership and does not manufacture runtime receipts. + +`reconcileActivation` considers only receipts matching the desired generation +and exact bundle hash. Heart-rate, breathing-rate, HRV-index and sleep-stage +components must all report applied with an effective timestamp before the +activation is marked applied. Old-generation or wrong-hash receipts cannot +acknowledge a newer binding. A final generation check fences concurrent intent +changes while reconciling. + +The Node startup hook starts one `globalThis` coordinator after migrations. +Every five seconds it migrates legacy stage personalization, repairs delivered +projections, delivers pending intents and reconciles receipts. Shutdown stops +its timer. Public manual activation, reset and rollback submit exact intent; +status polling remains read-only. Availability does not assert processor health +or adoption. The cross-runtime fixture verifies the public request → Python +boundary → stage receipt → restart → rollback handoff. +There are no hardware-command paths in these services. + +The Python `common.biometrics.runtime` reader independently validates the latest +projected payload, its envelope, shipped manifest and resolved hash. Reads are +bounded before materializing SQLite text. Missing tables, corrupt newest +generations and incompatible engines surface errors; the reader neither creates +schema nor silently selects an older generation. Snapshots retain immutable +configuration bytes and return fresh decoded values. + +Its two-phase boundary selector allows immediate changes at the caller's next +sample boundary and next-session changes only when occupancy is explicitly +false. Unknown occupancy, including an uninitialized restart, does not count as +an empty bed. Newer requests supersede pending ones, stale generations fail, and +a late commit cannot acknowledge a superseded selection. The caller must install +the selected configuration before committing selection and writing receipts. +The live piezo daemon now uses these primitives through `runtime_adapter.py`. +It polls projections every five seconds, installs configurations before ingesting +an accepted dual-channel frame, and writes receipts only after both channel +buffers have accepted that frame. Heart-rate, breathing-rate and HRV-index +receipts are separate from the stage-view consumer's acknowledgment. Heartbeats are +bounded to once per thirty seconds unless the applied configuration changes. + +In single-sleeper mode both physical channels use the home side's profile. +Each computed candidate carries its immutable processing context through +quality arbitration and write retries, including its original logical side and +physical source channel. The observation, quality row and processing bundle +commit in one SQLite transaction. Retained contexts recreate their processing +bundle after history reset. This does not imply raw input retention: provenance +explicitly labels raw identity unavailable and timestamps as processing-clock +based. Measured sensor calibration is not consumed by this numerical path and +its calibration hash remains null. + +Numerical changes reset sample buffers, tracking and presence state; stage-only +changes preserve numerical state. Routing changes reset state to prevent mixing +sleepers. A restart restores only a compatible projection previously acknowledged +by all three numerical components, with fresh estimator warmup. Pending +next-session requests still require known empty occupancy. Invalid projections +retain the actual compatible configuration and report degraded module health; +an incompatible restored configuration falls back to defaults. Missing schema +prevents custom adoption and preserves legacy writes. The coordinator records +fallback intent separately from the consumer receipts proving its adoption. + +The stage reader selects the newest projection for which all three numerical +components acknowledged the exact bundle hash and an effective boundary. Partial +adoption of a newer generation keeps the prior view. Reading the classifier for +use in the client or server records the stage-consumer receipt; merely inspecting +profile overview or enqueueing a backtest does not. No stage receipt is generated +by the background coordinator. Until a consumer requests the new view, the +activation can truthfully remain pending. + +`getClassifier` exposes the applied classifier and provenance separately from a +pending desired classifier. The editor retains pending edits and polls for +adoption. `getSleepStages` includes the selected view with classified results; +its interpretation is explicitly `current_classifier_view`, since the rules +rescore original readings rather than replace historical observations. Backtest +current-stage baselines use this same applied view. + +Legacy `setClassifier` writes now use the profile service and durable outbox. +They retain HR, breathing and HRV overrides, and null clears only stage overrides. +The response retains classifier/isDefault fields and adds pending activation +identity. Existing legacy columns remain migration baselines, never a second +writable authority. These changes are manual/unvalidated. Demo responses label +simulated adoption explicitly. + +`recoverRuntimeIntent` handles obsolete defaults, missing intent and unavailable +pinned engines. It publishes a new generation and audit event atomically with +its outbox payload. Fallback preserves the desired pinned revision, immutable +revision history and the previous rollback target. Repeated recovery passes are +idempotent. If that same pinned engine becomes compatible again, restoration +waits for the next session. Repair of a lost custom boundary also defaults to the +next session. The fallback explanation survives acknowledgment and remains +visible in the stage view and desired-profile warning. + +A conflicting biometric projection is retained for diagnosis and its delivery +marked failed. The next recovery pass creates a new generation; it does not +rewrite the conflicting immutable snapshot. Missing projections after history +reset are simply redelivered with their original identity. + +Profile overview includes numerical processor health separately from historical +adoption. All three components must report the same generation/hash and a +heartbeat no more than 120 seconds old to be current. Missing components, +incomplete adoption, failures, future timestamps and mixed generations are +reported explicitly. The retrospective stage consumer is on demand and is not +subject to that continuous heartbeat test. This reports health; it does not +claim an old activation was never applied or automatically switch algorithms. + +`biometricTuning.listActivationHistory` / `GET /biometric-tuning/activations` +returns side-scoped audit events, fifty per page using a generation cursor. +Results include exact hashes, revision identities, evidence and fallback reason; +internal request hashes and idempotency keys are omitted. diff --git a/docs/performance/biometrics-replay-2026-10-13/deployment-policy.json b/docs/performance/biometrics-replay-2026-10-13/deployment-policy.json new file mode 100644 index 000000000..c19bec11f --- /dev/null +++ b/docs/performance/biometrics-replay-2026-10-13/deployment-policy.json @@ -0,0 +1,17 @@ +{ + "admission": { + "qualificationId": "pod-192-168-1-88-replay-20261013-bda8d03d", + "minimumAvailableMemoryBytes": 805306368, + "minimumFreeDiskBytes": 2147483648, + "maximumLoadPerCpu": 1.1, + "maximumEventLoopDelayMs": 100, + "maximumPiezoAgeMs": 5000, + "maximumTemperatureC": 55, + "maximumSampleAgeMs": 2000 + }, + "budget": { + "jobWallMs": 7200000, + "waveformWallMs": 1800000, + "waveformCpuSeconds": 1200 + } +} diff --git a/docs/performance/biometrics-replay-2026-10-13/deployment-verification.json b/docs/performance/biometrics-replay-2026-10-13/deployment-verification.json new file mode 100644 index 000000000..61be2d857 --- /dev/null +++ b/docs/performance/biometrics-replay-2026-10-13/deployment-verification.json @@ -0,0 +1,100 @@ +{ + "verifiedAt": "2026-10-13T14:14:45.388115+00:00", + "build": { + "branch": "feat/biometrics-calibration", + "commitHash": "4c3a6a6e", + "commitTitle": "fix(biometrics): handle shadow RSS disappearance during clean exit", + "buildDate": "2026-10-13T14:12:14.467Z", + "version": null + }, + "admission": { + "httpStatus": 200, + "result": { + "configured": true, + "qualificationId": "pod-192-168-1-88-replay-20261013-bda8d03d", + "admitted": true, + "reasons": [] + } + }, + "manifest": { + "httpStatus": 200, + "result": { + "manifestHash": "96c2f01a329b4346b4a3f2801ba973650d7235a5c4bd00bd0ded96f1a5768b9b", + "activationAvailable": true + } + }, + "dataPath": { + "httpStatus": 200, + "result": { + "nodes": [ + { + "id": "sensor-piezo", + "status": "ok" + }, + { + "id": "sensor-cap", + "status": "ok" + }, + { + "id": "sensor-temp", + "status": "ok" + }, + { + "id": "frames", + "status": "ok" + }, + { + "id": "dac", + "status": "ok" + }, + { + "id": "piezo-processor", + "status": "ok" + }, + { + "id": "sleep-detector", + "status": "ok" + }, + { + "id": "environment-monitor", + "status": "ok" + }, + { + "id": "dac-monitor", + "status": "ok" + }, + { + "id": "database", + "status": "ok" + }, + { + "id": "scheduler", + "status": "ok" + }, + { + "id": "out-vitals", + "status": "ok" + }, + { + "id": "out-sleep", + "status": "ok" + }, + { + "id": "out-temp", + "status": "ok" + }, + { + "id": "out-live", + "status": "idle" + } + ], + "verdict": "ok" + } + }, + "services": { + "sleepypod.service": "active", + "sleepypod-piezo-processor.service": "active", + "sleepypod-sleep-detector.service": "active", + "sleepypod-environment-monitor.service": "active" + } +} diff --git a/docs/performance/biometrics-replay-2026-10-13/fullnight-db52f1da.json b/docs/performance/biometrics-replay-2026-10-13/fullnight-db52f1da.json new file mode 100644 index 000000000..13851cfe2 --- /dev/null +++ b/docs/performance/biometrics-replay-2026-10-13/fullnight-db52f1da.json @@ -0,0 +1,93 @@ +{ + "scope": "Eight-hour synthetic replay resource measurement; not shadow, clinical or activation qualification", + "sourceCommit": "db52f1da", + "architecture": "aarch64", + "nodeVersion": "24.16.0", + "pythonVersion": "3.10.4", + "completed": true, + "fixtureSeconds": 28800, + "compressedBytes": 95621299, + "canonicalObservationsUnchanged": true, + "readFailures": 0, + "replayBudget": { + "jobWallMs": 7200000, + "waveformWallMs": 1800000, + "waveformCpuSeconds": 1200 + }, + "operations": [ + { + "phase": "snapshot", + "succeeded": true, + "elapsedMs": 1512.0559429999994, + "eventLoopP95Ms": 23.134207, + "eventLoopMaxMs": 39.550975, + "parentPeakRssKiB": 130780 + }, + { + "phase": "retain", + "succeeded": true, + "elapsedMs": 21739.24188, + "eventLoopP95Ms": 22.036479, + "eventLoopMaxMs": 51.937279, + "parentPeakRssKiB": 130780 + }, + { + "phase": "waveform-replay", + "succeeded": true, + "elapsedMs": 2756681.940805, + "eventLoopP95Ms": 27.279359, + "eventLoopMaxMs": 154.009599, + "parentPeakRssKiB": 153608 + }, + { + "phase": "summary", + "succeeded": true, + "elapsedMs": 2994.1249830001034, + "eventLoopP95Ms": 22.020095, + "eventLoopMaxMs": 34.865151, + "parentPeakRssKiB": 153752 + }, + { + "summaryBytes": 59919, + "resultHash": "bda8d03d158b04bb3669ff613e0936ed46822c0083c4d37f9f30e5338bc95f97", + "compoundBuildHash": "2b6514629ca87fc82753d6b0940b1c94d754f834585ef115387eaf536ef58df1", + "pythonBuildHash": "212c5643dd24cae1cbf9c346b0729d7ea144c453cd56370ec810223c508a5bc3" + } + ], + "concurrentReadMs": { + "count": 363, + "p95": 1943.6675560000003, + "max": 2183.3170790000004 + }, + "internalApiMs": { + "count": 7390, + "p95": 57.09343400015496, + "max": 300.81892600000015 + }, + "internalApiFailures": 0, + "externalApiMs": { + "count": 17568, + "p95": 39.62092199799372, + "max": 500.0740819959901 + }, + "externalApiFailures": 0, + "minimumAvailableMemoryBytes": 946417664, + "minimumFreeDiskBytes": 3243700224, + "maximumTemperatureC": 49.965, + "maximumIngestionAgeMs": 1560, + "negativeSampleAgeCount": 71, + "peakObservedBenchmarkTreeRssBytes": 464650240, + "maximumBenchmarkProcessCount": 4, + "artifactSha256": { + "benchmark.cjs": "bc1dc66c597b020480830548c2de7368a7bdbe2e68a440779c7de79b8254145f", + "payload.tgz": "9cf50a0a8488a1ef8f79ccc41bdd7d086c9dd68bcbba61156d11d8f768813a36", + "measurements.json": "696bdd6462504844c90936afcfe1c8cac7429d141f16893270292cdcc59c29bc", + "probe-28800.json": "e9865a8019e8e3360b022daab587ff5397d317bb9e17c362231ebc3288747af3", + "fixture-28800.json": "165071ec9b67e9dd2143575e8c2758b99090d47c1fb6412291b778d4be59dce0" + }, + "workerSha256": "c635a135ad0cfc2a327292968d08ab043ca8b523dd4c0149be6b793c8a192a5e", + "coordinatorSha256": "c31f3e6dc614cc688d06d25158318cea9c48f3f9ca3d157b576f1269e7672c05", + "maximumLoadPerCpu": 1.0575, + "meanAggregateCpuBusyPercent": 68.06810450124667, + "maximumAggregateCpuBusyPercent": 99.00497512437812 +} diff --git a/docs/performance/biometrics-replay-2026-10-13/live-shadow-operational.json b/docs/performance/biometrics-replay-2026-10-13/live-shadow-operational.json new file mode 100644 index 000000000..91e9c6862 --- /dev/null +++ b/docs/performance/biometrics-replay-2026-10-13/live-shadow-operational.json @@ -0,0 +1,81 @@ +{ + "version": "live-shadow-operational-measurement-v1", + "date": "2026-10-13", + "runtimeFix": "shadow RSS disappearance during clean exit", + "scope": "loopback live input and read-only active observations with isolated sink; not production registry admission or accuracy qualification", + "windowSeconds": 180, + "state": "completed", + "reason": "observation_window_complete", + "isolatedSink": { + "pointCount": 6, + "byteCount": 5396, + "canonicalRows": 0 + }, + "build": { + "runnerHash": "6a2966046edffea389604b28b89a7d9b4417dd3dcda41e64f7adab2103b11d03", + "pythonBuildHash": "b16390a3796398903b80db34a1e758d6e8ee3fb30dcfc4034e4b30bbdb13d0e9", + "manifestHash": "96c2f01a329b4346b4a3f2801ba973650d7235a5c4bd00bd0ded96f1a5768b9b" + }, + "worker": { + "inputBytes": 1013420, + "outputBytes": 118657, + "peakObservedRssBytes": 113881088, + "rssSamples": 1192, + "maximumObservationGapMs": 497.2393699999993, + "budgets": { + "cpuSeconds": 120, + "addressSpaceBytes": 805306368, + "maximumRssBytes": 402653184, + "wallMs": 270000, + "acknowledgmentMs": 30000, + "idleMs": 10000, + "rssIntervalMs": 100, + "rssTimeoutMs": 1000 + }, + "memoryScope": "sum_rss_of_shadow_process_group", + "exactKernelPeak": false, + "cpuMs": 4953.220225 + }, + "limitations": [ + "cold_start_without_serialized_state", + "less_than_five_minutes_preroll", + "missing_occupancy_context", + "missing_pump_context" + ], + "probe": { + "samples": 529, + "httpRequests": 1587, + "httpFailures": 0, + "httpP95Ms": 40.213834996393416, + "httpMaxMs": 123.67384999379283, + "peakProcessTreeRssBytes": 295608320, + "minimumAvailableMemoryBytes": 1135259648, + "maximumTemperatureC": 44.828, + "meanHostCpuPercent": 56.69213951033886, + "maximumHostCpuPercent": 96.59574468085107 + }, + "cleanup": { + "launchExitCode": 0, + "ownedProcessesAbsentAfterExit": true, + "remoteStageRemoved": true + }, + "qualificationInstalled": false, + "profileBindingsChanged": false, + "priorAttempts": [ + { + "state": "failed", + "reason": "Measurement health admission refused", + "cleanupConfirmed": true + }, + { + "state": "failed", + "reason": "Shadow RSS observation unavailable", + "cleanupConfirmed": true + } + ], + "runtimeFixCommit": "4c3a6a6e", + "rawArtifactSha256": { + "biometrics-live-shadow-third-summary.json": "7e5dca3d1c4a6b3e0360d256a203054ac694b42481eee3ea3b7a333dedc858a9", + "biometrics-live-shadow-third-probe.json": "1b86d9a5c1f2fa1f045b19cd310f7dafb09673f1017c7b43e376884888f01d10" + } +} diff --git a/docs/performance/biometrics-replay-2026-10-13/shadow-4ee74c25.json b/docs/performance/biometrics-replay-2026-10-13/shadow-4ee74c25.json new file mode 100644 index 000000000..f1c02fe11 --- /dev/null +++ b/docs/performance/biometrics-replay-2026-10-13/shadow-4ee74c25.json @@ -0,0 +1,117 @@ +{ + "version": "synthetic-shadow-supervisor-measurement-v1", + "scope": "accelerated synthetic frames through shipped supervisor and kernel-limited child; no live capture, database sink, held-out accuracy or installation qualification", + "runnerHash": "aaf032040970edf5758d0dbb358afe7044ce0274ee82492b5fd6a62925ef930d", + "pythonBuildHash": "b16390a3796398903b80db34a1e758d6e8ee3fb30dcfc4034e4b30bbdb13d0e9", + "node": { + "version": "v24.16.0", + "platform": "linux", + "arch": "arm64" + }, + "manifestHash": "96c2f01a329b4346b4a3f2801ba973650d7235a5c4bd00bd0ded96f1a5768b9b", + "configHash": "933fb4329b8c24091565b43388576d641f2806e0fda52bad44f2a2968b997aba", + "seconds": 1800, + "sampleRateHz": 500, + "channels": 2, + "wallMs": 61340.47504500001, + "outputCount": 30, + "outputHash": "e75ead5601fc3c5da660c9778dea0f6a2c999900a4d345754a0ed07f0b4970cf", + "receipt": { + "health": { + "state": "completed", + "reason": "window_complete", + "records": 1800, + "outputs": 30, + "cpu_ms": 29170.628042, + "input_hash": "a523ceeec2382b220c9f95874c22d74922c21db93dcb313d10824b60e8c27ad2", + "counts": { + "records": 1800, + "accepted_frames": 1800, + "empty_channel_frames": 0, + "pump_gated_frames": 0, + "preroll_outputs": 0, + "scored_outputs": 30 + }, + "limitations": [ + "cold_start_without_serialized_state", + "less_than_five_minutes_preroll" + ] + }, + "inputBytes": 9997366, + "outputBytes": 998314, + "peakObservedRssBytes": 118648832, + "rssSamples": 347, + "maximumObservationGapMs": 333.41997099999935, + "budgets": { + "cpuSeconds": 1200, + "addressSpaceBytes": 805306368, + "maximumRssBytes": 402653184, + "wallMs": 1800000, + "acknowledgmentMs": 30000, + "idleMs": 30000, + "rssIntervalMs": 100, + "rssTimeoutMs": 1000 + }, + "memoryScope": "sum_rss_of_shadow_process_group", + "exactKernelPeak": false + }, + "partial": { + "kind": "partial_supervisor_observation", + "health": { + "state": "completed", + "reason": "window_complete", + "records": 1800, + "outputs": 30, + "cpu_ms": 29170.628042, + "input_hash": "a523ceeec2382b220c9f95874c22d74922c21db93dcb313d10824b60e8c27ad2", + "counts": { + "records": 1800, + "accepted_frames": 1800, + "empty_channel_frames": 0, + "pump_gated_frames": 0, + "preroll_outputs": 0, + "scored_outputs": 30 + }, + "limitations": [ + "cold_start_without_serialized_state", + "less_than_five_minutes_preroll" + ] + }, + "inputBytes": 9997366, + "outputBytes": 998314, + "diagnosticBytes": 0, + "peakObservedRssBytes": 118648832, + "rssSamples": 347, + "maximumObservationGapMs": 333.41997099999935, + "budgets": { + "cpuSeconds": 1200, + "addressSpaceBytes": 805306368, + "maximumRssBytes": 402653184, + "wallMs": 1800000, + "acknowledgmentMs": 30000, + "idleMs": 30000, + "rssIntervalMs": 100, + "rssTimeoutMs": 1000 + }, + "cleanupConfirmed": true, + "memoryScope": "sum_rss_of_shadow_process_group", + "exactKernelPeak": false + }, + "failure": null, + "parentEventLoopP95Ms": 26.099711, + "parentEventLoopMaxMs": 68.616191, + "parentPeakRssKiB": 150836, + "probe": { + "requests": 456, + "failures": 0, + "p95Ms": 45.41382499883184, + "maximumMs": 130.38527100434294, + "maximumTemperatureC": 47.513, + "minimumAvailableMemoryBytes": 1069072384, + "peakBenchmarkTreeRssBytes": 268959744 + }, + "artifactSha256": { + "shadow-measurements.json": "e57bf3bd10374b31059f60e39f8645d2be158ab88d2eb9a1abf1baa9b9a786bd", + "shadow-probe.json": "13fd0f3723b6ebc5650cc52868d8c02f5f4bee6cb0629303b53998f09ee883cd" + } +} diff --git a/docs/performance/biometrics-replay-2026-10-13/smoke-4453c0a5.json b/docs/performance/biometrics-replay-2026-10-13/smoke-4453c0a5.json new file mode 100644 index 000000000..c5c3d9078 --- /dev/null +++ b/docs/performance/biometrics-replay-2026-10-13/smoke-4453c0a5.json @@ -0,0 +1,88 @@ +{ + "scope": "30-minute synthetic worker resource smoke; not full-night, shadow, clinical or activation qualification", + "sourceCommit": "4453c0a5", + "architecture": "aarch64", + "nodeVersion": "24.16.0", + "pythonVersion": "3.10.4", + "completed": true, + "fixtureSeconds": 1800, + "compressedBytes": 5976451, + "canonicalObservationsUnchanged": true, + "readFailures": 0, + "replayBudget": { + "jobWallMs": 900000, + "waveformWallMs": 240000, + "waveformCpuSeconds": 180 + }, + "operations": [ + { + "phase": "snapshot", + "succeeded": true, + "elapsedMs": 1560.2145219999984, + "eventLoopP95Ms": 21.987327, + "eventLoopMaxMs": 29.081599, + "parentPeakRssKiB": 129868 + }, + { + "phase": "retain", + "succeeded": true, + "elapsedMs": 16172.227184, + "eventLoopP95Ms": 22.233087, + "eventLoopMaxMs": 42.532863, + "parentPeakRssKiB": 129868 + }, + { + "phase": "waveform-replay", + "succeeded": true, + "elapsedMs": 202759.751582, + "eventLoopP95Ms": 28.049407, + "eventLoopMaxMs": 186.384383, + "parentPeakRssKiB": 129868 + }, + { + "phase": "summary", + "succeeded": true, + "elapsedMs": 1541.8592840000056, + "eventLoopP95Ms": 21.790719, + "eventLoopMaxMs": 24.936447, + "parentPeakRssKiB": 129868 + }, + { + "summaryBytes": 58915, + "resultHash": "ed2505dfb5cc91be94ebeea9f26bddc5ee214c95e9ca730eebb459dbff9c989d", + "compoundBuildHash": "3f9ee531c01c501f543278dfb5641eb57eea9cf2906d7ce2f94f01d6ab1a2e34", + "pythonBuildHash": "0eb411804b1b4ba608a1a8db11406dcf983f0ff9bea0f2d8c5eb027b6a187254" + } + ], + "concurrentReadMs": { + "count": 27, + "p95": 1859.888511000012, + "max": 1860.4550230000168 + }, + "internalApiMs": { + "count": 638, + "p95": 65.67037400000845, + "max": 305.6198179999999 + }, + "internalApiFailures": 0, + "externalApiMs": { + "count": 1545, + "p95": 41.55511000135448, + "max": 129.56501200096682 + }, + "externalApiFailures": 0, + "minimumAvailableMemoryBytes": 992690176, + "minimumFreeDiskBytes": 3450945536, + "maximumTemperatureC": 46.696, + "maximumIngestionAgeMs": 992, + "negativeSampleAgeCount": 9, + "peakObservedBenchmarkTreeRssBytes": 433369088, + "maximumBenchmarkProcessCount": 4, + "artifactSha256": { + "benchmark.cjs": "bcfd8e5024a606aa1ede8f33481a884d553cc4344eeeb4fcd7c610a943e69984", + "payload.tgz": "82ace2b631e042d8c0993c21d5923580df614357bd00fc598692588b6e485543", + "measurements.json": "d0eb400d98dd78c12921b2bc2f42c1471c3d6a4e529c4b01be863a3f64c95f7e", + "probe.json": "134ded3108c2f678cfa8aafec1bee782ab502820b632214c2ae280c9e581f330", + "fixture-1800.json": "861e719fb0c8bad1ddf89c625e7a19388e32dc35984ff90504c00b3c3e06e4a6" + } +} diff --git a/drizzle.package-experiments.config.ts b/drizzle.package-experiments.config.ts new file mode 100644 index 000000000..6a1af3e37 --- /dev/null +++ b/drizzle.package-experiments.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from 'drizzle-kit' + +export default defineConfig({ + schema: './src/db/package-experiment-schema.ts', + out: './src/db/package-experiment-migrations', + dialect: 'sqlite', +}) diff --git a/eslint.config.ts b/eslint.config.ts index e9c4580a1..5c7857037 100644 --- a/eslint.config.ts +++ b/eslint.config.ts @@ -25,6 +25,13 @@ export default defineConfig([ tseslint.configs.strict, pluginReact.configs.flat.recommended, ...nextVitals, + // Next selects its Babel parser for JS and for .mts/.cts config files. + // The strict TypeScript rules above need TypeScript scope references; + // otherwise used runtime variables are reported as type-only usages. + { + files: ['**/*.js', '**/*.mjs', '**/*.cjs', '**/*.mts', '**/*.cts'], + languageOptions: { parser: tseslint.parser }, + }, globalIgnores([ '.claude/**', diff --git a/instrumentation.ts b/instrumentation.ts index 0d23c6258..20e323ef6 100644 --- a/instrumentation.ts +++ b/instrumentation.ts @@ -1,484 +1,20 @@ -import { shutdownBaseController } from '@/src/hardware/base/instance' -/** - * Server startup and process lifecycle management. - * - * Handles: - * - Job scheduler initialization with exponential-backoff retry - * - DAC hardware monitor initialization (non-blocking) - * - Hardware daemon pre-flight validation - * - Centralized SIGTERM/SIGINT signal handling - * - Global unhandled rejection/exception handlers - * - Graceful shutdown sequencing with 10 s force-exit watchdog - * - * Entry points: - * - `register()` — Next.js instrumentation hook, called automatically on server start. - * Runs only in the Node.js runtime; skipped on the Edge runtime. - * - `initializeScheduler()` — idempotent, safe to call from app code as a fallback. - */ - -import { getJobManager, shutdownJobManager } from '@/src/scheduler' -import { getAutomationEngine, shutdownAutomationEngine } from '@/src/automation' -import { startTemperatureController, stopTemperatureController } from '@/src/temperature/instance' -import { closeDatabase, closeBiometricsDatabase } from '@/src/db' -import { startBiometricsRetention, stopBiometricsRetention } from '@/src/db/retention' -import { startAutomationRunsRetention, stopAutomationRunsRetention } from '@/src/db/automationRunsRetention' -import { startHealthSampler, stopHealthSampler } from '@/src/lib/healthSampler' -import { getDacMonitor, shutdownDacMonitor } from '@/src/hardware/dacMonitor.instance' -import { startPiezoStreamServer, shutdownPiezoStreamServer } from '@/src/streaming/piezoStream' -import { startBonjourAnnouncement, stopBonjourAnnouncement } from '@/src/streaming/bonjourAnnounce' -import { startMqttBridge, shutdownMqttBridge } from '@/src/streaming/mqttBridge' -import { initializeKeepalives, shutdownKeepalives } from '@/src/services/temperatureKeepalive' -import { startAutoOffWatcher, stopAutoOffWatcher } from '@/src/services/autoOffWatcher' -import { startDatabaseIntegrityChecks, stopDatabaseIntegrityChecks } from '@/src/db/integrity' -import { startPerformanceMonitoring, stopPerformanceMonitoring, recordStartupPhase } from '@/src/lib/serverPerformance' -import { shutdownHomeKit, startHomeKitIfEnabled } from '@/src/homekit' - -let isInitialized = false -let isShuttingDown = false -let handlersRegistered = false -let initializationPromise: Promise | null = null -let hardwareReady = false -let hardwarePromise: Promise | null = null - -/** - * Centralized graceful shutdown coordinator. - * Sequences: wait for in-flight jobs → shutdown scheduler → close database → exit - */ -async function gracefulShutdown(signal: string): Promise { - if (isShuttingDown) return - isShuttingDown = true - - console.log(`Received ${signal}, starting graceful shutdown...`) - - // Force exit after 10s if graceful shutdown hangs - const forceExitTimer = setTimeout(() => { - console.error('Graceful shutdown timed out after 10s, forcing exit') - process.exit(1) - }, 10_000) - forceExitTimer.unref() - - stopPerformanceMonitoring() - await stopDatabaseIntegrityChecks() - - // Step 0: Stop keepalive timers - try { - await stopTemperatureController() - shutdownBaseController() - shutdownKeepalives() - } - catch (error) { - console.error('Error shutting down keepalives:', error) - } - - // Step 1: Shutdown scheduler (waits for in-flight jobs internally) - try { - await shutdownJobManager() - } - catch (error) { - console.error('Error shutting down scheduler:', error) - } - - // Step 1a: Stop the Autopilot rules engine tick - try { - await shutdownAutomationEngine() - } - catch (error) { - console.error('Error shutting down automation engine:', error) - } - - // Step 2: Shutdown piezo stream server - try { - await shutdownPiezoStreamServer() - } - catch (error) { - console.error('Error shutting down piezo stream server:', error) - } - - // Step 2b: Shutdown MQTT bridge (publish offline + close client cleanly) - try { - await shutdownMqttBridge() - } - catch (error) { - console.error('Error shutting down MQTT bridge:', error) - } - - // Step 3: Stop Bonjour announcement - try { - stopBonjourAnnouncement() - } - catch (error) { - console.error('Error stopping Bonjour:', error) - } - - // Step 3a: Stop HomeKit bridge (unpublish HAP mDNS, close TCP listener) - try { - await shutdownHomeKit() - } - catch (error) { - console.error('Error shutting down HomeKit:', error) - } - - // Step 4: Stop auto-off watcher (await in-flight power-off calls) - try { - await stopAutoOffWatcher() - } - catch (error) { - console.error('Error stopping auto-off watcher:', error) - } - - // Step 5: Shutdown DAC monitor - try { - await shutdownDacMonitor() - } - catch (error) { - console.error('Error shutting down DacMonitor:', error) - } - - // Step 6: Stop biometrics retention and health sampling before closing DB - try { - stopBiometricsRetention() - stopAutomationRunsRetention() - stopHealthSampler() - } - catch (error) { - console.error('Error stopping biometrics retention:', error) - } - - // Step 7: Close database connections - try { - closeDatabase() - closeBiometricsDatabase() - } - catch (error) { - console.error('Error closing database:', error) - } - - process.exit(0) -} - -/** - * Register global process handlers (signal handlers, error handlers). - * Safe to call multiple times - only registers once. - */ -function registerGlobalHandlers(): void { - if (handlersRegistered) return - handlersRegistered = true - - // Centralized signal handlers - process.on('SIGTERM', () => gracefulShutdown('SIGTERM')) - process.on('SIGINT', () => gracefulShutdown('SIGINT')) - - // Global unhandled rejection handler - log but don't crash - process.on('unhandledRejection', (reason: unknown) => { - console.error('Unhandled promise rejection:', reason) - // Don't exit - let the process continue serving other requests - }) - - // Global uncaught exception handler - log and attempt graceful shutdown - process.on('uncaughtException', (error: Error & { code?: string, address?: string, port?: number }) => { - // mDNS EPERM is non-fatal — iptables blocks multicast, just log and continue - if (error.code === 'EPERM' && error.address === '224.0.0.251' && error.port === 5353) { - console.warn('[bonjour] mDNS send blocked by iptables (non-fatal):', error.message) - return - } - console.error('Uncaught exception:', error) - // Process state may be corrupted, attempt graceful shutdown - gracefulShutdown('uncaughtException') - }) -} - -/** - * Retry a function with exponential backoff. - */ -async function withRetry( - fn: () => Promise, - label: string, - maxAttempts: number = 3, - baseDelayMs: number = 500 -): Promise { - let lastError: unknown - for (let attempt = 1; attempt <= maxAttempts; attempt++) { - if (isShuttingDown) throw new Error('Startup cancelled during shutdown') - try { - return await fn() - } - catch (error) { - lastError = error - if (attempt < maxAttempts) { - const delay = baseDelayMs * Math.pow(2, attempt - 1) - console.warn( - `${label} failed (attempt ${attempt}/${maxAttempts}), retrying in ${delay}ms:`, - error instanceof Error ? error.message : error - ) - await new Promise(resolve => setTimeout(resolve, delay)) - } - } - } - throw lastError -} - -/** - * Validate hardware daemon connectivity on startup. - * Logs a warning if unavailable but does not crash. - */ -// Hardware validation removed — the DacMonitor handles connection lifecycle. -// The DAC socket server starts first, then the monitor waits for frankenfirmware. - -/** - * Initialize the DAC hardware monitor. - * Non-blocking — logs a warning on failure but does not crash. - */ -const initializeDacMonitor = async (): Promise => { - try { - await getDacMonitor() - if (isShuttingDown) { - await shutdownDacMonitor() - return - } - // A restart mid-alarm or mid-snooze (deploy, crash) resumes it. - const { restoreActiveAlarms } = await import('@/src/hardware/alarmState') - const { restoreSnoozes } = await import('@/src/hardware/snoozeManager') - await restoreActiveAlarms() - restoreSnoozes() - } - catch (error) { - console.warn( - 'WARNING: DacMonitor failed to start:', - error instanceof Error ? error.message : error - ) - } -} - -async function initializeHardware(): Promise { - if (hardwareReady || isShuttingDown) return - if (hardwarePromise) return hardwarePromise - hardwarePromise = prepareHardware().finally(() => { - hardwarePromise = null - }) - return hardwarePromise +/** Runtime-neutral entrypoint. Keep literal runtime checks around dynamic + * imports so Next can exclude hardware/database code from its Edge bundle. */ +export function shouldRunInstrumentation(env: Record = process.env): boolean { + return !env.NEXT_RUNTIME || env.NEXT_RUNTIME === 'nodejs' } -async function prepareHardware(): Promise { - // Rehydrate the pump stall guard from persisted un-acknowledged alerts - // before anything that consults shouldBlock (keepalives, scheduler jobs) - // can re-energize a side whose fault predates this boot. - try { - const { rehydrate } = await import('@/src/hardware/pumpStallGuard') - rehydrate() - } - catch (error) { - console.warn( - '[pumpStallGuard] rehydration failed:', - error instanceof Error ? error.message : error - ) - } - - if (isShuttingDown) return - - // Start DAC socket server FIRST — this is the single listener on dac.sock. - // frankenfirmware will connect to it. Everything else (DacMonitor, device - // router, health checks) uses this server's connection. - try { - const { startDacServer } = await import('@/src/hardware/dacMonitor.instance') - await startDacServer() - } - catch (error) { - console.warn('[DAC] Socket server failed to start:', error instanceof Error ? error.message : error) - } - - if (isShuttingDown) return - - // Start DAC monitor (non-blocking — waits for frankenfirmware to connect) - initializeDacMonitor() - - if (isShuttingDown) return - hardwareReady = true - try { - startPiezoStreamServer() - } - catch (error) { - console.warn('WARNING: Piezo stream server failed to start:', error instanceof Error ? error.message : error) +export async function register(): Promise { + if (!process.env.NEXT_RUNTIME || process.env.NEXT_RUNTIME === 'nodejs') { + const { registerNodeInstrumentation } = await import('./src/server/instrumentation-node') + await registerNodeInstrumentation() } - recordStartupPhase('hardware-services-started') } -/** - * Initialize the job scheduler - * Safe to call multiple times - will only initialize once - */ +/** Compatibility entrypoint for callers that initialize the scheduler directly. */ export async function initializeScheduler(): Promise { - if (isInitialized || isShuttingDown) return - if (initializationPromise) return initializationPromise - initializationPromise = initializeBackgroundServices().finally(() => { - initializationPromise = null - }) - return initializationPromise -} - -async function initializeBackgroundServices(): Promise { - try { - await initializeHardware() - if (isShuttingDown) return - - // Record System → Health's data-path history once a minute (non-blocking). - // Started before the scheduler so a scheduler that fails to load still - // leaves a history of the failure. - startHealthSampler() - - const schedulerStartedAt = performance.now() - console.log('Initializing job scheduler...') - const jobManager = await withRetry( - () => getJobManager(), - 'Job manager initialization' - ) - if (isShuttingDown) return - recordStartupPhase('scheduler-ready', schedulerStartedAt) - const scheduler = jobManager.getScheduler() - const jobs = scheduler.getJobs() - - console.log(`Job scheduler initialized with ${jobs.length} scheduled jobs`) - - // Log next scheduled jobs for visibility - const upcomingJobs = jobs - .map((job) => { - const nextRun = scheduler.getNextInvocation(job.id) - return { - id: job.id, - type: job.type, - nextRun: nextRun ? nextRun.toISOString() : 'N/A', - } - }) - .filter(job => job.nextRun !== 'N/A') - .sort((a, b) => { - if (a.nextRun === 'N/A' || b.nextRun === 'N/A') return 0 - return new Date(a.nextRun).getTime() - new Date(b.nextRun).getTime() - }) - .slice(0, 5) - - if (upcomingJobs.length > 0) { - console.log('Next scheduled jobs:') - for (const job of upcomingJobs) { - console.log(` - ${job.id}: ${job.nextRun}`) - } - } - - isInitialized = true - - // Optional integrations do not hold up HTTP or the remaining services. - void startMqttBridge().then(() => { - if (isShuttingDown) return shutdownMqttBridge() - recordStartupPhase('mqtt-started') - }).catch((error) => { - console.warn('WARNING: MQTT bridge failed to start:', error instanceof Error ? error.message : error) - }) - - // Start auto-off watcher (polls biometrics DB for bed-exit events) - startAutoOffWatcher() - - // Start Bonjour/mDNS announcement (non-blocking) - startBonjourAnnouncement() - - // Start HomeKit bridge if user has opted in (non-blocking) - startHomeKitIfEnabled().then(() => { - if (isShuttingDown) return shutdownHomeKit() - }).catch((error) => { - console.warn( - '[homekit] startup failed:', - error instanceof Error ? error.message : error, - ) - }) - - // Start biometrics time-series retention loop (non-blocking) - startBiometricsRetention() - startAutomationRunsRetention() - - // Boot the Autopilot rules engine beside the scheduler (non-blocking). - // Shares the same hardware path; no-op until automations are created. - getAutomationEngine().catch((error) => { - console.warn('[automation] engine failed to start:', error instanceof Error ? error.message : error) - }).then(async () => { - if (isShuttingDown) return shutdownAutomationEngine() - await startTemperatureController() - if (isShuttingDown) { - await stopTemperatureController() - return - } - initializeKeepalives() - }).catch((error) => { - console.warn( - '[temperature] controller failed to start:', - error instanceof Error ? error.message : error, - ) - }) - } - catch (error) { - console.error('Failed to initialize job scheduler:', error) - // Don't crash the app if scheduler fails to initialize - } -} - -/** - * Next.js instrumentation hook (if supported) - * Automatically called by Next.js on server startup - */ -/** - * Gate for `register()`. Returns true only when the instrumentation hook - * should execute its body. - * - * Next.js calls the instrumentation hook **once per server runtime** — both - * Node and Edge bundles trigger it. The previous gate (`NEXT_RUNTIME==='nodejs' - * || typeof window === 'undefined'`) was too permissive: `typeof window` is - * undefined in every server runtime, so Edge slipped through and caused - * double-init (most visibly: two MQTT bridge clients connecting from the - * same pod). Strict check on `NEXT_RUNTIME` when set; allow when unset - * (tests, scripts). - */ -export function shouldRunInstrumentation(env: Record = process.env): boolean { - if (env.NEXT_RUNTIME && env.NEXT_RUNTIME !== 'nodejs') return false - return true -} - -export async function register(): Promise { - if (shouldRunInstrumentation()) { - // Register global handlers first (before any initialization that could fail) - registerGlobalHandlers() - startPerformanceMonitoring() - const migrationsStartedAt = performance.now() - - // Run pending database migrations before starting the app - const { runMigrations, seedDefaultData } = await import('@/src/db/migrate') - await runMigrations() - await seedDefaultData() - recordStartupPhase('migrations-ready', migrationsStartedAt) - - // Skip hardware initialization in CI — no dac.sock, no sensors, no scheduler needed. - // The server still starts and serves API routes (including /api/openapi.json). - if (process.env.CI) { - console.log('CI environment detected — skipping hardware initialization') - return - } - - // Validate and auto-repair iptables rules (mDNS, LAN access, NTP) - try { - const { checkAndRepairIptables } = await import('@/src/hardware/iptablesCheck') - const result = checkAndRepairIptables() - if (result.repaired.length > 0) { - console.warn(`[startup] Repaired ${result.repaired.length} missing iptables rules:`, result.repaired.join(', ')) - } - else if (result.ok) { - console.log('[startup] iptables rules verified') - } - } - catch (e) { - console.warn('[startup] iptables check skipped:', e instanceof Error ? e.message : e) - } - - await initializeHardware() - if (isShuttingDown) return - startDatabaseIntegrityChecks() - // The clock gate and scheduler loading still finish before scheduled writers - // start, but no longer prevent clients from loading settings or diagnostics. - void initializeScheduler() - recordStartupPhase('http-initialization-ready') + if (!process.env.NEXT_RUNTIME || process.env.NEXT_RUNTIME === 'nodejs') { + const node = await import('./src/server/instrumentation-node') + await node.initializeScheduler() } } diff --git a/modules/common/biometrics/__init__.py b/modules/common/biometrics/__init__.py new file mode 100644 index 000000000..51da23d76 --- /dev/null +++ b/modules/common/biometrics/__init__.py @@ -0,0 +1 @@ +"""Versioned biometric configuration contracts shared with TypeScript.""" diff --git a/modules/common/biometrics/build_identity.py b/modules/common/biometrics/build_identity.py new file mode 100644 index 000000000..aad7c47a0 --- /dev/null +++ b/modules/common/biometrics/build_identity.py @@ -0,0 +1,64 @@ +"""Versioned identity of the shipped replay sources and installed runtime. + +Distribution RECORD hashes identify installed wheel metadata, not an attestation +that every installed binary is unmodified. No archive or patient data is read. +""" +import hashlib +import importlib.metadata +from pathlib import Path +import platform +import sys +import sysconfig + +from .config import config_hash + +SOURCE_FILES = ( + 'common/__init__.py', 'common/cbor_raw.py', + 'common/biometrics/__init__.py', 'common/biometrics/build_identity.py', + 'common/biometrics/config.py', 'common/biometrics/manifest.v1.json', + 'common/biometrics/frames.py', 'common/biometrics/gates.py', + 'common/biometrics/processing.py', 'common/biometrics/routing.py', + 'common/biometrics/signal.py', 'common/biometrics/raw_artifacts.py', + 'common/biometrics/raw_records.py', 'common/biometrics/retained_replay.py', + 'common/biometrics/execution_lock.py', + 'common/biometrics/replay.py', 'common/biometrics/replay_worker.py', + 'piezo-processor/pyproject.toml', 'piezo-processor/uv.lock', +) +MAX_FILE_BYTES = 16 * 1024 * 1024 + + +def _digest_file(path): + digest = hashlib.sha256() + size = 0 + with path.open('rb') as stream: + while True: + chunk = stream.read(65536) + if not chunk: + break + size += len(chunk) + if size > MAX_FILE_BYTES: + raise ValueError('Replay build identity file exceeds its bound') + digest.update(chunk) + return digest.hexdigest() + + +def replay_build_identity(): + root = Path(__file__).resolve().parents[2] + sources = {name: _digest_file(root / name) for name in SOURCE_FILES} + dependencies = {} + for name in ('numpy', 'scipy'): + distribution = importlib.metadata.distribution(name) + record = distribution.read_text('RECORD') + wheel = distribution.read_text('WHEEL') + if not record or not wheel or len(record.encode()) > MAX_FILE_BYTES or len(wheel.encode()) > MAX_FILE_BYTES: + raise ValueError('Replay dependency build metadata is unavailable') + dependencies[name] = {'version': distribution.version, + 'record_sha256': hashlib.sha256(record.encode()).hexdigest(), + 'wheel_sha256': hashlib.sha256(wheel.encode()).hexdigest()} + identity = {'version': 'waveform-python-build-v1', 'sources': sources, + 'dependencies': dependencies, 'runtime': { + 'implementation': platform.python_implementation(), 'version': sys.version, + 'cache_tag': sys.implementation.cache_tag, 'soabi': sysconfig.get_config_var('SOABI'), + 'machine': platform.machine(), 'system': platform.system(), 'byteorder': sys.byteorder, + }} + return {'hash': config_hash(identity), 'identity': identity} diff --git a/modules/common/biometrics/collection_worker.py b/modules/common/biometrics/collection_worker.py new file mode 100644 index 000000000..06a89a676 --- /dev/null +++ b/modules/common/biometrics/collection_worker.py @@ -0,0 +1,55 @@ +"""Bounded cleanup of server-owned retained waveforms, never the archive.""" +import argparse +from contextlib import closing +import json +from pathlib import Path +import sqlite3 +import sys +import time + +from .raw_artifacts import RawArtifactStore +from .replay_worker import _unique_object, _invalid_constant, configure_process_limits + + +def run(input_stream, output_stream, database_path, retained_directory): + payload = input_stream.read(1025) + if len(payload) > 1024: + raise ValueError('RAW cleanup input exceeds byte budget') + work = json.loads(payload, object_pairs_hook=_unique_object, parse_constant=_invalid_constant) + if work != {'version': 'raw-collection-work-v1'}: + raise ValueError('Invalid RAW cleanup envelope') + directory = Path(retained_directory) + removed_released = removed_orphans = 0 + if directory.is_symlink(): + raise ValueError('Invalid retained directory') + if directory.exists(): + database = Path(database_path) + if database.is_symlink() or not database.is_file(): + raise ValueError('Invalid biometrics database') + with closing(sqlite3.connect(database.resolve().as_uri() + '?mode=rw', uri=True, timeout=1)) as connection: + connection.execute('PRAGMA foreign_keys = ON') + store = RawArtifactStore(directory) + now_ms = time.time_ns() // 1_000_000 + removed_released = len(store.collect_released(connection, now_ms=now_ms)) + removed_orphans = len(store.collect_orphans(connection, now_ms=now_ms)) + result = {'version': 'raw-collection-result-v1', 'released': removed_released, 'orphans': removed_orphans} + output_stream.write((json.dumps(result, separators=(',', ':')) + '\n').encode()) + output_stream.flush() + + +def main(): + parser = argparse.ArgumentParser(description='Collect retained RAW files') + parser.add_argument('--database-path', required=True) + parser.add_argument('--retained-directory', required=True) + args = parser.parse_args() + try: + configure_process_limits() + run(sys.stdin.buffer, sys.stdout.buffer, args.database_path, args.retained_directory) + return 0 + except Exception: + sys.stderr.write('RAW collection worker failed\n') + return 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/modules/common/biometrics/config.py b/modules/common/biometrics/config.py new file mode 100644 index 000000000..c41d0c7f5 --- /dev/null +++ b/modules/common/biometrics/config.py @@ -0,0 +1,160 @@ +"""Read the shipped contract; never write measured sensor calibration. + +finite-binary64-json-v1 encodes exact decimal expansions of IEEE-754 values. +Unlike language-specific JSON float printers, this gives identical bytes on +both runtimes, including subnormals and negative zero. Object keys use UTF-16 +ordering (the JavaScript ordering); arrays retain their meaningful order. +""" +import copy +import hashlib +import json +import math +from pathlib import Path + +_MANIFEST = json.loads(Path(__file__).with_name("manifest.v1.json").read_text()) + + +def canonical_json(value): + nodes = 0 + + def visit(item, depth): + nonlocal nodes + nodes += 1 + if nodes > 20000 or depth > 32: + raise ValueError("JSON resource limit") + if item is None: + return "null" + if isinstance(item, bool): + return "true" if item else "false" + if isinstance(item, (int, float)): + number = float(item) + if not math.isfinite(number): + raise ValueError("Non-finite JSON number") + if number == 0: + return "0" + numerator, denominator = abs(number).as_integer_ratio() + places = denominator.bit_length() - 1 + digits = str(numerator * 5 ** places) + if places: + digits = digits.zfill(places + 1) + digits = (digits[:-places] + "." + digits[-places:]).rstrip("0").rstrip(".") + return ("-" if number < 0 else "") + digits + if isinstance(item, str): + try: + length = len(item.encode("utf-16-be")) // 2 + except UnicodeEncodeError as error: + raise ValueError("Invalid JSON string") from error + if length > 65536: + raise ValueError("Invalid JSON string") + return json.dumps(item, ensure_ascii=False, separators=(",", ":")) + if isinstance(item, list): + return "[" + ",".join(visit(child, depth + 1) for child in item) + "]" + if isinstance(item, dict): + if any(not isinstance(key, str) for key in item): + raise ValueError("Expected string keys") + try: + keys = sorted(item, key=lambda key: key.encode("utf-16-be")) + except UnicodeEncodeError as error: + raise ValueError("Invalid JSON string") from error + return "{" + ",".join(visit(key, depth + 1) + ":" + visit(item[key], depth + 1) + for key in keys) + "}" + raise ValueError("Expected finite JSON data") + + return visit(value, 0) + + +def config_hash(value): + return hashlib.sha256(canonical_json(value).encode("utf-8")).hexdigest() + + +def manifest(): + return copy.deepcopy(_MANIFEST) + + +def _validate(schema, value, path): + def fail(): + raise ValueError("Invalid biometric parameter: " + path) + if "const" in schema and value != schema["const"]: + fail() + if "enum" in schema and value not in schema["enum"]: + fail() + kind = schema.get("type") + if kind == "object": + if not isinstance(value, dict): + fail() + properties = schema.get("properties", {}) + if any(key not in value for key in schema.get("required", [])): + fail() + for key, child in value.items(): + if key not in properties: + fail() + _validate(properties[key], child, path + "." + key) + elif kind == "array": + if not isinstance(value, list) or len(value) > schema.get("maxItems", math.inf): + fail() + for child in value: + _validate(schema["items"], child, path) + elif kind in ("number", "integer"): + if isinstance(value, bool) or not isinstance(value, (float, int)): + fail() + if not math.isfinite(value): + fail() + if kind == "integer" and (value != int(value) or abs(value) > 9007199254740991): + fail() + if value < schema.get("minimum", -math.inf) or value > schema.get("maximum", math.inf): + fail() + elif kind == "string": + if not isinstance(value, str): + fail() + elif kind == "boolean": + if not isinstance(value, bool): + fail() + else: + raise ValueError("Unsupported contract schema: " + path) + + +def validate_config(value): + canonical_json(value) + _validate(_MANIFEST["parameters"], value, "config") + for path in _MANIFEST["orderedRanges"]: + node = value + for key in path: + node = node[key] + if node["min"] > node["max"]: + raise ValueError("Invalid ordered range: " + ".".join(path)) + return value + + +def resolve_config(overrides, base_hash=None): + expected = config_hash(_MANIFEST) + if base_hash is not None and base_hash != expected: + raise ValueError("PROFILE_INCOMPATIBLE: unknown base manifest") + if len(overrides) > 64: + raise ValueError("RESOURCE_LIMIT: maximum 64 overrides") + resolved = copy.deepcopy(_MANIFEST["defaults"]) + paths = set() + for override in overrides: + if set(override) != {"path", "value"}: + raise ValueError("Invalid override fields") + path = override["path"] + if path in paths: + raise ValueError("Duplicate override path") + paths.add(path) + parts = path.split(".") + schema = _MANIFEST["parameters"] + parent = resolved + for index, part in enumerate(parts): + if schema["type"] != "object" or part not in schema["properties"]: + raise ValueError("Unknown override path: " + path) + schema = schema["properties"][part] + if index == len(parts) - 1: + if schema["type"] == "object": + raise ValueError("Override leaves or entire ordered arrays") + _validate(schema, override["value"], path) + parent[part] = copy.deepcopy(override["value"]) + else: + parent = parent[part] + validate_config(resolved) + return {"resolved": resolved, "hash": config_hash(resolved), "manifestHash": expected, + "overrides": copy.deepcopy(overrides), + "provenance": {override["path"]: "override" for override in overrides}} diff --git a/modules/common/biometrics/execution_lock.py b/modules/common/biometrics/execution_lock.py new file mode 100644 index 000000000..45412bb50 --- /dev/null +++ b/modules/common/biometrics/execution_lock.py @@ -0,0 +1,29 @@ +"""Host-wide ownership for trusted numerical replay and shadow workers. + +Keep the legacy shadow path so workers from before an upgrade still contend. +No request or CLI option can select another lock. The optional function argument +is for trusted tests. This lock does not replace worker resource limits. +""" +import os +import stat + +EXECUTION_LOCK_PATH = '/run/sleepypod-shadow.lock' + + +def acquire_execution_lock(path=EXECUTION_LOCK_PATH): + """Nonblocking flock; never unlink or truncate the shared runtime inode.""" + import fcntl + fd = os.open(path, os.O_CREAT | os.O_RDWR | os.O_CLOEXEC | os.O_NOFOLLOW | os.O_NONBLOCK, 0o600) + try: + info = os.fstat(fd) + if (not stat.S_ISREG(info.st_mode) or info.st_nlink != 1 + or info.st_uid != os.geteuid() or info.st_mode & 0o077 or info.st_size != 0): + raise RuntimeError('Invalid biometric execution lock') + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + current = os.stat(path, follow_symlinks=False) + if (current.st_dev, current.st_ino) != (info.st_dev, info.st_ino): + raise RuntimeError('Biometric execution lock replaced') + return fd + except BaseException: + os.close(fd) + raise diff --git a/modules/common/biometrics/frames.py b/modules/common/biometrics/frames.py new file mode 100644 index 000000000..7d3dc4086 --- /dev/null +++ b/modules/common/biometrics/frames.py @@ -0,0 +1,33 @@ +"""Shared little-endian firmware sample decoding and lost-marker repair.""" +import numpy as np + + +LOST_SAMPLE = np.iinfo(np.int32).max + + +def decode_int32_samples(buf) -> np.ndarray: + """Decode an int32 sample buffer, tolerating truncated tails and + repairing lost-sample markers. + + A RAW record cut mid-write can carry a payload whose length is not a + multiple of 4; np.frombuffer would raise ValueError and kill the module. + Truncate to whole samples instead. A LOST_SAMPLE marker is replaced by + linear interpolation between its neighbours (the nearest valid sample at + a record edge); a record with no valid sample decodes as empty. + """ + if not isinstance(buf, (bytes, bytearray, memoryview)): + return np.empty(0, dtype=np.dtype(" None: + if record.get("type") != "frzHealth": + return + for side in ("left", "right"): + data = record.get(side) + if not isinstance(data, dict): + continue + rpm = 0.0 + for key in ("pumpRpm", "pump_rpm", "pumpRPM", "rpm"): + val = data.get(key) + if val is not None: + try: + rpm = float(val) + except (TypeError, ValueError): + rpm = 0.0 + break + # New NATS firmware nests the same reading at side.pump.rpm. + pump = data.get("pump") + if rpm == 0 and isinstance(pump, dict): + val = pump.get("rpm") + if val is not None: + try: + rpm = float(val) + except (TypeError, ValueError): + rpm = 0.0 + if rpm == 0: + duty = next((data.get(key) for key in + ("pumpDuty", "pump_duty", "duty") + if data.get(key) is not None), None) + if duty is None and isinstance(pump, dict): + duty = next((pump.get(key) for key in ("duty", "power") + if pump.get(key) is not None), None) + if duty is not None: + try: + rpm = PUMP_ACTIVE_RPM_MIN + 1.0 if float(duty) > 0 else 0.0 + except (TypeError, ValueError): + rpm = 0.0 + + now_active = rpm >= PUMP_ACTIVE_RPM_MIN + if self._was_active[side] and not now_active: + self._pump_off_at[side] = self._clock() + self._pump_rpm[side] = rpm + self._was_active[side] = now_active + + def is_side_pump_active(self, side: str) -> bool: + """Pump active = currently running OR within the trailing guard window.""" + if self._pump_rpm[side] >= PUMP_ACTIVE_RPM_MIN: + return True + return self._clock() - self._pump_off_at[side] < PUMP_OFF_GUARD_S + + def is_asymmetric_for(self, side: str) -> bool: + """Own pump active, opposite pump idle (and outside its own guard). + This is the configuration that produces phantom presence: pump + vibration on the powered side without competing signal on the + other side to balance/reject.""" + other = "right" if side == "left" else "left" + return self.is_side_pump_active(side) and not self.is_side_pump_active(other) + + def is_symmetric_active(self) -> bool: + """Both pumps active. RPM mismatch (Pod 5 live: 1940/2004) produces + a beat frequency in the cardiac band, generating bilateral phantom + vitals. PumpGate catches symmetric spikes but not steady-state.""" + return self.is_side_pump_active("left") and self.is_side_pump_active("right") + + +class PumpGate: + """Detects pump activity from dual-channel energy spikes. + + In streaming mode, each incoming record's L+R chunks are checked. + If a pump is detected, a guard period is set during which all + subsequent records are dropped (not ingested into side buffers). + """ + + def __init__(self, fs: float = 500, *, clock: Callable[[], float]): + self._clock = clock + self._fs = fs + self._baseline: Optional[float] = None + self._pump_until: float = 0.0 # monotonic seconds when guard expires + + def is_pump_active(self) -> bool: + return self._clock() < self._pump_until + + def check(self, left_chunk: np.ndarray, right_chunk: np.ndarray) -> bool: + """Check a single record's L+R chunks for pump activity. + + Returns True if the record should be DROPPED (pump detected or + guard period active), False if the record is clean. + """ + # If we're still in the guard period, drop + if self.is_pump_active(): + return True + + # Compute energy for both channels + le = float(np.mean(left_chunk.astype(np.float64) ** 2)) + re = float(np.mean(right_chunk.astype(np.float64) ** 2)) + + if le == 0 or re == 0: + return False + + ratio = min(le, re) / max(le, re) + + if self._baseline is not None and self._baseline > 0: + spike = max(le, re) / self._baseline + if ratio > PUMP_CORRELATION_MIN and spike > PUMP_ENERGY_MULTIPLIER: + # Pump detected — set guard period + self._pump_until = self._clock() + PUMP_GUARD_S + log.debug("Pump detected (spike=%.1f, ratio=%.2f), " + "guard until +%.0fs", spike, ratio, PUMP_GUARD_S) + return True + + # Update baseline with exponential moving average (only on clean data). + # Guard: if first record has unusually high energy (module started during + # pump), don't initialize baseline from it — wait for a quieter sample. + avg = (le + re) / 2 + if self._baseline is None: + self._baseline = avg + self._baseline_samples = 1 + elif avg < self._baseline * 3: + self._baseline = 0.95 * self._baseline + 0.05 * avg + self._baseline_samples = getattr(self, '_baseline_samples', 0) + 1 + # If early samples are settling, allow baseline to drop faster + if self._baseline_samples < 10: + self._baseline = min(self._baseline, avg) + + return False + +# --------------------------------------------------------------------------- +# Presence detection — hysteresis state machine (Paalasmaa et al. 2012) +# --------------------------------------------------------------------------- + +class PresenceDetector: + """Hysteresis-based presence detection with autocorrelation quality. + + Uses median std of 1-10 Hz filtered signal in 5-second windows as + the primary feature, with autocorrelation quality as secondary. + Requires 3 consecutive low windows to exit PRESENT state. + + The enter/exit gap protects against an empty bed that vibrates (pump, + prime). When the capacitance sensor already confirms a body is in bed + (`relaxed=True`), that protection costs an hour of missed vitals per + night: a sleeper who rolls into a weak-signal position exits after three + windows and then has to move hard to re-enter. Relaxed re-entry uses the + exit criteria, so presence resumes as soon as the signal stops being weak. + """ + ABSENT = 0 + PRESENT = 1 + + def __init__(self): + self.state = self.ABSENT + self.consecutive_low = 0 + self.enter_threshold = 400_000 + self.exit_threshold = 150_000 + self.exit_count = 3 # 3 consecutive low windows to declare absent + self.acr_threshold = 0.45 + + def _is_weak(self, window_std: float, acr_qual: float) -> bool: + return window_std < self.exit_threshold and acr_qual < self.acr_threshold * 0.5 + + def update(self, window_std: float, acr_qual: float, relaxed: bool = False) -> bool: + """Update state and return True if user is present. + + `relaxed`: the bed is known occupied by another sensor; re-enter on + anything that would not count as an exit window.""" + if self.state == self.ABSENT: + if relaxed: + entered = not self._is_weak(window_std, acr_qual) + else: + entered = window_std > self.enter_threshold or acr_qual > self.acr_threshold + if entered: + self.state = self.PRESENT + self.consecutive_low = 0 + return True + return False + else: + if self._is_weak(window_std, acr_qual): + self.consecutive_low += 1 + if self.consecutive_low >= self.exit_count: + self.state = self.ABSENT + self.consecutive_low = 0 + return False + return True # still present (hysteresis) + else: + self.consecutive_low = 0 + return True + diff --git a/modules/common/biometrics/inspection_worker.py b/modules/common/biometrics/inspection_worker.py new file mode 100644 index 000000000..07402fa0f --- /dev/null +++ b/modules/common/biometrics/inspection_worker.py @@ -0,0 +1,69 @@ +"""Inspect one cold archive per request without retaining or changing its bytes.""" +import argparse +from dataclasses import asdict +import json +import sys +import zlib + +from .raw_artifacts import SOURCE_NAME +from .raw_sources import ArchiveSource, archive_names +from .replay_worker import _unique_object, _invalid_constant, configure_process_limits + +MAX_INPUT_BYTES = 1024 +MAX_OUTPUT_BYTES = 65536 + + +def inspect_next(archive_directory, after_name): + # Names are a traversal cursor, not a snapshot of a mutable directory. A + # subsequent sweep from null discovers additions before the current cursor. + try: + names = archive_names(archive_directory) + except (OSError, ValueError): + return {'status': 'archive_unavailable', 'nextCursor': None} + name = next((name for name in names if after_name is None or name > after_name), None) + if name is None: + return {'status': 'complete', 'nextCursor': None} + try: + inspection = ArchiveSource(archive_directory, name).inspect() + inspection['artifact'] = asdict(inspection['artifact']) + except (OSError, ValueError, EOFError, zlib.error): + # Advance past bad sources so they cannot prevent a full sweep. Do not + # invalidate catalog rows by filename: another scan may have replaced it. + return {'status': 'source_unavailable', 'nextCursor': name} + return {'status': 'inspected', 'nextCursor': name, 'inspection': inspection} + + +def run(input_stream, output_stream, archive_directory): + payload = input_stream.read(MAX_INPUT_BYTES + 1) + if len(payload) > MAX_INPUT_BYTES: + raise ValueError('RAW inspection input exceeds byte budget') + work = json.loads(payload, object_pairs_hook=_unique_object, parse_constant=_invalid_constant) + if (not isinstance(work, dict) or set(work) != {'version', 'after_name'} + or work['version'] != 'raw-inspection-work-v1'): + raise ValueError('Invalid RAW inspection envelope') + after = work['after_name'] + if after is not None and (not isinstance(after, str) or len(after) > 255 or not SOURCE_NAME.fullmatch(after)): + raise ValueError('Invalid RAW inspection cursor') + result = inspect_next(archive_directory, after) + encoded = (json.dumps(result, allow_nan=False, separators=(',', ':')) + '\n').encode() + if len(encoded) > MAX_OUTPUT_BYTES: + raise ValueError('RAW inspection output exceeds byte budget') + output_stream.write(encoded) + output_stream.flush() + + +def main(): + parser = argparse.ArgumentParser(description='Inspect one server-owned RAW archive') + parser.add_argument('--archive-directory', required=True) + args = parser.parse_args() + try: + configure_process_limits() + run(sys.stdin.buffer, sys.stdout.buffer, args.archive_directory) + return 0 + except Exception: + sys.stderr.write('RAW inspection worker failed\n') + return 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/modules/common/biometrics/manifest.v1.json b/modules/common/biometrics/manifest.v1.json new file mode 100644 index 000000000..fdef253cd --- /dev/null +++ b/modules/common/biometrics/manifest.v1.json @@ -0,0 +1,646 @@ +{ + "id": "sleepypod-biometric-bundle", + "version": "1.0.0", + "schemaVersion": 1, + "canonicalization": "finite-binary64-json-v1", + "sampleRateHz": 500, + "hardware": [ + "pod3", + "pod4", + "pod5" + ], + "parameters": { + "type": "object", + "properties": { + "heart_rate": { + "type": "object", + "properties": { + "trackingToleranceBpm": { + "type": "number", + "minimum": 5, + "maximum": 30, + "default": 15, + "unit": "bpm", + "description": "Gaussian tracker consistency width; affects heart rate and downstream stages." + }, + "trackingHistoryLength": { + "type": "integer", + "minimum": 1, + "maximum": 10, + "default": 5, + "unit": "windows", + "description": "Number of accepted heart-rate estimates retained by the tracker." + } + }, + "required": [ + "trackingToleranceBpm", + "trackingHistoryLength" + ], + "additionalProperties": false + }, + "breathing_rate": { + "type": "object", + "properties": { + "intervalLowerRatio": { + "type": "number", + "minimum": 0.25, + "maximum": 0.9, + "default": 0.5, + "unit": "ratio", + "description": "Reject breathing intervals below this fraction of the median." + }, + "intervalUpperRatio": { + "type": "number", + "minimum": 1.1, + "maximum": 2, + "default": 1.5, + "unit": "ratio", + "description": "Reject breathing intervals above this fraction of the median." + } + }, + "required": [ + "intervalLowerRatio", + "intervalUpperRatio" + ], + "additionalProperties": false + }, + "hrv_index": { + "type": "object", + "properties": { + "harmonicTolerance": { + "type": "number", + "minimum": 0.05, + "maximum": 0.3, + "default": 0.18, + "unit": "ratio", + "description": "Tolerance for window IBI harmonic correction; does not produce beat-to-beat RMSSD." + }, + "minimumWindows": { + "type": "integer", + "minimum": 10, + "maximum": 30, + "default": 10, + "unit": "windows", + "description": "Minimum accepted 10-second IBI windows before reporting the index." + } + }, + "required": [ + "harmonicTolerance", + "minimumWindows" + ], + "additionalProperties": false + }, + "sleep_stages": { + "type": "object", + "properties": { + "engine": { + "type": "string", + "const": "rules" + }, + "ruleset": { + "type": "object", + "properties": { + "outliers": { + "type": "object", + "properties": { + "heartRate": { + "type": "object", + "properties": { + "min": { + "type": "number" + }, + "max": { + "type": "number" + } + }, + "required": [ + "min", + "max" + ], + "additionalProperties": false + }, + "hrv": { + "type": "object", + "properties": { + "min": { + "type": "number" + }, + "max": { + "type": "number" + } + }, + "required": [ + "min", + "max" + ], + "additionalProperties": false + }, + "breathingRate": { + "type": "object", + "properties": { + "min": { + "type": "number" + }, + "max": { + "type": "number" + } + }, + "required": [ + "min", + "max" + ], + "additionalProperties": false + }, + "heartRateSpikeStdDevs": { + "type": "number", + "minimum": 0, + "maximum": 10 + } + }, + "required": [ + "heartRate", + "hrv", + "breathingRate", + "heartRateSpikeStdDevs" + ], + "additionalProperties": false + }, + "lowSignalBelow": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "lowSignalRules": { + "maxItems": 32, + "type": "array", + "items": { + "type": "object", + "properties": { + "stage": { + "type": "string", + "enum": [ + "wake", + "light", + "deep", + "rem" + ] + }, + "when": { + "maxItems": 8, + "type": "array", + "items": { + "type": "object", + "properties": { + "metric": { + "type": "string", + "enum": [ + "hrRatio", + "heartRate", + "hrv", + "breathingRate", + "movement" + ] + }, + "op": { + "type": "string", + "enum": [ + "<", + "<=", + ">", + ">=" + ] + }, + "value": { + "type": "number" + } + }, + "required": [ + "metric", + "op", + "value" + ], + "additionalProperties": false + } + } + }, + "required": [ + "stage", + "when" + ], + "additionalProperties": false + } + }, + "rules": { + "maxItems": 32, + "type": "array", + "items": { + "type": "object", + "properties": { + "stage": { + "type": "string", + "enum": [ + "wake", + "light", + "deep", + "rem" + ] + }, + "when": { + "maxItems": 8, + "type": "array", + "items": { + "type": "object", + "properties": { + "metric": { + "type": "string", + "enum": [ + "hrRatio", + "heartRate", + "hrv", + "breathingRate", + "movement" + ] + }, + "op": { + "type": "string", + "enum": [ + "<", + "<=", + ">", + ">=" + ] + }, + "value": { + "type": "number" + } + }, + "required": [ + "metric", + "op", + "value" + ], + "additionalProperties": false + } + } + }, + "required": [ + "stage", + "when" + ], + "additionalProperties": false + } + }, + "fallback": { + "type": "string", + "enum": [ + "wake", + "light", + "deep", + "rem" + ] + }, + "smoothIsolatedEpochs": { + "type": "boolean" + }, + "transitions": { + "maxItems": 16, + "type": "array", + "items": { + "type": "object", + "properties": { + "from": { + "type": "string", + "enum": [ + "wake", + "light", + "deep", + "rem" + ] + }, + "to": { + "type": "string", + "enum": [ + "wake", + "light", + "deep", + "rem" + ] + }, + "via": { + "type": "string", + "enum": [ + "wake", + "light", + "deep", + "rem" + ] + } + }, + "required": [ + "from", + "to", + "via" + ], + "additionalProperties": false + } + } + }, + "required": [ + "outliers", + "lowSignalBelow", + "lowSignalRules", + "rules", + "fallback", + "smoothIsolatedEpochs", + "transitions" + ], + "additionalProperties": false + } + }, + "required": [ + "engine", + "ruleset" + ], + "additionalProperties": false + } + }, + "required": [ + "heart_rate", + "breathing_rate", + "hrv_index", + "sleep_stages" + ], + "additionalProperties": false + }, + "defaults": { + "heart_rate": { + "trackingToleranceBpm": 15, + "trackingHistoryLength": 5 + }, + "breathing_rate": { + "intervalLowerRatio": 0.5, + "intervalUpperRatio": 1.5 + }, + "hrv_index": { + "harmonicTolerance": 0.18, + "minimumWindows": 10 + }, + "sleep_stages": { + "engine": "rules", + "ruleset": { + "outliers": { + "heartRate": { + "min": 45, + "max": 130 + }, + "hrv": { + "min": 1, + "max": 300 + }, + "breathingRate": { + "min": 8, + "max": 25 + }, + "heartRateSpikeStdDevs": 2 + }, + "lowSignalBelow": 0.3, + "lowSignalRules": [ + { + "stage": "wake", + "when": [ + { + "metric": "movement", + "op": ">", + "value": 200 + } + ] + } + ], + "rules": [ + { + "stage": "wake", + "when": [ + { + "metric": "movement", + "op": ">", + "value": 200 + } + ] + }, + { + "stage": "deep", + "when": [ + { + "metric": "hrRatio", + "op": "<", + "value": 0.92 + } + ] + }, + { + "stage": "rem", + "when": [ + { + "metric": "hrRatio", + "op": ">=", + "value": 0.95 + }, + { + "metric": "hrv", + "op": "<", + "value": 25 + }, + { + "metric": "movement", + "op": "<", + "value": 30 + } + ] + }, + { + "stage": "rem", + "when": [ + { + "metric": "hrRatio", + "op": ">=", + "value": 0.95 + }, + { + "metric": "movement", + "op": "<", + "value": 50 + }, + { + "metric": "hrv", + "op": "<", + "value": 40 + } + ] + }, + { + "stage": "wake", + "when": [ + { + "metric": "hrRatio", + "op": ">=", + "value": 0.95 + }, + { + "metric": "movement", + "op": ">", + "value": 100 + } + ] + } + ], + "fallback": "light", + "smoothIsolatedEpochs": true, + "transitions": [ + { + "from": "wake", + "to": "deep", + "via": "light" + }, + { + "from": "deep", + "to": "wake", + "via": "light" + }, + { + "from": "deep", + "to": "rem", + "via": "light" + }, + { + "from": "rem", + "to": "deep", + "via": "light" + } + ] + } + } + }, + "orderedRanges": [ + [ + "sleep_stages", + "ruleset", + "outliers", + "heartRate" + ], + [ + "sleep_stages", + "ruleset", + "outliers", + "hrv" + ], + [ + "sleep_stages", + "ruleset", + "outliers", + "breathingRate" + ] + ], + "components": { + "piezo_preprocessing": { + "algorithmId": "piezo-presence-pump-v2", + "version": "1.0.0", + "runtime": "python3.10", + "requiredInputs": [ + "dual-channel-piezo", + "pump-state", + "occupancy", + "sensor-calibration", + "side-mode" + ], + "dependencies": [], + "metricId": null, + "unit": null, + "description": "Shared presence, occupancy veto, pump gating and channel arbitration; not user tunable.", + "causal": true + }, + "heart_rate": { + "algorithmId": "shs-tracker-v2", + "version": "1.0.0", + "runtime": "python3.10", + "requiredInputs": [ + "piezo-30s" + ], + "dependencies": [ + "piezo_preprocessing" + ], + "metricId": "heart_rate", + "unit": "bpm", + "description": "SHS heart-rate estimate with inter-window tracking.", + "causal": true + }, + "breathing_rate": { + "algorithmId": "hilbert-envelope-v2", + "version": "1.0.0", + "runtime": "python3.10", + "requiredInputs": [ + "piezo-60s" + ], + "dependencies": [ + "piezo_preprocessing" + ], + "metricId": "breathing_rate", + "unit": "breaths/min", + "description": "Breathing rate from the cardiac-band amplitude envelope.", + "causal": true + }, + "hrv_index": { + "algorithmId": "window-ibi-variability-v1", + "version": "1.0.0", + "runtime": "python3.10", + "requiredInputs": [ + "piezo-300s" + ], + "dependencies": [ + "piezo_preprocessing" + ], + "metricId": "window_ibi_variability_v1", + "unit": "ms", + "description": "Successive differences of window-level IBI estimates. Not clinical beat-to-beat RMSSD or SDNN.", + "causal": true + }, + "sleep_stages": { + "algorithmId": "retrospective-stage-rules-v1", + "version": "1.0.0", + "runtime": "node", + "requiredInputs": [ + "vitals", + "movement", + "piezo-calibration-quality" + ], + "dependencies": [ + "heart_rate", + "breathing_rate", + "hrv_index" + ], + "metricId": "sleep_stages", + "unit": "stage", + "description": "Retrospective wake/light/deep/REM classification using nightly mean HR and neighboring epochs.", + "causal": false + } + }, + "sourceHashes": { + "modules/common/biometrics/signal.py": "fff73469815673c8daebba9e2c9e7bcf97708939ad7a5639a52f0a53c9d75c18", + "modules/piezo-processor/main.py": "75db2ea2f005fd537d994b306332c1abf97c0c28abb496a8d3af03816ccf8550", + "src/lib/sleep-stages.ts": "b1800e49ce8525d314333da6a29a63131a002d160bb294fa97b24df6e655e228", + "modules/piezo-processor/runtime_adapter.py": "0a22a2fa05aae6569bfc4b53fba80d75e89f702f021ccb61f82ce64e284c5bb8", + "modules/common/biometrics/runtime.py": "dbe327ce6e6ae315cad83439903e4c5d6a5c594468086739504caa8ed028a944", + "modules/common/biometrics/gates.py": "c46285ff1cdf95bd4184823fabd6ceb66b005f7c410ee87880de4e563edd6e7c", + "modules/common/biometrics/routing.py": "04872bab0f797e0bedd6e7114e686bbf7a5b4110ed2052c9b160e5aa1f7d0839", + "modules/common/biometrics/processing.py": "2d04538499498f3b4fa9fa7383d1d3e6c0f66c1db3455bfd67ede53be7d080ef", + "modules/common/biometrics/frames.py": "99427d69cf63613c96e45f2f280b53a6e7db18997a28e579cd5e412684db259f", + "modules/common/biometrics/replay.py": "50c70408dca033dcf17d5bf6a1aa7afc9ed531d09aa3151eb61941f7c504639f" + }, + "dependencyHashes": { + "modules/piezo-processor/uv.lock": "2803397e553a32588071d6be84c2bce5326552da03076261f48afce0ba919869", + "pnpm-lock.yaml": "bf4c8635e3f34acd1637bdc33fab0c1a9ee1287dcad827d60c8b9b4027118a41" + }, + "legacySourceHashes": { + "modules/piezo-processor/main.py": "c98bdb3d6ee38b5de5d36760238978a8fefe2153a0cdf3e82ea1872d2d0da34f", + "src/lib/sleep-stages.ts": "853f6b1f14618fbac11da43d3246b95aac55af8b782b2cbb1be3f7627bf560b4" + } +} diff --git a/modules/common/biometrics/processing.py b/modules/common/biometrics/processing.py new file mode 100644 index 000000000..c4ecf8895 --- /dev/null +++ b/modules/common/biometrics/processing.py @@ -0,0 +1,225 @@ +"""Stateful per-channel processing shared by live ingestion and replay. + +All time, occupancy, routing, provenance and output are caller-owned. The +clock returns Unix seconds, and estimates and candidate timestamps use the +same event time. Retain +left-before-right ingestion: cross-channel rejection intentionally sees the +other side's most recently evaluated features, not a simultaneous recompute. +There are no database, hardware, transport or wall-clock reads here. +""" +from collections import deque +from datetime import datetime, timezone +import logging +from typing import Callable, Optional + +import numpy as np + +from .signal import (SAMPLE_RATE, _bandpass, _autocorr_quality, + ProcessingParameters, DEFAULT_PARAMETERS, WindowEstimator) +from .gates import (FrzHealthPumpState, PresenceDetector, + PUMP_COUPLING_STD_FACTOR, PUMP_COUPLING_ACR_THRESHOLD) +from .routing import VITALS_INTERVAL_S + +HR_WINDOW_S = 30 +BREATHING_WINDOW_S = 60 +HRV_WINDOW_S = 300 +log = logging.getLogger(__name__) + + +class StatefulSideProcessor: + def __init__(self, side: str, *, clock: Callable[[], float], + writer: Callable[[object], bool], candidate_factory: Callable, + pump_state: Optional[FrzHealthPumpState] = None, + parameters: ProcessingParameters = DEFAULT_PARAMETERS, + autocorr_quality: Callable = _autocorr_quality): + self.side = side + self._clock = clock + self._writer = writer + self._candidate_factory = candidate_factory + self._quality = autocorr_quality + self._hr_buf: deque = deque(maxlen=HR_WINDOW_S * SAMPLE_RATE) + self._hrv_buf: deque = deque(maxlen=HRV_WINDOW_S * SAMPLE_RATE) + self._br_buf: deque = deque(maxlen=BREATHING_WINDOW_S * SAMPLE_RATE) + self._last_write = 0.0 + self._presence = PresenceDetector() + self.parameters = parameters + self.run_context: Optional[object] = None + self._estimator = WindowEstimator(parameters) + self._hr_tracker = self._estimator.hr_tracker + self._other: Optional['StatefulSideProcessor'] = None # set after both sides created + self._last_med_std: float = 0.0 # cached for cross-channel comparison + self._last_acr_qual: float = 0.0 + self._pump_state = pump_state + # An optional arbitration sink; None uses the caller-provided writer. + # main() routes both sides through SingleSleeperVitals.submit. + self.sink: Optional[Callable[[object], bool]] = None + # The bed's own answer to "is anyone here" (sleep-detector presence): + # False vetoes piezo presence, None (unknown) leaves it alone. + self.bed_occupied: Optional[Callable[[], Optional[bool]]] = None + # Single-sleeper home side (None in per-side mode). The sleep-detector + # merges both zones into the home side, so only the home side's + # processor may treat `bed_occupied() is True` as its own evidence. + self.home_side: Optional[Callable[[], Optional[str]]] = None + + def install_runtime(self, parameters, run_context, reset_side=False): + if parameters != self.parameters or reset_side: + self._hr_buf.clear() + self._hrv_buf.clear() + self._br_buf.clear() + self._estimator = WindowEstimator(parameters) + self._hr_tracker = self._estimator.hr_tracker + self._last_write = 0.0 + self._last_med_std = self._last_acr_qual = 0.0 + self._presence = PresenceDetector() + self.parameters = parameters + self.run_context = run_context + + def ingest(self, samples: np.ndarray) -> None: + self._hr_buf.extend(samples) + self._hrv_buf.extend(samples) + self._br_buf.extend(samples) + self._maybe_write() + + def _maybe_write(self) -> None: + now = self._clock() + if now - self._last_write < VITALS_INTERVAL_S: + return + + hr_arr = np.array(self._hr_buf) + if len(hr_arr) < int(10 * SAMPLE_RATE): + return # Not enough data yet + + # --- Presence detection (hysteresis + autocorrelation quality) --- + filt = _bandpass(hr_arr, 1.0, 10.0, SAMPLE_RATE) + w = int(5 * SAMPLE_RATE) + stds = [np.std(filt[j:j + w]) + for j in range(0, len(filt) - w + 1, w)] + med_std = float(np.median(stds)) if stds else 0.0 + acr_qual = self._quality(hr_arr) + cross_channel_rejected = False + + # Cross-channel rejection: if the other side has stronger or similar + # signal AND strong autocorrelation (someone is actually there), this + # side's signal is likely vibration coupling, not a person. + # A real person produces ~3-5x stronger signal on their side. + # + # Note: due to sequential processing in main() (left before right), + # left sees right's values from the previous cycle (~60s stale). + # This is acceptable — coupling produces similar energy regardless + # of which side checks first. + if self._other is not None and self._other._last_med_std > 0: + other_std = self._other._last_med_std + other_acr = self._other._last_acr_qual + # Only suppress if: other side has real periodicity (someone there), + # other side's energy is >= ours, and our energy is below threshold + if (other_acr > 0.3 + and other_std > med_std * 0.7 + and med_std < self._presence.enter_threshold): + acr_qual = 0.0 + cross_channel_rejected = True + + # Cache AFTER suppression so the other side sees post-suppression values + self._last_med_std = med_std + self._last_acr_qual = acr_qual + + # The bed's own answer to "is anyone here" (sleep-detector capacitance + # presence), read once per cycle: True relaxes re-entry below, False + # vetoes piezo presence, None (unknown) leaves the piezo alone. + bed = self.bed_occupied() if self.bed_occupied is not None else None + # Only occupancy measured on this side may relax the piezo. With one + # side away the reading is merged under the home side, so the away + # side inherits a True it cannot attribute to its own zone and keeps + # the strict path. A side whose energy cross-channel rejection has + # already attributed to the other sleeper keeps it too. + own_side = self.home_side is None or self.home_side() in (None, self.side) + relaxed = bed is True and own_side and not cross_channel_rejected + + # Pump-coupling guard. PumpGate handles symmetric spikes (ramps) + # but misses two steady-state modes: asymmetric (own-pump-on with + # opposite idle — vibration couples into same-side piezo, no + # competing signal for cross-channel rejection) and symmetric + # (both pumps on at mismatched RPMs — beat frequency lands in the + # cardiac band, both sides look "real"). In both, require elevated + # energy AND strong autocorrelation to enter PRESENT; coupling + # rarely produces both, only a real person does. + # + # The guard answers "is this vibration a person or the pump". When + # the capacitance sensor says a person is in bed on this side and + # the energy is not the other sleeper's, that question is settled; + # on Pod 5 both pumps run all night, so without this bypass a + # sleeper who drifted to a weak-signal position stayed ABSENT for + # an hour at a time (05:57-07:01 on 2026-10-11) until a hard + # movement cleared the raised bar. + if (self._pump_state is not None and not relaxed + and self._presence.state == PresenceDetector.ABSENT): + symmetric = self._pump_state.is_symmetric_active() + asymmetric = self._pump_state.is_asymmetric_for(self.side) + if symmetric or asymmetric: + std_threshold = self._presence.enter_threshold * PUMP_COUPLING_STD_FACTOR + if not (med_std > std_threshold and acr_qual > PUMP_COUPLING_ACR_THRESHOLD): + log.debug( + "%s: pump-coupling guard suppressed presence " + "(med_std=%.0f, acr=%.2f, mode=%s)", + self.side, med_std, acr_qual, + "symmetric" if symmetric else "asymmetric", + ) + return + + if relaxed and self._presence.state == PresenceDetector.ABSENT: + log.debug("%s: bed reads occupied, relaxed piezo re-entry (med_std=%.0f, acr=%.2f)", + self.side, med_std, acr_qual) + if cross_channel_rejected: + # The other side owns this energy. Zeroing acr alone cannot end + # presence (exit needs weak energy too), so count the window as + # weak and let the normal hysteresis exit instead of latching, + # and never submit vitals computed from it meanwhile. + self._presence.update(0.0, 0.0) + present = False + else: + present = self._presence.update(med_std, acr_qual, relaxed=relaxed) + # Piezo presence is vibration energy and rhythm; a prime or the pump + # shaking an empty bed passes it. Trust the capacitance-based + # presence when the sleep-detector says nobody is there. + if present and bed is False: + present = False + + if not present: + # Reset the interval cursor so a return from extended absence + # doesn't trigger burst processing on every ingest until the + # first write completes (#325). + self._last_write = now + return # No user detected — skip + + # Shared numerical estimator. Gating and live sinks remain in this adapter. + estimate = self._estimator.estimate(now, hr_arr, np.array(self._br_buf), + np.array(self._hrv_buf)) + hr, hrv, br = estimate.heart_rate, estimate.hrv_index, estimate.breathing_rate + hr_raw, hr_score = estimate.hr_raw, estimate.hr_score + + if hr is not None or hrv is not None or br is not None: + ts = datetime.fromtimestamp(now, timezone.utc) + snr = max(0.0, min(1.0, acr_qual)) + hr_conf = max(0.0, min(1.0, hr_score)) if hr is not None else 0.0 + quality = round(0.55 * snr + 0.45 * hr_conf, 3) + flags = [] + if hr is None: + flags.append("no_hr") + if hrv is None: + flags.append("no_hrv") + if br is None: + flags.append("no_br") + if med_std < self._presence.enter_threshold: + flags.append("low_signal") + cand = self._candidate_factory(self.side, ts, hr, hrv, br, quality, flags or None, hr_raw, self.run_context) + wrote = self.sink(cand) if self.sink is not None else self._writer(cand) + log.info("vitals %s — HR=%.1f HRV=%.1f BR=%.1f q=%.2f", self.side, + hr or 0, hrv or 0, br or 0, quality) + # Only advance the downsample cursor when the write actually + # committed — otherwise a transient sqlite error would skip + # VITALS_INTERVAL_S of valid samples. + if wrote: + self._last_write = now + else: + # No metric to write — advance so we don't recompute on every ingest. + self._last_write = now + diff --git a/modules/common/biometrics/publication_worker.py b/modules/common/biometrics/publication_worker.py new file mode 100644 index 000000000..5dd21c6fa --- /dev/null +++ b/modules/common/biometrics/publication_worker.py @@ -0,0 +1,54 @@ +"""Fixed server-owned entry point for atomic waveform dataset publication.""" +import argparse +import json +from pathlib import Path +import sqlite3 +import sys + +from .replay_worker import _unique_object, _invalid_constant, configure_process_limits + +MAX_INPUT_BYTES = 10 * 1024 * 1024 + + +def run(input_stream, output_stream, *, database, archive_directory, retained_directory): + payload = input_stream.read(MAX_INPUT_BYTES + 1) + if len(payload) > MAX_INPUT_BYTES: + raise ValueError('Waveform publication input exceeds byte budget') + work = json.loads(payload, object_pairs_hook=_unique_object, parse_constant=_invalid_constant) + if not isinstance(work, dict) or set(work) != {'version', 'prepared', 'selections'} or work['version'] != 'waveform-publication-work-v1': + raise ValueError('Invalid waveform publication envelope') + from .raw_artifacts import RawArtifactStore + from .raw_publication import publish_waveform_dataset + # Never create an empty database at a mistyped deployment location. + connection = sqlite3.connect(Path(database).resolve().as_uri() + '?mode=rw', uri=True, timeout=5) + try: + connection.execute('PRAGMA foreign_keys = ON') + result = publish_waveform_dataset(RawArtifactStore(retained_directory), connection, + archive_directory, work['selections'], work['prepared']) + # The coordinator re-reads the descriptor by ID. Keep IPC independent of + # the manifest size, and permit idempotent recovery after a lost reply. + output_stream.write((json.dumps({'type': 'result', 'id': result['id'], + 'contentHash': result['contentHash'], 'available': result['available']}, separators=(',', ':')) + '\n').encode()) + output_stream.flush() + finally: + connection.close() + + +def main(): + parser = argparse.ArgumentParser(description='Publish fixed retained waveform inputs') + parser.add_argument('--database', required=True) + parser.add_argument('--archive-directory', required=True) + parser.add_argument('--retained-directory', required=True) + args = parser.parse_args() + try: + configure_process_limits() + run(sys.stdin.buffer, sys.stdout.buffer, database=args.database, + archive_directory=args.archive_directory, retained_directory=args.retained_directory) + return 0 + except Exception: + sys.stderr.write('Waveform publication worker failed\n') + return 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/modules/common/biometrics/raw_artifacts.py b/modules/common/biometrics/raw_artifacts.py new file mode 100644 index 000000000..ce8422626 --- /dev/null +++ b/modules/common/biometrics/raw_artifacts.py @@ -0,0 +1,388 @@ +"""Private retained RAW artifacts; all directories are server-owned configuration. + +Copy gzip bytes rather than keeping a checksum or a link into the rolling +archive. Publication and quota checks share a process lock. Readers verify the +retained checksum before exposing a bounded decompression stream. +""" +from contextlib import contextmanager +from dataclasses import dataclass +import fcntl +import gzip +import hashlib +import os +from pathlib import Path +import re +import shutil +import stat +import uuid + +CHUNK = 64 * 1024 +SOURCE_NAME = re.compile(r'[A-Za-z0-9_.-]+\.RAW\.gz\Z') +ARTIFACT_NAME = re.compile(r'[a-f0-9]{64}\.RAW\.gz\Z') +PART_NAME = re.compile(r'\.part-[a-f0-9]{32}\Z') + + +@dataclass(frozen=True) +class RawLimits: + compressed_bytes: int = 256 * 1024 * 1024 + expanded_bytes: int = 512 * 1024 * 1024 + total_bytes: int = 1024 * 1024 * 1024 + minimum_free_bytes: int = 512 * 1024 * 1024 + max_artifacts: int = 1024 + + def __post_init__(self): + if any(type(value) is not int or value <= 0 for value in vars(self).values()): + raise ValueError('RAW limits must be positive integers') + + +@dataclass(frozen=True) +class RawArtifact: + name: str + sha256: str + compressed_bytes: int + expanded_bytes: int + + def __post_init__(self): + if not ARTIFACT_NAME.fullmatch(self.name) or self.name != self.sha256 + '.RAW.gz': + raise ValueError('Invalid retained RAW identity') + if any(type(value) is not int or value < 0 for value in (self.compressed_bytes, self.expanded_bytes)): + raise ValueError('Invalid retained RAW size') + + +def _identity(info): + return (info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, info.st_ctime_ns) + + +def _regular(fd): + info = os.fstat(fd) + if not stat.S_ISREG(info.st_mode): + raise ValueError('RAW input must be a regular file') + return info + + +class BoundedRawReader: + def __init__(self, stream, limit): + self.stream, self.limit, self.position = stream, limit, 0 + + def read(self, size=-1): + if type(size) is not int or size < 0 or size > 1_000_000: + raise ValueError('RAW reads must be bounded to at most one record') + data = self.stream.read(min(size, self.limit - self.position + 1)) + self.position += len(data) + if self.position > self.limit: + raise ValueError('RAW expanded byte limit exceeded') + if size > 0 and not data and self.position != self.limit: + raise ValueError('RAW expanded size mismatch') + return data + + def tell(self): + return self.position + + +class RawArtifactStore: + def __init__(self, directory, limits=RawLimits(), *, read_only=False): + self.read_only = read_only + self.directory = Path(directory) + self.limits = limits + if not read_only: + self.directory.mkdir(mode=0o700, parents=True, exist_ok=True) + if self.directory.is_symlink() or not self.directory.is_dir(): + raise ValueError('Retained RAW directory must not be a symlink') + if not read_only: + os.chmod(self.directory, 0o700) + + @contextmanager + def _lock(self): + flags = os.O_RDONLY if self.read_only else os.O_CREAT | os.O_RDWR + fd = os.open(self.directory / '.lock', flags | os.O_NOFOLLOW | os.O_NONBLOCK, 0o600) + try: + _regular(fd) + fcntl.flock(fd, fcntl.LOCK_EX) + yield + finally: + os.close(fd) + + def _usage(self): + used, count = 0, 0 + for entry in self.directory.iterdir(): + if ARTIFACT_NAME.fullmatch(entry.name) or PART_NAME.fullmatch(entry.name): + info = entry.lstat() + if not stat.S_ISREG(info.st_mode): + raise ValueError('Invalid retained RAW directory entry') + used += info.st_size + count += int(bool(ARTIFACT_NAME.fullmatch(entry.name))) + return used, count + + def _sync_directory(self): + fd = os.open(self.directory, os.O_RDONLY) + try: + os.fsync(fd) + finally: + os.close(fd) + + def retain(self, archive_directory, name, expected=None, *, publish=None): + """Retain one source, optionally committing references under the lock.""" + if publish is not None and (not callable(publish) or expected is None): + raise ValueError('Publication requires a callable and inspected descriptor') + if self.read_only: + raise ValueError('Retained RAW store is read-only') + with self._lock(): + descriptor = self._retain_locked(archive_directory, name, expected) + if publish is not None: + publish(descriptor) + return descriptor + + def retain_batch(self, archive_directory, selections, *, publish): + """Publish a whole selected batch while holding the file lifecycle lock. + + The trusted callback begins and commits SQLite only after every copy is + verified. Never enter with an existing SQLite transaction or reacquire + the file lock inside the callback. A failed callback leaves complete + unreferenced files, but cannot publish a partial dataset. + """ + if self.read_only: + raise ValueError('Retained RAW store is read-only') + if not isinstance(selections, (list, tuple)) or not 1 <= len(selections) <= self.limits.max_artifacts or not callable(publish): + raise ValueError('Invalid retained RAW batch') + frozen, hashes = [], set() + for selection in selections: + if not isinstance(selection, (list, tuple)) or len(selection) != 2: + raise ValueError('Expected source name and inspected descriptor') + name, expected = selection + if not isinstance(name, str) or not SOURCE_NAME.fullmatch(name) or not isinstance(expected, RawArtifact): + raise ValueError('Invalid inspected RAW selection') + if expected.sha256 in hashes: + raise ValueError('Select each retained artifact once') + hashes.add(expected.sha256) + frozen.append((name, expected)) + with self._lock(): + descriptors = tuple(self._retain_locked(archive_directory, name, expected) for name, expected in frozen) + return publish(descriptors) + + def _retain_locked(self, archive_directory, name, expected): + if self.read_only: + raise ValueError('Retained RAW store is read-only') + if not isinstance(name, str) or not SOURCE_NAME.fullmatch(name): + raise ValueError('Invalid RAW archive filename') + if expected is not None and not isinstance(expected, RawArtifact): + raise ValueError('Expected an inspected RAW descriptor') + # Exact retained bytes are independent of the rolling source. Reuse + # them only after checking both compressed identity and expanded size. + if expected is not None: + cached = self.directory / expected.name + if cached.exists() or cached.is_symlink(): + fd = self._verify(expected) + with os.fdopen(fd, 'rb') as source, gzip.GzipFile(fileobj=source) as stream: + reader = BoundedRawReader(stream, expected.expanded_bytes) + while reader.read(CHUNK): + pass + self._sync_directory() + return expected + archive = Path(archive_directory) + if archive.is_symlink() or not archive.is_dir(): + raise ValueError('Invalid RAW archive directory') + # A dead copier can leave a private temporary file, never a published + # descriptor. The shared lock proves no other copier is using it. + for entry in self.directory.iterdir(): + if PART_NAME.fullmatch(entry.name): + entry.unlink() + source_fd = os.open(archive / name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + temporary = self.directory / ('.part-' + uuid.uuid4().hex) + try: + before = _regular(source_fd) + if before.st_size > self.limits.compressed_bytes: + raise ValueError('RAW compressed byte limit exceeded') + if shutil.disk_usage(self.directory).free - before.st_size < self.limits.minimum_free_bytes: + raise ValueError('Insufficient free space for retained RAW input') + digest, copied = hashlib.sha256(), 0 + target_fd = os.open(temporary, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600) + with os.fdopen(target_fd, 'wb') as target: + while True: + data = os.read(source_fd, CHUNK) + if not data: + break + copied += len(data) + if copied > self.limits.compressed_bytes: + raise ValueError('RAW compressed byte limit exceeded') + digest.update(data) + target.write(data) + target.flush() + os.fsync(target.fileno()) + if _identity(before) != _identity(_regular(source_fd)): + raise ValueError('RAW archive changed during retention') + expanded = 0 + with gzip.open(temporary, 'rb') as stream: + while True: + data = stream.read(min(CHUNK, self.limits.expanded_bytes - expanded + 1)) + if not data: + break + expanded += len(data) + if expanded > self.limits.expanded_bytes: + raise ValueError('RAW expanded byte limit exceeded') + sha256 = digest.hexdigest() + descriptor = RawArtifact(sha256 + '.RAW.gz', sha256, copied, expanded) + if expected is not None and descriptor != expected: + raise ValueError('RAW archive changed since inspection') + destination = self.directory / descriptor.name + if destination.exists() or destination.is_symlink(): + os.close(self._verify(descriptor)) + self._sync_directory() + return descriptor + # Include the unpublished copy when checking the publication quota. + used, count = self._usage() + if count >= self.limits.max_artifacts: + raise ValueError('Retained RAW artifact count exceeded') + if used > self.limits.total_bytes: + raise ValueError('Retained RAW storage quota exceeded') + os.replace(temporary, destination) + self._sync_directory() + return descriptor + finally: + os.close(source_fd) + temporary.unlink(missing_ok=True) + + def _verify(self, descriptor): + if not isinstance(descriptor, RawArtifact): + raise ValueError('Expected retained RAW descriptor') + if descriptor.compressed_bytes > self.limits.compressed_bytes or descriptor.expanded_bytes > self.limits.expanded_bytes: + raise ValueError('Retained RAW exceeds configured limits') + fd = os.open(self.directory / descriptor.name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + try: + info = _regular(fd) + if info.st_size != descriptor.compressed_bytes: + raise ValueError('Retained RAW size mismatch') + digest, count = hashlib.sha256(), 0 + while True: + data = os.read(fd, CHUNK) + if not data: + break + count += len(data) + if count > descriptor.compressed_bytes: + raise ValueError('Retained RAW changed during verification') + digest.update(data) + if _identity(info) != _identity(_regular(fd)): + raise ValueError('Retained RAW changed during verification') + if digest.hexdigest() != descriptor.sha256: + raise ValueError('Retained RAW checksum mismatch') + os.lseek(fd, 0, os.SEEK_SET) + return fd + except BaseException: + os.close(fd) + raise + + @contextmanager + def open_raw(self, descriptor): + with self._lock(): + fd = self._verify(descriptor) + with os.fdopen(fd, 'rb') as source, gzip.GzipFile(fileobj=source) as stream: + yield BoundedRawReader(stream, descriptor.expanded_bytes) + + def collect_released(self, connection, *, now_ms, minimum_age_ms=3_600_000, + maximum_entries=10_000): + """Collect only explicitly released metadata with no live references. + + Unknown files and metadata are preserved, including crash orphans. The + file mutex also covers readers and reference publication. BEGIN + IMMEDIATE prevents reference changes between the last check and unlink. + Tombstones remain in SQLite; scanning present files avoids repeatedly + selecting absent tombstones and starving later eligible files. + """ + if self.read_only: + raise ValueError('Retained RAW store is read-only') + if (type(now_ms) is not int or now_ms < 0 or type(minimum_age_ms) is not int + or minimum_age_ms < 0 or type(maximum_entries) is not int + or maximum_entries <= 0): + raise ValueError('Invalid RAW collection bounds') + if connection.in_transaction: + raise ValueError('RAW collection requires an independent transaction') + removed = [] + with self._lock(): + # Bound traversal before deleting anything; never partially process + # a directory that exceeds the agreed inventory budget. + names = [] + with os.scandir(self.directory) as entries: + for count, entry in enumerate(entries, 1): + if count > maximum_entries: + raise ValueError('RAW collection entry limit exceeded') + if ARTIFACT_NAME.fullmatch(entry.name): + names.append(entry.name) + connection.execute('BEGIN IMMEDIATE') + try: + for name in sorted(names): + eligible = connection.execute( + """SELECT content_hash FROM biometric_raw_artifacts AS artifact + WHERE name = ? AND released_at_ms IS NOT NULL + AND released_at_ms <= ? + AND NOT EXISTS ( + SELECT 1 FROM biometric_dataset_raw_artifacts AS reference + WHERE reference.artifact_hash = artifact.content_hash + AND reference.released_at_ms IS NULL)""", + (name, now_ms - minimum_age_ms)).fetchone() + if eligible is None: + continue + if name != eligible[0] + '.RAW.gz': + raise ValueError('Released RAW metadata identity mismatch') + path = self.directory / name + if not stat.S_ISREG(path.lstat().st_mode): + raise ValueError('Released RAW artifact must be a regular file') + path.unlink() + removed.append(name) + if removed: + self._sync_directory() + connection.commit() + except BaseException: + connection.rollback() + raise + return removed + + def collect_orphans(self, connection, *, now_ms, minimum_age_ms=3_600_000, + maximum_entries=10_000): + """Reconcile old unpublished copies without guessing metadata ownership. + + Publication holds this same mutex through its SQLite commit. A file with + any metadata row is preserved, including released tombstones; its normal + release collector owns that lifecycle. Unknown names are never touched. + """ + if self.read_only: + raise ValueError('Retained RAW store is read-only') + if (type(now_ms) is not int or now_ms < 0 or type(minimum_age_ms) is not int + or minimum_age_ms < 0 or type(maximum_entries) is not int + or maximum_entries <= 0): + raise ValueError('Invalid RAW collection bounds') + if connection.in_transaction: + raise ValueError('RAW collection requires an independent transaction') + with self._lock(): + names = [] + with os.scandir(self.directory) as entries: + for count, entry in enumerate(entries, 1): + if count > maximum_entries: + raise ValueError('RAW collection entry limit exceeded') + if ARTIFACT_NAME.fullmatch(entry.name) or PART_NAME.fullmatch(entry.name): + names.append(entry.name) + connection.execute('BEGIN IMMEDIATE') + try: + candidates = [] + for name in sorted(names): + # Check both identities so inconsistent metadata cannot make + # a referenced file look like an orphan. + if ARTIFACT_NAME.fullmatch(name) and connection.execute( + 'SELECT 1 FROM biometric_raw_artifacts WHERE name = ? OR content_hash = ?', + (name, name[:-7])).fetchone() is not None: + continue + path = self.directory / name + info = path.lstat() + if not stat.S_ISREG(info.st_mode): + raise ValueError('Orphan RAW artifact must be a regular file') + if info.st_mtime_ns > (now_ms - minimum_age_ms) * 1_000_000: + continue + candidates.append(path) + # Complete the bounded inventory and metadata checks first. + for path in candidates: + path.unlink() + if candidates: + self._sync_directory() + connection.commit() + return [path.name for path in candidates] + except BaseException: + connection.rollback() + raise diff --git a/modules/common/biometrics/raw_publication.py b/modules/common/biometrics/raw_publication.py new file mode 100644 index 000000000..c8d4d101e --- /dev/null +++ b/modules/common/biometrics/raw_publication.py @@ -0,0 +1,160 @@ +"""Atomic publication of server-prepared waveform datasets and RAW references. + +The coordinator validates and hashes the complete snapshot with its shared +TypeScript data-hash contract. This internal publisher never accepts a dataset +row directly from an RPC request. It verifies selected archive identities, +then inserts that exact prepared snapshot and all references in one transaction. +""" +import json +import re +import uuid + +from .raw_artifacts import RawArtifact, SOURCE_NAME + +HASH = re.compile(r'[a-f0-9]{64}\Z') +MAX_DATASET_BYTES = 8 * 1024 * 1024 +MAX_TOTAL_BYTES = 64 * 1024 * 1024 + + +def _json(value): + return json.dumps(value, ensure_ascii=False, allow_nan=False, separators=(',', ':')) + + +def _describe(row): + keys = ('id', 'side', 'subjectId', 'requestHash', 'contentHash', 'manifest', + 'byteLength', 'createdAtMs', 'releasedAtMs') + result = dict(zip(keys, row)) + result['manifest'] = json.loads(result['manifest']) + result['available'] = result['releasedAtMs'] is None + return result + + +def _existing(connection, key, request_hash): + row = connection.execute('''SELECT id, side, subject_id, request_hash, content_hash, + manifest, byte_length, created_at_ms, released_at_ms FROM biometric_datasets + WHERE idempotency_key = ?''', (key,)).fetchone() + if row is None: + return None + result = _describe(row) + if result['requestHash'] != request_hash: + raise ValueError('Dataset idempotency key changed') + return result + + +def publish_waveform_dataset(store, connection, archive_directory, selections, prepared): + """Copy under the file mutex; keep SQLite's write lock out of file I/O.""" + from .retained_replay import RetainedReplayInput, _integer, _keys + if connection.in_transaction: + raise ValueError('Waveform publication requires an independent transaction') + if not connection.execute('PRAGMA foreign_keys').fetchone()[0]: + raise ValueError('Waveform publication requires foreign keys') + # Freeze caller-owned dictionaries before copying potentially large files. + serialized = _json(prepared) + if len(serialized.encode('utf-8')) > MAX_DATASET_BYTES: + raise ValueError('Waveform dataset exceeds its byte budget') + dataset = json.loads(serialized) + _keys(dataset, 'id side subjectId idempotencyKey requestHash contentHash manifest snapshot createdAtMs') + if str(uuid.UUID(dataset['id'])) != dataset['id']: + raise ValueError('Invalid waveform dataset identity') + if dataset['side'] not in ('left', 'right'): + raise ValueError('Invalid waveform dataset side') + for key in ('subjectId', 'idempotencyKey'): + if not isinstance(dataset[key], str) or not 1 <= len(dataset[key]) <= 200: + raise ValueError('Invalid waveform dataset key') + for key in ('requestHash', 'contentHash'): + if not isinstance(dataset[key], str) or not HASH.fullmatch(dataset[key]): + raise ValueError('Invalid waveform dataset hash') + _integer(dataset['createdAtMs']) + snapshot, manifest = dataset['snapshot'], dataset['manifest'] + if (not isinstance(snapshot, dict) or snapshot.get('version') != 'waveform-dataset-v1' + or snapshot.get('side') != dataset['side'] or snapshot.get('subjectId') != dataset['subjectId'] + or not isinstance(manifest, dict) or manifest.get('version') != 'waveform-dataset-manifest-v1'): + raise ValueError('Invalid prepared waveform snapshot') + cohorts = manifest.get('splitProtocol', {}).get('assignments', []) + if not isinstance(cohorts, list) or not 1 <= len(cohorts) <= 60: + raise ValueError('Invalid waveform cohort assignments') + cohort_map = {item['cohortId']: item for item in cohorts} + if len(cohort_map) != len(cohorts): + raise ValueError('Duplicate waveform cohort assignment') + inputs = snapshot.get('waveformInputs') + if not isinstance(inputs, list) or not 1 <= len(inputs) <= 60: + raise ValueError('Invalid waveform dataset inputs') + artifacts, evidence, selected_cohorts = {}, {}, set() + for item in inputs: + _keys(item, 'cohortId input') + cohort_id = item['cohortId'] + if cohort_id not in cohort_map or cohort_id in selected_cohorts: + raise ValueError('Waveform inputs must match whole-night cohorts') + selected_cohorts.add(cohort_id) + retained = RetainedReplayInput.parse(item['input']) + cohort = cohort_map[cohort_id] + if retained.score_start_ms != cohort['startMs'] or retained.score_end_ms != cohort['endMs']: + raise ValueError('Waveform scoring window must match its whole-night cohort') + for source in retained.artifacts: + artifact = source.artifact + identity = (artifact, source.first_event_ms, source.last_event_ms) + if artifact.sha256 in evidence and evidence[artifact.sha256] != identity: + raise ValueError('Conflicting waveform artifact descriptors') + artifacts[artifact.sha256] = artifact + evidence[artifact.sha256] = identity + if len(artifacts) > store.limits.max_artifacts or sum(item.compressed_bytes for item in artifacts.values()) > store.limits.total_bytes: + raise ValueError('Waveform dataset RAW storage budget exceeded') + if selected_cohorts != set(cohort_map): + raise ValueError('Retain waveform inputs for every selected cohort') + if not isinstance(selections, list) or len(selections) != len(artifacts): + raise ValueError('Archive selection must match frozen waveform inputs') + names = {} + for item in selections: + _keys(item, 'source_name artifact') + _keys(item['artifact'], 'name sha256 compressed_bytes expanded_bytes') + artifact = RawArtifact(**item['artifact']) + name = item['source_name'] + if not isinstance(name, str) or not SOURCE_NAME.fullmatch(name): + raise ValueError('Invalid archive selection name') + if artifact.sha256 in names or artifacts.get(artifact.sha256) != artifact: + raise ValueError('Archive selection must match frozen waveform inputs') + names[artifact.sha256] = name + existing = _existing(connection, dataset['idempotencyKey'], dataset['requestHash']) + if existing is not None: + return existing # Includes released tombstones; never resurrect a retry. + manifest_json, snapshot_json = _json(manifest), _json(snapshot) + byte_length = len(manifest_json.encode('utf-8')) + len(snapshot_json.encode('utf-8')) + + def publish(descriptors): + connection.execute('BEGIN IMMEDIATE') + try: + raced = _existing(connection, dataset['idempotencyKey'], dataset['requestHash']) + if raced is not None: + connection.commit() + return raced + total = connection.execute('SELECT coalesce(sum(byte_length), 0) FROM biometric_datasets').fetchone()[0] + if total + byte_length > MAX_TOTAL_BYTES: + raise ValueError('Waveform datasets exceeded the retained storage budget') + connection.execute('''INSERT INTO biometric_datasets + (id, side, subject_id, idempotency_key, request_hash, content_hash, manifest, + snapshot, byte_length, created_at_ms) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''', + (dataset['id'], dataset['side'], dataset['subjectId'], dataset['idempotencyKey'], + dataset['requestHash'], dataset['contentHash'], manifest_json, snapshot_json, + byte_length, dataset['createdAtMs'])) + for position, descriptor in enumerate(descriptors): + existing_artifact = connection.execute('''SELECT name, compressed_bytes, expanded_bytes + FROM biometric_raw_artifacts WHERE content_hash = ?''', (descriptor.sha256,)).fetchone() + identity = (descriptor.name, descriptor.compressed_bytes, descriptor.expanded_bytes) + if existing_artifact is not None and tuple(existing_artifact) != identity: + raise ValueError('Conflicting retained artifact metadata') + connection.execute('''INSERT INTO biometric_raw_artifacts + (content_hash, name, compressed_bytes, expanded_bytes, created_at_ms) + VALUES (?, ?, ?, ?, ?) ON CONFLICT(content_hash) DO UPDATE SET released_at_ms = NULL''', + (descriptor.sha256, *identity, dataset['createdAtMs'])) + connection.execute('''INSERT INTO biometric_dataset_raw_artifacts + (id, dataset_id, artifact_hash, position) VALUES (?, ?, ?, ?)''', + (str(uuid.uuid4()), dataset['id'], descriptor.sha256, position)) + result = _existing(connection, dataset['idempotencyKey'], dataset['requestHash']) + connection.commit() + return result + except BaseException: + connection.rollback() + raise + + return store.retain_batch(archive_directory, + [(names[digest], artifact) for digest, artifact in artifacts.items()], publish=publish) diff --git a/modules/common/biometrics/raw_records.py b/modules/common/biometrics/raw_records.py new file mode 100644 index 000000000..706eaa2b9 --- /dev/null +++ b/modules/common/biometrics/raw_records.py @@ -0,0 +1,234 @@ +"""Bounded firmware CBOR records for retained replay inputs. + +Decode containers incrementally instead of trusting attacker/corruption supplied +container lengths to a decoder allocation. Firmware uses indefinite maps, so +these remain supported along with definite containers and strings. Semantic +tags and non-text map keys are outside this versioned firmware-record contract. +""" +from dataclasses import dataclass +import struct + +from common.cbor_raw import read_raw_record + + +@dataclass(frozen=True) +class DecodeLimits: + record_bytes: int = 128 * 1024 + string_bytes: int = 64 * 1024 + nodes: int = 4096 + depth: int = 16 + + def __post_init__(self): + if any(type(value) is not int or value <= 0 for value in vars(self).values()): + raise ValueError('Invalid RAW decoding limits') + + +class _Decoder: + def __init__(self, payload, limits): + self.payload, self.limits, self.position, self.nodes = payload, limits, 0, 0 + + def take(self, size): + if size > len(self.payload) - self.position: + raise ValueError('Truncated inner CBOR record') + start = self.position + self.position += size + return self.payload[start:self.position] + + def argument(self, additional): + if additional < 24: + return additional + size = {24: 1, 25: 2, 26: 4, 27: 8}.get(additional) + if size is None: + raise ValueError('Invalid CBOR length encoding') + return int.from_bytes(self.take(size), 'big') + + def node(self): + self.nodes += 1 + if self.nodes > self.limits.nodes: + raise ValueError('CBOR node budget exceeded') + + def ended(self): + if self.position >= len(self.payload): + raise ValueError('Truncated indefinite CBOR container') + if self.payload[self.position] == 0xff: + self.position += 1 + return True + return False + + def value(self, depth=0): + self.node() + if depth > self.limits.depth: + raise ValueError('CBOR nesting budget exceeded') + header = self.take(1)[0] + major, additional = header >> 5, header & 31 + if major in (0, 1): + number = self.argument(additional) + return number if major == 0 else -1 - number + if major in (2, 3): + if additional == 31: + chunks, total = [], 0 + while not self.ended(): + self.node() + chunk = self.take(1)[0] + if chunk >> 5 != major or chunk & 31 == 31: + raise ValueError('Invalid indefinite CBOR string chunk') + size = self.argument(chunk & 31) + total += size + if total > self.limits.string_bytes: + raise ValueError('CBOR string budget exceeded') + data = self.take(size) + if major == 3: + data.decode('utf-8', errors='strict') + chunks.append(data) + data = b''.join(chunks) + else: + size = self.argument(additional) + if size > self.limits.string_bytes: + raise ValueError('CBOR string budget exceeded') + data = self.take(size) + return data if major == 2 else data.decode('utf-8', errors='strict') + if major in (4, 5): + count = None if additional == 31 else self.argument(additional) + multiplier = 2 if major == 5 else 1 + if count is not None and count * multiplier > self.limits.nodes - self.nodes: + raise ValueError('CBOR container budget exceeded') + result = {} if major == 5 else [] + consumed = 0 + while count is None or consumed < count: + if count is None and self.ended(): + break + item = self.value(depth + 1) + if major == 5: + if not isinstance(item, str): + raise ValueError('CBOR record map keys must be text') + if item in result: + raise ValueError('Duplicate CBOR record key') + result[item] = self.value(depth + 1) + else: + result.append(item) + consumed += 1 + return result + if major == 7: + if additional in (20, 21, 22): + return {20: False, 21: True, 22: None}[additional] + encoding = {25: ('>e', 2), 26: ('>f', 4), 27: ('>d', 8)}.get(additional) + if encoding: + return struct.unpack(encoding[0], self.take(encoding[1]))[0] + raise ValueError('Unsupported CBOR record value') + + +def decode_record(payload, limits=DecodeLimits()): + if not isinstance(payload, bytes) or len(payload) > limits.record_bytes: + raise ValueError('RAW inner record byte budget exceeded') + decoder = _Decoder(payload, limits) + result = decoder.value() + if decoder.position != len(payload): + raise ValueError('Trailing bytes after CBOR record') + if not isinstance(result, dict) or not isinstance(result.get('type'), str): + raise ValueError('RAW record must have a textual type') + return result + + +class RawRecordBudget: + """One count budget shared by all artifacts, including empty placeholders.""" + def __init__(self, maximum): + if type(maximum) is not int or maximum <= 0: + raise ValueError('Invalid RAW record count limit') + self.maximum, self.consumed = maximum, 0 + + def consume(self): + self.consumed += 1 + if self.consumed > self.maximum: + raise ValueError('RAW record count limit exceeded') + + +def iter_retained_records(store, descriptor, limits=DecodeLimits(), max_records=1_000_000, *, budget=None): + """Yield decoded records without buffering a file or concealing truncation.""" + if type(max_records) is not int or max_records <= 0: + raise ValueError('Invalid RAW record count limit') + if budget is not None and not isinstance(budget, RawRecordBudget): + raise ValueError('Invalid shared RAW record budget') + with store.open_raw(descriptor) as stream: + count = 0 + while True: + offset = stream.tell() + try: + payload = read_raw_record(stream) + except EOFError: + if stream.tell() != offset: + raise ValueError('Truncated retained RAW framing') from None + return + count += 1 + if budget is not None: + budget.consume() + if count > max_records: + raise ValueError('RAW record count limit exceeded') + if payload is not None: + yield decode_record(payload, limits) + + +def inventory_retained(store, descriptor, limits=DecodeLimits(), max_records=1_000_000): + """Inventory recorded event times and channel bytes, not inferred occupancy. + + Timestamp bounds are event bounds, not a claim of continuous signal support. + Missing timestamps or reversals make this ordered replay contract unavailable. + """ + import math + + result = {'version': 'retained-raw-inventory-v1', 'artifact_sha256': descriptor.sha256, + 'decoded_records': 0, 'piezo_records': 0, 'pump_records': 0, + 'usable_dual_frames': 0, 'empty_channel_frames': 0, + 'oversize_channel_frames': 0, 'invalid_timestamps': 0, + 'timestamp_reversals': 0, 'duplicate_timestamps': 0, + 'first_event_ms': None, 'last_event_ms': None, + 'channels': {side: {'samples': 0, 'lost_markers': 0, 'truncated_tail_bytes': 0} + for side in ('left1', 'right1')}} + previous = None + for record in iter_retained_records(store, descriptor, limits, max_records): + result['decoded_records'] += 1 + kind = record['type'] + if kind not in ('piezo-dual', 'frzHealth'): + continue + timestamp = record.get('ts') + if (type(timestamp) not in (int, float) or not math.isfinite(timestamp) + or timestamp < 0 or timestamp > 9_007_199_254_740): + result['invalid_timestamps'] += 1 + else: + timestamp_ms = math.floor(timestamp * 1000 + 0.5) + if previous is not None: + result['timestamp_reversals'] += int(timestamp_ms < previous) + result['duplicate_timestamps'] += int(timestamp_ms == previous) + previous = timestamp_ms + result['first_event_ms'] = timestamp_ms if result['first_event_ms'] is None else min(result['first_event_ms'], timestamp_ms) + result['last_event_ms'] = timestamp_ms if result['last_event_ms'] is None else max(result['last_event_ms'], timestamp_ms) + if kind == 'frzHealth': + result['pump_records'] += 1 + continue + result['piezo_records'] += 1 + usable, oversized = True, False + for side, totals in result['channels'].items(): + payload = record.get(side) + if not isinstance(payload, bytes): + usable = False + continue + if len(payload) > 20_000: + oversized = True + usable = False + continue + size = len(payload) - len(payload) % 4 + count = size // 4 + lost = sum(value == 2_147_483_647 for (value,) in struct.iter_unpack(' lost + result['usable_dual_frames'] += int(usable) + result['empty_channel_frames'] += int(not usable and not oversized) + result['oversize_channel_frames'] += int(oversized) + result['ordered_waveform_available'] = (result['usable_dual_frames'] > 0 + and result['invalid_timestamps'] == 0 and result['timestamp_reversals'] == 0 + and result['oversize_channel_frames'] == 0) + result['limitations'] = ['Event timestamps do not establish continuous waveform coverage.', + 'Pump record counts do not prove complete historical pump context.', + 'Sleeper mode, occupancy and calibration require separately retained context.'] + return result diff --git a/modules/common/biometrics/raw_sources.py b/modules/common/biometrics/raw_sources.py new file mode 100644 index 000000000..3b3e385d5 --- /dev/null +++ b/modules/common/biometrics/raw_sources.py @@ -0,0 +1,100 @@ +"""Read-only inspection of server-owned cold archives before selected retention. + +This adapter has the same bounded reader interface as retained artifacts, but +its inspection is only evidence about the current source. Selection must pass +the inspected descriptor to RawArtifactStore.retain(expected=...) before a +replay dataset can claim that bytes have been frozen. +""" +from contextlib import contextmanager +import gzip +import hashlib +import os +from pathlib import Path + +from .raw_artifacts import (RawArtifact, RawLimits, BoundedRawReader, SOURCE_NAME, + CHUNK, _identity, _regular) +from .raw_records import inventory_retained + + +def archive_names(directory, *, maximum_entries=10000): + if type(maximum_entries) is not int or maximum_entries <= 0: + raise ValueError('Invalid archive discovery bound') + root = Path(directory) + if root.is_symlink() or not root.is_dir(): + raise ValueError('Invalid RAW archive directory') + names = [] + with os.scandir(root) as entries: + for count, entry in enumerate(entries, 1): + if count > maximum_entries: + raise ValueError('RAW archive discovery entry limit exceeded') + if SOURCE_NAME.fullmatch(entry.name) and entry.is_file(follow_symlinks=False): + names.append(entry.name) + return sorted(names) + + +class ArchiveSource: + def __init__(self, directory, name, limits=RawLimits()): + root = Path(directory) + if root.is_symlink() or not root.is_dir() or not isinstance(name, str) or not SOURCE_NAME.fullmatch(name): + raise ValueError('Invalid RAW archive source') + self.path, self.limits = root / name, limits + + def _open(self): + fd = os.open(self.path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + try: + info = _regular(fd) + if info.st_size > self.limits.compressed_bytes: + raise ValueError('RAW compressed byte limit exceeded') + return fd, info + except BaseException: + os.close(fd) + raise + + def _hash(self, fd): + digest, size = hashlib.sha256(), 0 + while True: + chunk = os.read(fd, CHUNK) + if not chunk: + break + size += len(chunk) + if size > self.limits.compressed_bytes: + raise ValueError('RAW compressed byte limit exceeded') + digest.update(chunk) + return digest.hexdigest(), size + + def inspect(self): + fd, before = self._open() + with os.fdopen(fd, 'rb') as source: + digest, size = self._hash(source.fileno()) + source.seek(0) + expanded = 0 + with gzip.GzipFile(fileobj=source) as stream: + while True: + chunk = stream.read(min(CHUNK, self.limits.expanded_bytes - expanded + 1)) + if not chunk: + break + expanded += len(chunk) + if expanded > self.limits.expanded_bytes: + raise ValueError('RAW expanded byte limit exceeded') + if _identity(before) != _identity(_regular(source.fileno())): + raise ValueError('RAW archive changed during inspection') + descriptor = RawArtifact(digest + '.RAW.gz', digest, size, expanded) + # Reopening with this exact hash detects replacement between passes. + inventory = inventory_retained(self, descriptor) + return {'source_name': self.path.name, 'retained': False, + 'artifact': descriptor, 'inventory': inventory} + + @contextmanager + def open_raw(self, descriptor): + if not isinstance(descriptor, RawArtifact) or descriptor.expanded_bytes > self.limits.expanded_bytes: + raise ValueError('Invalid inspected RAW descriptor') + fd, before = self._open() + with os.fdopen(fd, 'rb') as source: + digest, size = self._hash(source.fileno()) + if (digest, size) != (descriptor.sha256, descriptor.compressed_bytes): + raise ValueError('RAW archive changed since inspection') + source.seek(0) + with gzip.GzipFile(fileobj=source) as stream: + yield BoundedRawReader(stream, descriptor.expanded_bytes) + if _identity(before) != _identity(_regular(source.fileno())): + raise ValueError('RAW archive changed during reading') diff --git a/modules/common/biometrics/replay.py b/modules/common/biometrics/replay.py new file mode 100644 index 000000000..ccd490ce0 --- /dev/null +++ b/modules/common/biometrics/replay.py @@ -0,0 +1,194 @@ +"""Bounded event-time waveform replay using the shipped stateful processor. + +Callers stream retained records and complete recorded context snapshots. This +module never opens live state or a database. Output is delivered incrementally +to an experiment sink; the driver retains only bounded processor buffers. +""" +from dataclasses import dataclass +from datetime import datetime +from typing import Callable, Mapping, Optional, Tuple + +from .config import config_hash, validate_config +from .frames import decode_int32_samples +from .gates import FrzHealthPumpState, PumpGate +from .processing import StatefulSideProcessor +from .routing import SingleSleeperRouter +from .signal import ProcessingParameters + +SIDES = ('left', 'right') +MAX_CHANNEL_BYTES = 5000 * 4 # Ten seconds at the supported 500 Hz rate. + + +@dataclass(frozen=True) +class RecordedContext: + active_sides: Optional[Tuple[str, ...]] = None + left_occupied: Optional[bool] = None + right_occupied: Optional[bool] = None + pump_context_known: bool = False + + def __post_init__(self): + if self.active_sides not in (None, (), ('left',), ('right',), SIDES): + raise ValueError('Invalid recorded sleeper mode') + for value in (self.left_occupied, self.right_occupied): + if value is not None and type(value) is not bool: + raise ValueError('Invalid recorded occupancy') + if type(self.pump_context_known) is not bool: + raise ValueError('Invalid pump context availability') + + +@dataclass(frozen=True) +class ReplayProvenance: + side: str + source_side: str + config_hash: str + + +@dataclass(frozen=True) +class ReplayCandidate: + side: str + ts: datetime + heart_rate: Optional[float] + hrv: Optional[float] + breathing_rate: Optional[float] + quality_score: float + flags: Optional[Tuple[str, ...]] + hr_raw: Optional[float] + run_context: ReplayProvenance + + def __post_init__(self): + object.__setattr__(self, 'flags', tuple(self.flags) if self.flags else None) + + +@dataclass(frozen=True) +class ReplayReading: + candidate: ReplayCandidate + side: str + timestamp_ms: int + limitations: Tuple[str, ...] + + +class WaveformReplay: + """Replay [retained_start_ms, score_end_ms); score only the requested window. + + Timestamps must be integer Unix milliseconds in retained record order. + Equal timestamps are allowed (pump/context and piezo records can coincide). + Both physical channels are always processed, left first. Configurations are + fixed experiment snapshots indexed by logical side, not live bindings. + """ + def __init__(self, configs: Mapping[str, dict], *, retained_start_ms: int, + score_start_ms: int, score_end_ms: int, + sink: Callable[[ReplayReading], None], clock_mode='recorded_event_time'): + for value in (retained_start_ms, score_start_ms, score_end_ms): + if type(value) is not int or value < 0: + raise ValueError('Invalid replay timestamp') + if not retained_start_ms <= score_start_ms < score_end_ms: + raise ValueError('Invalid replay window') + if set(configs) != set(SIDES): + raise ValueError('Both logical-side configurations are required') + for config in configs.values(): + validate_config(config) + self.parameters = {side: ProcessingParameters.from_config(configs[side]) for side in SIDES} + self.hashes = {side: config_hash(configs[side]) for side in SIDES} + self.retained_start_ms, self.score_start_ms, self.score_end_ms = retained_start_ms, score_start_ms, score_end_ms + self._now_ms = retained_start_ms + self._closed = False + self._context = RecordedContext() + self._sink = sink + if clock_mode not in ('recorded_event_time', 'observed_arrival_time'): + raise ValueError('Invalid processing clock mode') + self.limitations = {'cold_start_without_serialized_state'} + if clock_mode == 'recorded_event_time': + self.limitations.add('event_time_replaces_arrival_time') + if score_start_ms - retained_start_ms < 300_000: + self.limitations.add('less_than_five_minutes_preroll') + self.counts = {'records': 0, 'accepted_frames': 0, 'empty_channel_frames': 0, + 'pump_gated_frames': 0, 'preroll_outputs': 0, 'scored_outputs': 0} + clock = lambda: self._now_ms / 1000 + self.pumps = FrzHealthPumpState(clock=clock) + self.gate = PumpGate(clock=clock) + self.router = SingleSleeperRouter(self, self._write, clock=clock) + self.processors = {} + for side in SIDES: + processor = StatefulSideProcessor(side, clock=clock, + writer=lambda candidate: self._write(candidate, None), + candidate_factory=ReplayCandidate, pump_state=self.pumps, + parameters=self.parameters[side]) + processor.sink = self.router.submit + processor.home_side = self.home_side + processor.bed_occupied = lambda side=side: self._occupied(self.home_side() or side) + self.processors[side] = processor + self.processors['left']._other = self.processors['right'] + self.processors['right']._other = self.processors['left'] + + def active_sides(self): + # The fallback is explicitly approximate, never reported as known mode. + return self._context.active_sides if self._context.active_sides is not None else SIDES + + def home_side(self): + active = self.active_sides() + return active[0] if len(active) == 1 else None + + def _occupied(self, side): + return self._context.left_occupied if side == 'left' else self._context.right_occupied + + def _write(self, candidate, destination): + timestamp_ms = round(candidate.ts.timestamp() * 1000) + if self.score_start_ms <= timestamp_ms < self.score_end_ms: + # Like live provenance, a held candidate retains its computed side + # even when the recorded mode changes before its sink is flushed. + self._sink(ReplayReading(candidate, candidate.run_context.side, + timestamp_ms, tuple(sorted(self.limitations)))) + self.counts['scored_outputs'] += 1 + else: + self.counts['preroll_outputs'] += 1 + return True + + def feed(self, timestamp_ms: int, record: dict, context: RecordedContext): + if self._closed: + raise ValueError('Replay is closed') + if type(timestamp_ms) is not int or not self._now_ms <= timestamp_ms < self.score_end_ms: + raise ValueError('Replay records must be ordered inside the retained window') + if not isinstance(record, dict) or not isinstance(context, RecordedContext): + raise ValueError('Invalid replay record or context') + for side in SIDES: + payload = record.get(side + '1', b'') + if isinstance(payload, (bytes, bytearray, memoryview)) and len(payload) > MAX_CHANNEL_BYTES: + raise ValueError('Replay channel exceeds bounded frame size') + self._now_ms, self._context = timestamp_ms, context + self.counts['records'] += 1 + if context.active_sides is None: + self.limitations.add('missing_sleeper_mode') + if context.left_occupied is None or context.right_occupied is None: + self.limitations.add('missing_occupancy_context') + if not context.pump_context_known: + self.limitations.add('missing_pump_context') + self.router.tick() + if record.get('type') == 'frzHealth': + self.pumps.update(record) + return + if record.get('type') != 'piezo-dual': + return + left = decode_int32_samples(record.get('left1', b'')) + right = decode_int32_samples(record.get('right1', b'')) + if left.size == 0 or right.size == 0: + self.counts['empty_channel_frames'] += 1 + return + if self.gate.check(left, right): + self.counts['pump_gated_frames'] += 1 + return + for side in SIDES: + processor = self.processors[side] + logical = self.home_side() or side + previous = processor.run_context + provenance = ReplayProvenance(logical, side, self.hashes[logical]) + processor.install_runtime(self.parameters[logical], provenance, + reset_side=previous is None or previous.side != logical) + self.processors['left'].ingest(left) + self.processors['right'].ingest(right) + self.counts['accepted_frames'] += 1 + + def close(self): + if not self._closed: + self._now_ms = self.score_end_ms + self.router.flush() + self._closed = True diff --git a/modules/common/biometrics/replay_worker.py b/modules/common/biometrics/replay_worker.py new file mode 100644 index 000000000..8d5cf8be6 --- /dev/null +++ b/modules/common/biometrics/replay_worker.py @@ -0,0 +1,167 @@ +"""Fixed shipped waveform worker; JSON input and bounded JSON-lines output. + +This is process/resource separation for trusted shipped algorithms, not a +sandbox for uploaded executable code. The coordinator owns staging and fenced +result publication; reading events alone never constitute a successful job. +""" +import argparse +import json +import os +import subprocess +import re +import sys +import time + +from .execution_lock import acquire_execution_lock + +MAX_INPUT_BYTES = 8 * 1024 * 1024 +MAX_OUTPUT_BYTES = 16 * 1024 * 1024 + + +def _unique_object(pairs): + value = {} + for key, item in pairs: + if key in value: + raise ValueError('Duplicate input field') + value[key] = item + return value + + +def _invalid_constant(_): + raise ValueError('Nonfinite JSON input') + + +def peak_rss_bytes(): + """Kernel process-lifetime high water mark; never substitute current RSS. + + Linux procfs also works on Yocto without the optional resource extension. + Missing or unrecognized telemetry remains unknown, never zero. + """ + try: + if sys.platform == 'linux': + with open('/proc/self/status', 'r', encoding='ascii') as source: + status = source.read(16385) + if len(status) > 16384: + return None + matches = re.findall(r'^VmHWM:\s+(\d+) kB$', status, re.MULTILINE) + value = int(matches[0]) * 1024 if len(matches) == 1 else None + elif sys.platform == 'darwin': + import resource + value = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + else: + return None + return value if type(value) is int and 0 < value <= 2 ** 53 - 1 else None + except (OSError, UnicodeError, ImportError, ValueError): + return None + + +def run(input_stream, output_stream, retained_directory): + started_wall = time.perf_counter_ns() + started_cpu = time.process_time_ns() + payload = input_stream.read(MAX_INPUT_BYTES + 1) + if len(payload) > MAX_INPUT_BYTES: + raise ValueError('Waveform work input exceeds byte budget') + work = json.loads(payload, object_pairs_hook=_unique_object, parse_constant=_invalid_constant) + if not isinstance(work, dict) or set(work) not in ({'version', 'input', 'configs'}, {'version', 'input', 'configs', 'expected_build_hash'}) or work['version'] != 'retained-waveform-work-v1': + raise ValueError('Invalid waveform work envelope') + from .build_identity import replay_build_identity + build = replay_build_identity() + if 'expected_build_hash' in work and (not isinstance(work['expected_build_hash'], str) + or not re.fullmatch(r'[a-f0-9]{64}', work['expected_build_hash']) + or work['expected_build_hash'] != build['hash']): + raise ValueError('Replay build identity changed') + # Import numerical dependencies after CLI resource/BLAS setup. + from .raw_artifacts import RawArtifactStore + from .retained_replay import replay_retained + store = RawArtifactStore(retained_directory, read_only=True) + output_bytes = 0 + + def emit(event): + nonlocal output_bytes + encoded = (json.dumps(event, allow_nan=False, separators=(',', ':')) + '\n').encode('utf-8') + output_bytes += len(encoded) + if output_bytes > MAX_OUTPUT_BYTES: + raise ValueError('Waveform work output exceeds byte budget') + output_stream.write(encoded) + output_stream.flush() + + def reading(row): + candidate = row.candidate + emit({'type': 'reading', 'reading': { + 'side': row.side, 'timestampMs': row.timestamp_ms, + 'heartRate': candidate.heart_rate, 'hrvIndex': candidate.hrv, + 'hrvMetric': 'window_ibi_variability_v1', + 'breathingRate': candidate.breathing_rate, 'qualityScore': candidate.quality_score, + 'flags': candidate.flags, 'hrRaw': candidate.hr_raw, + 'sourceSide': candidate.run_context.source_side, + 'configHash': candidate.run_context.config_hash, 'limitations': row.limitations, + }}) + + result = replay_retained(store, work['input'], work['configs'], reading) + # Detect source/dependency metadata changes during this process as well. + if replay_build_identity()['hash'] != build['hash']: + raise ValueError('Replay build identity changed during execution') + emit({'type': 'result', 'result': {**result, 'build_hash': build['hash'], + 'resources': {'version': 'waveform-process-resources-v1', + 'wallMs': (time.perf_counter_ns() - started_wall) / 1_000_000, + 'cpuMs': (time.process_time_ns() - started_cpu) / 1_000_000, + 'peakRssBytes': peak_rss_bytes(), + 'scope': 'python_process_until_result', + 'memoryScope': 'process_lifetime_high_water_mark'}}}) + + +def configure_process_limits(cpu_seconds=120): + if type(cpu_seconds) is not int or not 1 <= cpu_seconds <= 3600: + raise ValueError('Invalid waveform CPU budget') + # Provisional execution caps; qualified scheduling/admission belongs to + # the parent. It must also enforce wall time and kill on cancellation. + try: + import resource + except ModuleNotFoundError as error: + if error.name != 'resource' or sys.platform != 'linux': + raise + # Yocto may omit Python's optional resource extension. Use the fixed + # system utility to limit this process before importing numerical code. + # Missing utility, refusal, or timeout remains a startup failure. + subprocess.run(['/usr/bin/prlimit', '--pid', str(os.getpid()), + '--core=0:0', f'--cpu={cpu_seconds}:{cpu_seconds + 1}', '--nofile=64:64'], + check=True, timeout=5, stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + else: + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + resource.setrlimit(resource.RLIMIT_CPU, (cpu_seconds, cpu_seconds + 1)) + resource.setrlimit(resource.RLIMIT_NOFILE, (64, 64)) + os.nice(19) + for key in ('OPENBLAS_NUM_THREADS', 'OMP_NUM_THREADS', 'MKL_NUM_THREADS', 'NUMEXPR_NUM_THREADS'): + os.environ[key] = '1' + + +def main(): + parser = argparse.ArgumentParser(description='Replay fixed retained waveform inputs') + parser.add_argument('--cpu-seconds', type=int, default=120) + target = parser.add_mutually_exclusive_group(required=True) + target.add_argument('--retained-directory') + target.add_argument('--describe-build', action='store_true') + args = parser.parse_args() + lock_fd = None + try: + if not args.describe_build: + lock_fd = acquire_execution_lock() + configure_process_limits(args.cpu_seconds) + if args.describe_build: + from .build_identity import replay_build_identity + sys.stdout.write(json.dumps(replay_build_identity(), allow_nan=False, separators=(',', ':')) + '\n') + else: + run(sys.stdin.buffer, sys.stdout.buffer, args.retained_directory) + return 0 + except Exception: + # Never echo reference exports, frame values, paths or tracebacks. + sys.stderr.write('Retained waveform worker failed\n') + return 1 + finally: + if lock_fd is not None: + os.close(lock_fd) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/modules/common/biometrics/retained_replay.py b/modules/common/biometrics/retained_replay.py new file mode 100644 index 000000000..1286d3fab --- /dev/null +++ b/modules/common/biometrics/retained_replay.py @@ -0,0 +1,163 @@ +"""Replay immutable artifact descriptors with a frozen context timeline. + +Sink rows are provisional until this function returns successfully: a later +archive can fail verification. The durable worker must publish them atomically +with its successful result, never into canonical observations. +""" +from dataclasses import dataclass +import math + +from .raw_artifacts import RawArtifact +from .raw_records import RawRecordBudget, iter_retained_records +from .replay import RecordedContext, WaveformReplay + +MAX_SAFE_INTEGER = 9_007_199_254_740_991 + + +def _integer(value): + if type(value) is not int or not 0 <= value <= MAX_SAFE_INTEGER: + raise ValueError('Expected a nonnegative safe integer') + return value + + +def _keys(value, expected): + if not isinstance(value, dict) or set(value) != set(expected.split()): + raise ValueError('Unexpected retained replay fields') + + +def _timestamp(record): + value = record.get('ts') + if type(value) not in (int, float) or not math.isfinite(value) or not 0 <= value <= 9_007_199_254_740: + raise ValueError('Invalid retained event timestamp') + # Same rounding contract as the inspected inventory; no local timezone or + # Python bankers-rounding dependency at half-millisecond boundaries. + return _integer(math.floor(value * 1000 + 0.5)) + + +@dataclass(frozen=True) +class ArtifactInput: + artifact: RawArtifact + first_event_ms: int + last_event_ms: int + + +@dataclass(frozen=True) +class ContextInterval: + start_ms: int + end_ms: int + context: RecordedContext + + +@dataclass(frozen=True) +class RetainedReplayInput: + retained_start_ms: int + score_start_ms: int + score_end_ms: int + artifacts: tuple + contexts: tuple + + @classmethod + def parse(cls, raw): + _keys(raw, 'version retained_start_ms score_start_ms score_end_ms artifacts contexts') + if raw['version'] != 'retained-waveform-input-v1': + raise ValueError('Unsupported retained waveform input version') + start, score, end = (_integer(raw[key]) for key in ('retained_start_ms', 'score_start_ms', 'score_end_ms')) + if not start <= score < end: + raise ValueError('Invalid retained replay window') + if not isinstance(raw['artifacts'], list) or not 1 <= len(raw['artifacts']) <= 64: + raise ValueError('Select between one and 64 retained artifacts') + artifacts, hashes, compressed = [], set(), 0 + for item in raw['artifacts']: + _keys(item, 'artifact first_event_ms last_event_ms') + _keys(item['artifact'], 'name sha256 compressed_bytes expanded_bytes') + artifact = RawArtifact(**item['artifact']) + first, last = _integer(item['first_event_ms']), _integer(item['last_event_ms']) + if first > last or last < start or first >= end: + raise ValueError('Artifact has no events in the retained window') + if artifact.sha256 in hashes or (artifacts and first <= artifacts[-1].last_event_ms): + raise ValueError('Retained artifacts must be distinct and have nonoverlapping ordered event bounds') + hashes.add(artifact.sha256) + compressed += artifact.compressed_bytes + if compressed > 1024 * 1024 * 1024: + raise ValueError('Retained replay compressed input budget exceeded') + artifacts.append(ArtifactInput(artifact, first, last)) + if not isinstance(raw['contexts'], list) or len(raw['contexts']) > 20000: + raise ValueError('Retained context interval budget exceeded') + contexts = [] + for item in raw['contexts']: + _keys(item, 'start_ms end_ms active_sides left_occupied right_occupied pump_context_known') + first, last = _integer(item['start_ms']), _integer(item['end_ms']) + if not start <= first < last <= end or (contexts and first < contexts[-1].end_ms): + raise ValueError('Context intervals must be ordered and nonoverlapping inside the retained window') + active = item['active_sides'] + if active is not None and not isinstance(active, list): + raise ValueError('Recorded active sides must be a list or unknown') + context = RecordedContext(None if active is None else tuple(active), + item['left_occupied'], item['right_occupied'], item['pump_context_known']) + contexts.append(ContextInterval(first, last, context)) + return cls(start, score, end, tuple(artifacts), tuple(contexts)) + + +def replay_retained(store, raw_input, configs, sink, *, cancelled=lambda: False, + maximum_readings=20000, maximum_records=1_000_000): + """Run fixed shipped code; no archive fallback, live DB, or control access. + + Cross-artifact overlap is rejected rather than guessing which duplicate + frames to drop. Gaps are preserved; event bounds never imply continuity. + Every selected artifact is consumed to EOF, including out-of-window records, + so decompression/framing and the inspected event bounds are verified. + """ + if type(maximum_readings) is not int or maximum_readings <= 0 or type(maximum_records) is not int or maximum_records <= 0: + raise ValueError('Invalid retained replay resource bounds') + snapshot = RetainedReplayInput.parse(raw_input) + readings = 0 + + def write(row): + nonlocal readings + if cancelled(): + raise ValueError('Retained replay cancelled') + readings += 1 + if readings > maximum_readings: + raise ValueError('Retained replay output budget exceeded') + sink(row) + + replay = WaveformReplay(configs, retained_start_ms=snapshot.retained_start_ms, + score_start_ms=snapshot.score_start_ms, score_end_ms=snapshot.score_end_ms, sink=write) + budget = RawRecordBudget(maximum_records) + index, decoded = 0, 0 + previous = None + for source in snapshot.artifacts: + if cancelled(): + raise ValueError('Retained replay cancelled') + first = last = None + for record in iter_retained_records(store, source.artifact, max_records=maximum_records, budget=budget): + if cancelled(): + raise ValueError('Retained replay cancelled') + decoded += 1 + if decoded > maximum_records: + raise ValueError('Retained replay decoded record budget exceeded') + if record['type'] not in ('piezo-dual', 'frzHealth'): + continue + timestamp = _timestamp(record) + if previous is not None and timestamp < previous: + raise ValueError('Retained event timestamps reversed') + previous = timestamp + first = timestamp if first is None else first + last = timestamp + if not source.first_event_ms <= timestamp <= source.last_event_ms: + raise ValueError('Retained event lies outside inspected bounds') + if not snapshot.retained_start_ms <= timestamp < snapshot.score_end_ms: + continue + while index < len(snapshot.contexts) and timestamp >= snapshot.contexts[index].end_ms: + index += 1 + interval = snapshot.contexts[index] if index < len(snapshot.contexts) else None + context = interval.context if interval and timestamp >= interval.start_ms else RecordedContext() + replay.feed(timestamp, record, context) + if first != source.first_event_ms or last != source.last_event_ms: + raise ValueError('Retained event bounds differ from inspected source') + if cancelled(): + raise ValueError('Retained replay cancelled') + replay.close() + return {'version': 'retained-waveform-run-v1', 'decoded_records': decoded, 'raw_records': budget.consumed, + 'counts': dict(replay.counts), 'limitations': sorted(replay.limitations), + 'config_hashes': dict(replay.hashes), 'artifact_hashes': [item.artifact.sha256 for item in snapshot.artifacts]} diff --git a/modules/common/biometrics/routing.py b/modules/common/biometrics/routing.py new file mode 100644 index 000000000..2e1d5a628 --- /dev/null +++ b/modules/common/biometrics/routing.py @@ -0,0 +1,102 @@ +"""Single-sleeper arbitration shared by live processing and replay. + +The caller supplies recorded/current mode, clock, and output sink. Candidates +are opaque except for physical side and quality. This module cannot read +current occupancy, query a database, or select a canonical output destination. +""" +from typing import Callable, Generic, Optional, Protocol, Sequence, TypeVar + +VITALS_INTERVAL_S = 60 + + +class Candidate(Protocol): + side: str + quality_score: float + + +class SleeperMode(Protocol): + def home_side(self) -> Optional[str]: ... + def active_sides(self) -> Sequence[str]: ... + + +CandidateT = TypeVar("CandidateT", bound=Candidate) + + +class SingleSleeperRouter(Generic[CandidateT]): + """Routes vitals rows when one side is in away mode (single sleeper). + + A solo sleeper who rolls onto the away side is picked up by that side's + piezo too — the same heartbeat, within a few bpm. Each cycle's + candidates from the two sides are paired and + only the higher-quality one is written, under the home side, so the + home session shows one heart-rate series with no gaps or duplicates. + A candidate whose partner doesn't arrive within VITALS_INTERVAL_S (the + other side saw no one) is written alone under the home side. + + Outside single-sleeper mode rows are written under their own side. + """ + + def __init__(self, mode: SleeperMode, writer: Callable[[CandidateT, Optional[str]], bool], + *, clock: Callable[[], float]): + self._writer = writer + self._mode = mode + self._clock = clock + self._pending: Optional[CandidateT] = None + self._retry: Optional[CandidateT] = None + self._retry_home: Optional[str] = None + self._pending_home: Optional[str] = None + self._pending_at = 0.0 + + def submit(self, cand: CandidateT) -> bool: + """Accept one side's candidate. True means the side may advance its + write cursor: the row was written, or is held (for pairing or a + write retry). A held candidate is only dropped once written.""" + # Completed intervals must never pair with a later candidate. + if self._retry is not None and not self.flush(): + return False + home = self._mode.home_side() + if not self._mode.active_sides(): + self._pending = self._retry = None + return True + if home is None: + if not self.flush(): + return False + return self._writer(cand, None) + pending = self._pending + if pending is not None and pending.side != cand.side and self._pending_home == home: + best = cand if cand.quality_score > pending.quality_score else pending + self._pending = best + self.flush() + return True + # A same-side repeat means its partner never arrived: write the + # older one first. If that fails, refuse the new candidate so its + # side keeps it and resubmits, rather than dropping the older row. + if not self.flush(): + return False + self._pending, self._pending_home, self._pending_at = cand, home, self._clock() + return True + + def tick(self) -> None: + """Write a held candidate whose partner didn't arrive in time.""" + if not self._mode.active_sides(): + self._pending = self._retry = None + return + held = self._pending is not None or self._retry is not None + if held and self._clock() - self._pending_at >= VITALS_INTERVAL_S: + self.flush() + + def flush(self) -> bool: + """Write the held candidate, if any. On failure it stays held and + idle retries wait VITALS_INTERVAL_S; a new submission retries first. + True when nothing is left held.""" + if self._retry is None: + self._retry, self._retry_home = self._pending, self._pending_home + self._pending = None + if self._retry is None: + return True + if self._writer(self._retry, self._retry_home): + self._retry = None + return True + self._pending_at = self._clock() + return False + diff --git a/modules/common/biometrics/runtime.py b/modules/common/biometrics/runtime.py new file mode 100644 index 000000000..8d1e59744 --- /dev/null +++ b/modules/common/biometrics/runtime.py @@ -0,0 +1,139 @@ +"""Validated runtime projections and sample-boundary selection. + +This module never changes desired bindings or fabricates applied receipts. +Callers install a selected snapshot in their estimator before acknowledging it. +Unknown occupancy is deliberately not an empty-bed boundary. +""" +from dataclasses import dataclass +import json +from uuid import UUID + +from .config import canonical_json, config_hash, manifest, validate_config + +_MAX_PAYLOAD_BYTES = 65536 +_FIELDS = {"schemaVersion", "side", "generation", "revisionId", "manifestHash", + "resolved", "resolvedHash", "boundary"} + + +def _unique_object(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError("Duplicate runtime payload field") + result[key] = value + return result + + +@dataclass(frozen=True) +class RuntimeSnapshot: + """Deep immutability: mutable configuration is decoded afresh on access.""" + side: str + generation: int + resolved_hash: str + manifest_hash: str + boundary: str + revision_id: object + _resolved_json: str + + @property + def resolved(self): + return json.loads(self._resolved_json) + + @classmethod + def parse(cls, payload): + if not isinstance(payload, str) or len(payload.encode("utf-8")) > _MAX_PAYLOAD_BYTES: + raise ValueError("Runtime payload exceeds resource limit") + try: + value = json.loads(payload, object_pairs_hook=_unique_object) + except (RecursionError, json.JSONDecodeError) as error: + raise ValueError("Invalid runtime payload JSON") from error + if not isinstance(value, dict) or set(value) != _FIELDS: + raise ValueError("Invalid runtime payload fields") + if type(value["schemaVersion"]) is not int or value["schemaVersion"] != 1: + raise ValueError("Unsupported runtime schema") + if value["side"] not in ("left", "right"): + raise ValueError("Invalid runtime side") + generation = value["generation"] + if type(generation) is not int or not 0 < generation <= 9007199254740991: + raise ValueError("Invalid runtime generation") + if value["boundary"] not in ("immediate", "next_session"): + raise ValueError("Invalid runtime boundary") + revision = value["revisionId"] + if revision is not None: + if not isinstance(revision, str): + raise ValueError("Invalid runtime revision") + try: + if str(UUID(revision)) != revision.lower(): + raise ValueError("Invalid runtime revision") + except (ValueError, AttributeError) as error: + raise ValueError("Invalid runtime revision") from error + if value["manifestHash"] != config_hash(manifest()): + raise ValueError("PROFILE_INCOMPATIBLE: unknown runtime manifest") + resolved = validate_config(value["resolved"]) + if config_hash(resolved) != value["resolvedHash"]: + raise ValueError("Runtime configuration hash mismatch") + return cls(value["side"], generation, value["resolvedHash"], + value["manifestHash"], value["boundary"], revision, + canonical_json(resolved)) + + +def read_latest_snapshot(connection, side): + """Read one bounded projection. Missing schema is surfaced, never migrated. + + An unavailable/corrupt newest generation must not silently select an older + generation. The daemon decides its explicit fallback and reports the error. + """ + if side not in ("left", "right"): + raise ValueError("Invalid runtime side") + # Do not materialize an oversized SQLite TEXT into the daemon process. + row = connection.execute( + "SELECT side, generation, resolved_hash, boundary, " + "CASE WHEN length(CAST(payload AS BLOB)) <= ? THEN payload ELSE NULL END " + "FROM biometric_runtime_configs WHERE side = ? " + "ORDER BY generation DESC LIMIT 1", (_MAX_PAYLOAD_BYTES, side)).fetchone() + if row is None: + return None + snapshot = RuntimeSnapshot.parse(row[4]) + if (snapshot.side, snapshot.generation, snapshot.resolved_hash, snapshot.boundary) != tuple(row[:4]): + raise ValueError("Runtime projection envelope mismatch") + return snapshot + + +class BoundarySelection: + """Two-phase selection: no application claim until the consumer commits. + + Supply a previously applied snapshot only after restoring its estimator. + A restart with no restored snapshot and an occupied/unknown bed waits for + the next empty boundary for next-session requests. + """ + def __init__(self, side, applied=None): + if side not in ("left", "right") or (applied is not None and applied.side != side): + raise ValueError("Runtime side mismatch") + self.side = side + self.applied = applied + self.pending = None + + def offer(self, snapshot): + if snapshot.side != self.side: + raise ValueError("Runtime side mismatch") + current = self.pending or self.applied + if current is not None: + if snapshot.generation < current.generation: + raise ValueError("Runtime generation regression") + if snapshot.generation == current.generation: + if snapshot != current: + raise ValueError("Runtime generation conflict") + return + self.pending = snapshot + + def ready(self, occupied): + if self.pending is None: + return None + if self.pending.boundary == "immediate" or occupied is False: + return self.pending + return None + + def commit(self, snapshot): + if self.pending != snapshot: + raise ValueError("Runtime selection changed before adoption") + self.applied, self.pending = snapshot, None diff --git a/modules/common/biometrics/shadow.py b/modules/common/biometrics/shadow.py new file mode 100644 index 000000000..6ed1a5259 --- /dev/null +++ b/modules/common/biometrics/shadow.py @@ -0,0 +1,135 @@ +"""Prospective adapter for the shipped core, with an isolated candidate state. + +No database, hardware, code loading or source selection. A separately admitted +host supplies observed arrival times/context and connects the sink to shadow +storage. CPU checks are cooperative; the host must also enforce process limits. +""" +from copy import deepcopy +from dataclasses import dataclass +import hashlib +import time + +from .config import canonical_json +from .replay import MAX_CHANNEL_BYTES, RecordedContext, ReplayReading, WaveformReplay + + +@dataclass(frozen=True) +class ShadowOutput: + sequence: int + input_hash: str + reading: ReplayReading + + +class ProspectiveShadow: + def __init__(self, configs, *, side, start_ms, end_ms, maximum_records, + maximum_outputs, maximum_cpu_ms, sink, cpu_clock=time.process_time_ns): + if side not in ('left', 'right'): + raise ValueError('Invalid shadow side') + for value in (maximum_records, maximum_outputs, maximum_cpu_ms): + if type(value) is not int or not 0 < value <= 9_007_199_254_740_991: + raise ValueError('Invalid shadow resource budget') + if type(start_ms) is not int or type(end_ms) is not int or not 0 <= start_ms < end_ms <= 9_007_199_254_740_991 or end_ms - start_ms > 7 * 86400000: + raise ValueError('Invalid prospective window') + self.side, self.start_ms, self.end_ms = side, start_ms, end_ms + self.maximum_records, self.maximum_outputs = maximum_records, maximum_outputs + self.maximum_cpu_ns = maximum_cpu_ms * 1_000_000 + self._cpu_clock, self._sink = cpu_clock, sink + self._cpu_ns, self._records, self._outputs = 0, 0, 0 + self._staged = [] + self._digest = hashlib.sha256(b'biometric-shadow-input-v1\n').digest() + self.state, self.reason = 'running', None + self._processor = WaveformReplay(deepcopy(configs), retained_start_ms=start_ms, + score_start_ms=start_ms, score_end_ms=end_ms, sink=self._capture, + clock_mode='observed_arrival_time') + + def _capture(self, reading): + if reading.side != self.side: + return + if len(self._staged) >= 4 or self._outputs + len(self._staged) >= self.maximum_outputs: + raise ValueError('Shadow output budget exceeded') + self._staged.append(reading) + + def _read_cpu(self): + value = self._cpu_clock() + if type(value) is not int or value < 0: + raise ValueError('Invalid shadow CPU clock') + return value + + def _execute(self, action): + if self.state != 'running': + raise ValueError('Shadow processor is stopped') + self._staged = [] + try: + before = self._read_cpu() + if self._cpu_ns >= self.maximum_cpu_ns: + raise ValueError('Shadow CPU budget exceeded') + try: + action() + finally: + # Failed numerical/validation calls still consumed CPU. Do not + # report a zero-cost failure or permit reuse after clock failure. + elapsed = self._read_cpu() - before + if elapsed < 0: + raise ValueError('Shadow CPU clock regressed') + self._cpu_ns += elapsed + if self._cpu_ns > self.maximum_cpu_ns: + raise ValueError('Shadow CPU budget exceeded') + # Do not publish this frame's provisional output after a budget or + # processing failure. A sink failure terminates further processing. + for reading in self._staged: + self._sink(ShadowOutput(self._outputs, self._digest.hex(), reading)) + self._outputs += 1 + except Exception as error: + self.state, self.reason = 'failed', str(error)[:500] + raise + finally: + self._staged = [] + + def feed(self, observed_at_ms, record, context): + """Observed arrival time matches the live core's clock semantics. + Preserve source event time in record metadata, but never use it as the + live processing clock. Both channels/context feed the isolated state. + """ + def action(): + if self._records >= self.maximum_records: + raise ValueError('Shadow record budget exceeded') + if not isinstance(record, dict) or record.get('type') not in ('piezo-dual', 'frzHealth') or not isinstance(context, RecordedContext): + raise ValueError('Invalid shadow record or context') + for channel in ('left1', 'right1'): + if channel in record: + value = record[channel] + if not isinstance(value, (bytes, bytearray, memoryview)) or len(value) > MAX_CHANNEL_BYTES: + raise ValueError('Invalid bounded shadow channel') + frozen = {key: bytes(value) if key in ('left1', 'right1') else deepcopy(value) for key, value in record.items()} + identity = dict(frozen) + for channel in ('left1', 'right1'): + if channel in frozen: + value = frozen[channel] + if not isinstance(value, (bytes, bytearray, memoryview)) or len(value) > MAX_CHANNEL_BYTES: + raise ValueError('Invalid bounded shadow channel') + frozen[channel] = bytes(value) + identity[channel] = {'bytes': len(value), 'sha256': hashlib.sha256(value).hexdigest()} + encoded = canonical_json({'observedAtMs': observed_at_ms, 'record': identity, + 'context': {'activeSides': list(context.active_sides) if context.active_sides is not None else None, + 'leftOccupied': context.left_occupied, 'rightOccupied': context.right_occupied, + 'pumpContextKnown': context.pump_context_known}}).encode() + self._digest = hashlib.sha256(self._digest + encoded).digest() + self._processor.feed(observed_at_ms, frozen, context) + self._records += 1 + self._execute(action) + + def finish(self): + """Normal completion flushes arbitration once. Cancellation does not.""" + self._execute(self._processor.close) + self.state, self.reason = 'completed', 'window_complete' + + def stop(self, reason='cancelled'): + if self.state == 'running': + self.state, self.reason = 'stopped', str(reason)[:500] + self._staged = [] + + def health(self): + return {'state': self.state, 'reason': self.reason, 'records': self._records, + 'outputs': self._outputs, 'cpu_ms': self._cpu_ns / 1_000_000, + 'input_hash': self._digest.hex(), 'counts': dict(self._processor.counts), + 'limitations': tuple(sorted(self._processor.limitations))} diff --git a/modules/common/biometrics/shadow_build.py b/modules/common/biometrics/shadow_build.py new file mode 100644 index 000000000..16d901efd --- /dev/null +++ b/modules/common/biometrics/shadow_build.py @@ -0,0 +1,55 @@ +"""Read-only identity probe for the shipped prospective processor. + +Includes shared replay dependencies and shadow-specific control/transport code. +Wheel metadata hashes are provenance, not full installed-binary attestation. +""" +import json +import importlib.metadata +from pathlib import Path +import sys + +from . import build_identity +from .config import config_hash + +SHADOW_SOURCE_FILES = ( + 'common/biometrics/shadow_build.py', + 'common/biometrics/shadow.py', + 'common/biometrics/shadow_worker.py', + 'common/biometrics/shadow_process.py', +) + + +def shadow_build_identity(): + identity = build_identity.replay_build_identity()['identity'] + identity['version'] = 'shadow-python-build-v1' + root = Path(__file__).resolve().parents[2] + identity['sources'].update({name: build_identity._digest_file(root / name) + for name in SHADOW_SOURCE_FILES}) + dependency_files = {} + for name in ('numpy', 'scipy'): + distribution = importlib.metadata.distribution(name) + files = distribution.files or [] + paths = {} + for key, filename in (('record', 'RECORD'), ('wheel', 'WHEEL')): + matches = [item for item in files if item.name == filename + and item.parent.name.endswith('.dist-info')] + if len(matches) != 1: + raise ValueError('Shadow dependency metadata paths unavailable') + paths[key] = str(Path(distribution.locate_file(matches[0])).absolute()) + dependency_files[name] = paths + return {'hash': config_hash(identity), 'identity': identity, + 'dependencyFiles': dependency_files} + + +def main(): + try: + sys.stdout.write(json.dumps(shadow_build_identity(), allow_nan=False, + separators=(',', ':')) + '\n') + return 0 + except Exception: + sys.stderr.write('Shadow build identity unavailable\n') + return 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/modules/common/biometrics/shadow_process.py b/modules/common/biometrics/shadow_process.py new file mode 100644 index 000000000..324a144cf --- /dev/null +++ b/modules/common/biometrics/shadow_process.py @@ -0,0 +1,106 @@ +"""Linux process entrypoint for the trusted, shipped shadow algorithm. + +Install kernel limits before importing the numerical processor. This is resource +containment, not an isolation boundary for uploaded code. The parent must still +bound wall/idle time, sample RSS, and reap the process on cancellation or loss of +its lease. Address space is not RSS and must be budgeted separately. +""" +import argparse +import os +import signal +import subprocess +import sys + +from .replay_worker import configure_process_limits +from .execution_lock import EXECUTION_LOCK_PATH as SHADOW_LOCK_PATH, acquire_execution_lock + + +def configure_parent_death(expected_parent_pid): + """Kill this worker if its launching host dies, including the arm-time race. + + Linux PR_SET_PDEATHSIG applies to this process, not future forked children: + https://man7.org/linux/man-pages/man2/PR_SET_PDEATHSIG.2const.html + """ + if sys.platform != 'linux': + raise RuntimeError('Shadow parent-death protection requires Linux') + if type(expected_parent_pid) is not int or not 1 <= expected_parent_pid <= 2_147_483_647: + raise ValueError('Invalid shadow parent PID') + if os.getppid() != expected_parent_pid: + raise RuntimeError('Shadow host is no longer the parent') + import ctypes + libc = ctypes.CDLL(None, use_errno=True) + libc.prctl.argtypes = [ctypes.c_int, ctypes.c_ulong, ctypes.c_ulong, ctypes.c_ulong, ctypes.c_ulong] + libc.prctl.restype = ctypes.c_int + if libc.prctl(1, signal.SIGKILL, 0, 0, 0) != 0: # PR_SET_PDEATHSIG + raise OSError(ctypes.get_errno(), 'Shadow parent-death protection unavailable') + # No signal is delivered if the parent died before prctl was armed. + if os.getppid() != expected_parent_pid: + raise RuntimeError('Shadow host changed while arming parent-death protection') + + +def acquire_shadow_lock(path=SHADOW_LOCK_PATH): + if sys.platform != 'linux': + raise RuntimeError('Shadow execution lock requires Linux') + return acquire_execution_lock(path) + + +def configure_wall_deadline(wall_ms): + """Worker-local elapsed-time limit, independent of the Node event loop.""" + if sys.platform != 'linux': + raise RuntimeError('Shadow wall deadline requires Linux') + if type(wall_ms) is not int or not 0 < wall_ms <= 7 * 86_400_000 + 300_000: + raise ValueError('Invalid shadow wall budget') + signal.signal(signal.SIGALRM, signal.SIG_DFL) + signal.pthread_sigmask(signal.SIG_UNBLOCK, {signal.SIGALRM}) + signal.setitimer(signal.ITIMER_REAL, wall_ms / 1000) + + +def configure_shadow_limits(cpu_seconds, address_space_bytes): + if sys.platform != 'linux': + raise RuntimeError('Shadow process limits require Linux') + if type(address_space_bytes) is not int or not 0 < address_space_bytes <= 9_007_199_254_740_991: + raise ValueError('Invalid shadow address-space budget') + # Reuse the replay worker's CPU/core/file limits, low priority, and BLAS + # thread settings. A failure must prevent all numerical imports and input. + configure_process_limits(cpu_seconds) + try: + import resource + except ModuleNotFoundError as error: + if error.name != 'resource': + raise + # Yocto builds can omit the optional Python extension. + subprocess.run(['/usr/bin/prlimit', '--pid', str(os.getpid()), + f'--as={address_space_bytes}:{address_space_bytes}'], + check=True, timeout=5, stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + else: + resource.setrlimit(resource.RLIMIT_AS, (address_space_bytes, address_space_bytes)) + + +def main(argv=None): + parser = argparse.ArgumentParser(description='Run a resource-limited shipped shadow processor') + parser.add_argument('--parent-pid', type=int, required=True) + parser.add_argument('--wall-ms', type=int, required=True) + parser.add_argument('--cpu-seconds', type=int, required=True) + parser.add_argument('--address-space-bytes', type=int, required=True) + args = parser.parse_args(argv) + lock_fd = None + try: + configure_parent_death(args.parent_pid) + lock_fd = acquire_shadow_lock() + configure_wall_deadline(args.wall_ms) + configure_shadow_limits(args.cpu_seconds, args.address_space_bytes) + from .shadow_worker import run + run(sys.stdin.buffer, sys.stdout.buffer) + return 0 + except Exception: + # No frame values, paths, reference data or tracebacks in host logs. + sys.stderr.write('Shadow process failed\n') + return 1 + finally: + if lock_fd is not None: + os.close(lock_fd) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/modules/common/biometrics/shadow_worker.py b/modules/common/biometrics/shadow_worker.py new file mode 100644 index 000000000..aa566ca6f --- /dev/null +++ b/modules/common/biometrics/shadow_worker.py @@ -0,0 +1,132 @@ +"""Bounded JSONL interface for an externally supervised shipped shadow worker. + +stdin carries configuration and frames; stdout carries acknowledgments and +candidate-only readings. No DB, live source, network or hardware is opened. +The parent must impose OS process budgets and terminate idle/cancelled workers. +""" +import base64 +import json +import sys +from uuid import UUID + +from .config import config_hash +from .replay import MAX_CHANNEL_BYTES, RecordedContext +from .shadow import ProspectiveShadow +from .processing import HRV_WINDOW_S + +MAX_LINE_BYTES = 128 * 1024 + + +def positive(value): + if type(value) is not int or not 0 < value <= 9_007_199_254_740_991: + raise ValueError('Invalid transport budget') + return value + + +def exact(value, keys): + if not isinstance(value, dict) or set(value) != set(keys.split()): + raise ValueError('Unexpected shadow protocol fields') + + +def run(source, destination): + input_bytes = output_bytes = 0 + maximum_input = MAX_LINE_BYTES + maximum_output = MAX_LINE_BYTES + + def read(): + nonlocal input_bytes + line = source.readline(MAX_LINE_BYTES + 1) + if not line: + raise ValueError('Unexpected shadow protocol EOF') + input_bytes += len(line) + if len(line) > MAX_LINE_BYTES or input_bytes > maximum_input or not line.endswith(b'\n'): + raise ValueError('Shadow input transport limit exceeded') + return json.loads(line.decode('utf-8')) + + init = read() + exact(init, 'schema runId attemptId configs configHashes side startMs endMs maximumRecords maximumOutputs maximumCpuMs maximumInputBytes maximumOutputBytes') + if init['schema'] != 'biometric-shadow-worker-v1': + raise ValueError('Unsupported shadow worker protocol') + for key in ('runId', 'attemptId'): + if not isinstance(init[key], str) or str(UUID(init[key])) != init[key]: + raise ValueError('Invalid shadow attempt identity') + maximum_input, maximum_output = positive(init['maximumInputBytes']), positive(init['maximumOutputBytes']) + if input_bytes > maximum_input: + raise ValueError('Shadow input transport limit exceeded') + if not isinstance(init['configs'], dict) or set(init['configs']) != {'left', 'right'}: + raise ValueError('Both shadow configurations required') + hashes = {side: config_hash(config) for side, config in init['configs'].items()} + if hashes != init['configHashes']: + raise ValueError('Shadow configuration identity mismatch') + + def write(payload): + nonlocal output_bytes + response = {'runId': init['runId'], 'attemptId': init['attemptId'], **payload} + line = (json.dumps(response, allow_nan=False, separators=(',', ':')) + '\n').encode() + output_bytes += len(line) + if len(line) > MAX_LINE_BYTES or output_bytes > maximum_output: + raise ValueError('Shadow output transport limit exceeded') + destination.write(line) + destination.flush() + + pending = [] + def sink(output): + candidate = output.reading.candidate + pending.append({'sequence': output.sequence, 'inputHash': output.input_hash, + 'side': output.reading.side, 'sourceSide': candidate.run_context.source_side, + 'timestampMs': output.reading.timestamp_ms, 'configHash': candidate.run_context.config_hash, + 'readings': [ + {'metric': 'heart_rate', 'unit': 'bpm', 'windowMs': None, 'value': candidate.heart_rate, 'quality': candidate.quality_score}, + {'metric': 'breathing_rate', 'unit': 'breaths/min', 'windowMs': None, 'value': candidate.breathing_rate, 'quality': candidate.quality_score}, + {'metric': 'window_ibi_variability_v1', 'unit': 'ms', 'windowMs': HRV_WINDOW_S * 1000, 'value': candidate.hrv, 'quality': candidate.quality_score}, + ], 'flags': candidate.flags, 'limitations': output.reading.limitations}) + + processor = ProspectiveShadow(init['configs'], side=init['side'], start_ms=init['startMs'], end_ms=init['endMs'], + maximum_records=init['maximumRecords'], maximum_outputs=init['maximumOutputs'], maximum_cpu_ms=init['maximumCpuMs'], sink=sink) + write({'type': 'ready', 'configHashes': hashes}) + expected_sequence = 0 + try: + while True: + message = read() + if message == {'type': 'finish'}: + processor.finish() + write({'type': 'finished', 'outputs': pending, 'health': processor.health()}) + return + if message == {'type': 'cancel'}: + processor.stop() + write({'type': 'cancelled', 'health': processor.health()}) + return + exact(message, 'type sequence observedAtMs record context') + if message['type'] != 'frame' or type(message['sequence']) is not int or message['sequence'] != expected_sequence: + raise ValueError('Out-of-order shadow protocol frame') + exact(message['context'], 'activeSides leftOccupied rightOccupied pumpContextKnown') + context = message['context'] + if context['activeSides'] is not None and not isinstance(context['activeSides'], list): + raise ValueError('Invalid shadow active sides') + recorded = RecordedContext(tuple(context['activeSides']) if context['activeSides'] is not None else None, + context['leftOccupied'], context['rightOccupied'], context['pumpContextKnown']) + if not isinstance(message['record'], dict): + raise ValueError('Invalid shadow record') + record = dict(message['record']) + for channel in ('left1', 'right1'): + if channel in record: + encoded = record[channel] + if not isinstance(encoded, str) or len(encoded) > ((MAX_CHANNEL_BYTES + 2) // 3) * 4: + raise ValueError('Invalid encoded shadow channel') + record[channel] = base64.b64decode(encoded, validate=True) + if len(record[channel]) > MAX_CHANNEL_BYTES: + raise ValueError('Shadow channel limit exceeded') + processor.feed(message['observedAtMs'], record, recorded) + write({'type': 'frame_result', 'sequence': expected_sequence, 'outputs': pending, 'health': processor.health()}) + pending.clear() + expected_sequence += 1 + finally: + processor.stop('transport_closed') + + +if __name__ == '__main__': + try: + run(sys.stdin.buffer, sys.stdout.buffer) + except Exception as error: + sys.stderr.write(str(error)[:500] + '\n') + sys.exit(1) diff --git a/modules/common/biometrics/signal.py b/modules/common/biometrics/signal.py new file mode 100644 index 000000000..dface3b08 --- /dev/null +++ b/modules/common/biometrics/signal.py @@ -0,0 +1,498 @@ +"""Shared numerical core for live processing and deterministic replay. + +No database, sensor transport, hardware or wall-clock access. Callers own +occupancy, pump gating, window boundaries, state reset, timestamps and sinks. +Defaults were captured against dev 3e9027ed before extraction. +""" +from collections import deque +from dataclasses import dataclass +import logging +from typing import Optional + +import numpy as np +from scipy.signal import butter, sosfiltfilt, hilbert, find_peaks + +from .config import manifest, resolve_config, validate_config + +SAMPLE_RATE = 500 +HR_BAND = (0.8, 8.5) +log = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class ProcessingParameters: + tracking_tolerance_bpm: float + tracking_history_length: int + breathing_interval_lower_ratio: float + breathing_interval_upper_ratio: float + hrv_harmonic_tolerance: float + hrv_minimum_windows: int + + def __post_init__(self): + resolve_config([ + {"path": "heart_rate.trackingToleranceBpm", "value": self.tracking_tolerance_bpm}, + {"path": "heart_rate.trackingHistoryLength", "value": self.tracking_history_length}, + {"path": "breathing_rate.intervalLowerRatio", "value": self.breathing_interval_lower_ratio}, + {"path": "breathing_rate.intervalUpperRatio", "value": self.breathing_interval_upper_ratio}, + {"path": "hrv_index.harmonicTolerance", "value": self.hrv_harmonic_tolerance}, + {"path": "hrv_index.minimumWindows", "value": self.hrv_minimum_windows}, + ]) + + @classmethod + def from_config(cls, config): + validate_config(config) + return cls(config["heart_rate"]["trackingToleranceBpm"], + config["heart_rate"]["trackingHistoryLength"], + config["breathing_rate"]["intervalLowerRatio"], + config["breathing_rate"]["intervalUpperRatio"], + config["hrv_index"]["harmonicTolerance"], + config["hrv_index"]["minimumWindows"]) + + +DEFAULT_PARAMETERS = ProcessingParameters.from_config(manifest()["defaults"]) + + +def _bandpass(sig: np.ndarray, lo: float, hi: float, fs: float, + order: int = 4) -> np.ndarray: + """Bandpass filter using SOS form for numerical stability.""" + sos = butter(order, [lo, hi], btype='band', fs=fs, output='sos') + return sosfiltfilt(sos, sig) + + + +def _autocorr_quality(sig: np.ndarray, fs: float = SAMPLE_RATE) -> float: + """Peak height of best autocorrelation peak in cardiac range. + + Person on bed produces periodic signal with high peak. + Empty bed produces noise with low peak. + """ + try: + filtered = _bandpass(sig, 0.8, 8.5, fs) + seg = filtered[:int(min(len(filtered), 10 * fs))] + seg = seg - np.mean(seg) + norm = np.std(seg) + if norm < 1e-10: + return 0.0 + seg = seg / norm + + n = len(seg) + fft_size = 1 + while fft_size < 2 * n: + fft_size *= 2 + fft_f = np.fft.rfft(seg, n=fft_size) + acr = np.fft.irfft(fft_f * np.conj(fft_f), n=fft_size)[:n] + if acr[0] == 0: + return 0.0 + acr = acr / acr[0] + + min_lag = int(fs * 60 / 150) + max_lag = min(int(fs * 60 / 40), len(acr) - 1) + if max_lag <= min_lag: + return 0.0 + return float(max(np.max(acr[min_lag:max_lag + 1]), 0.0)) + except Exception: + return 0.0 + + + + +def _compute_autocorr(filtered: np.ndarray, fs: float) -> Optional[np.ndarray]: + """Compute normalised autocorrelation via FFT.""" + n = len(filtered) + fft_size = 1 + while fft_size < 2 * n: + fft_size *= 2 + fft_f = np.fft.rfft(filtered, n=fft_size) + acr = np.fft.irfft(fft_f * np.conj(fft_f), n=fft_size)[:n] + if acr[0] == 0: + return None + return acr / acr[0] + + +def subharmonic_summation_hr(samples: np.ndarray, fs: float = SAMPLE_RATE, + bpm_range: tuple = (45, 120), + n_harmonics: int = 3): + """Subharmonic summation for robust fundamental HR detection. + + Only scores autocorrelation PEAKS (not every lag) to avoid edge effects + where long lags accumulate noise. For each candidate peak lag L, + score = weighted sum of autocorr at L, L/2, L/3. + + Hermes 1988 (JASA), adapted for BCG by Bruser et al. 2011. + + Returns (heart_rate_bpm, score) or (None, 0.0) on failure. + """ + filtered = _bandpass(samples, HR_BAND[0], HR_BAND[1], fs) + acr = _compute_autocorr(filtered, fs) + if acr is None: + return None, 0.0 + + min_lag = int(fs * 60 / bpm_range[1]) + max_lag = int(fs * 60 / bpm_range[0]) + max_lag = min(max_lag, len(acr) - 1) + + if max_lag <= min_lag: + return None, 0.0 + + # Find actual peaks in autocorrelation (not every lag) + search = acr[min_lag:max_lag + 1] + peaks, _props = find_peaks(search, height=0.02, distance=int(fs * 0.15)) + if len(peaks) == 0: + return None, 0.0 + + candidate_lags = peaks + min_lag + weights = [1.0, 0.8, 0.6][:n_harmonics] + + best_lag = None + best_score = 0.0 + + for lag in candidate_lags: + score = 0.0 + for k in range(n_harmonics): + exact_lag = lag / (k + 1) + lag_lo = int(exact_lag) + lag_hi = lag_lo + 1 + if lag_hi >= len(acr): + continue + frac = exact_lag - lag_lo + val = acr[lag_lo] * (1 - frac) + acr[lag_hi] * frac + score += weights[k] * max(val, 0) + if score > best_score: + best_score = score + best_lag = lag + + if best_lag is None or best_score < 0.1: + return None, best_score + + hr = 60.0 * fs / best_lag + return hr, best_score + + +class HRTracker: + """Inter-window HR tracking with Gaussian consistency constraint. + + HR cannot jump > max_delta BPM between consecutive windows at rest. + Catches residual harmonic escapes via half/double correction. + Bruser et al. 2011. + """ + + # After this many consecutive uncorrectable readings, re-seed history at + # the new level. A lone outlier must not poison the median, but a + # sustained shift (e.g. different occupant) previously froze the tracker: + # the stale median rejected every new reading forever. + FALLBACK_RESET_STREAK = 3 + + def __init__(self, max_delta: float = DEFAULT_PARAMETERS.tracking_tolerance_bpm, + history_len: int = DEFAULT_PARAMETERS.tracking_history_length): + # Bounded deque: only the last history_len entries are ever consulted, + # so retaining the full list would leak ~1440 floats/day (#325). + self.history: deque = deque(maxlen=history_len) + self.max_delta = max_delta + self.history_len = history_len + self._fallback_streak = 0 + + def update(self, hr_candidate: Optional[float], + score: float) -> Optional[float]: + if hr_candidate is None: + return None + + if not self.history: + self.history.append(hr_candidate) + return hr_candidate + + recent = float(np.median(list(self.history))) + delta = abs(hr_candidate - recent) + + # Gaussian consistency weight + consistency = np.exp(-0.5 * (delta / self.max_delta) ** 2) + if consistency > 0.3: + self._fallback_streak = 0 + self.history.append(hr_candidate) + return hr_candidate + + # Try half (harmonic correction) + half_hr = hr_candidate / 2 + if 40 <= half_hr <= 120: + half_delta = abs(half_hr - recent) + half_consistency = np.exp(-0.5 * (half_delta / self.max_delta) ** 2) + if half_consistency > 0.3: + self._fallback_streak = 0 + self.history.append(half_hr) + return half_hr + + # Try double (sub-harmonic correction) + double_hr = hr_candidate * 2 + if 40 <= double_hr <= 180: + double_delta = abs(double_hr - recent) + double_consistency = np.exp( + -0.5 * (double_delta / self.max_delta) ** 2) + if double_consistency > 0.3: + self._fallback_streak = 0 + self.history.append(double_hr) + return double_hr + + # Accept anyway but keep a lone outlier out of history. A sustained + # streak of uncorrectable readings means the tracked level itself is + # stale (e.g. different occupant) — previously the frozen median then + # rejected every reading forever, so re-seed at the new level. + self._fallback_streak += 1 + if self._fallback_streak >= self.FALLBACK_RESET_STREAK: + self.history.clear() + self.history.append(hr_candidate) + self._fallback_streak = 0 + return hr_candidate + +# --------------------------------------------------------------------------- +# Breathing rate — Hilbert envelope (PMC9354426) +# --------------------------------------------------------------------------- + +def compute_breathing_rate(samples: np.ndarray, + fs: float = SAMPLE_RATE, + parameters: ProcessingParameters = DEFAULT_PARAMETERS) -> Optional[float]: + """Breathing rate via Hilbert envelope of the cardiac band. + + Pipeline: bandpass 0.8-10 Hz -> Hilbert transform -> amplitude envelope + -> bandpass 0.1-0.7 Hz -> peak counting. + """ + try: + cardiac = _bandpass(samples, 0.8, 10.0, fs) + envelope = np.abs(hilbert(cardiac)) + resp = _bandpass(envelope, 0.1, 0.7, fs) + peaks, _ = find_peaks(resp, distance=int(2.0 * fs)) + if len(peaks) < 3: + return None + intervals = np.diff(peaks) / fs + med = np.median(intervals) + valid = intervals[(intervals > med * parameters.breathing_interval_lower_ratio) + & (intervals < med * parameters.breathing_interval_upper_ratio)] + if len(valid) < 2: + return None + br = 60.0 / float(np.mean(valid)) + return br if 6 <= br <= 30 else None + except Exception as e: + log.debug("Breathing rate computation failed: %s", e) + return None + +# --------------------------------------------------------------------------- +# HR variability index — window-level IBI with harmonic gate +# +# NOTE: This is NOT clinical RMSSD. It computes successive differences of +# window-level IBI estimates (~59 per 5 min with 10s windows), not beat-to-beat +# intervals (~300+ per 5 min). The output measures HR stability across windows, +# not vagal beat-to-beat modulation. See #221 for the beat-level upgrade path. +# --------------------------------------------------------------------------- + +# Harmonic tolerance for IBI gating (18% per DSP expert recommendation) +_HRV_HARMONIC_TOL = DEFAULT_PARAMETERS.hrv_harmonic_tolerance +# Sub-window length for IBI estimation (10s per DSP expert recommendation — +# gives ~59 windows per 5 min vs ~19 with 30s windows) +_HRV_WINDOW_S = 10 +# Minimum accepted IBI estimates after filtering +_HRV_MIN_IBIS = DEFAULT_PARAMETERS.hrv_minimum_windows +# Maximum gap between consecutive accepted windows before skipping the diff +_HRV_MAX_GAP_WINDOWS = 3 + + +def _harmonic_gate(ibi_ms: float, tracker_ibi_ms: float, + tolerance: float = DEFAULT_PARAMETERS.hrv_harmonic_tolerance) -> Optional[float]: + """Reject or correct IBIs that are harmonic multiples of the expected value. + + Returns the accepted/corrected IBI in ms, or None to discard. + """ + if tracker_ibi_ms <= 0: + return ibi_ms # no prior yet — accept as-is + + for multiplier in [1.0, 2.0, 0.5, 3.0, 1.0 / 3.0]: + candidate = tracker_ibi_ms * multiplier + if candidate > 0 and abs(ibi_ms - candidate) / candidate < tolerance: + if multiplier == 1.0: + return ibi_ms # fundamental — accept + else: + return tracker_ibi_ms # harmonic — substitute tracker value + return None # outside all harmonic relationships — discard + + +def compute_hrv(samples: np.ndarray, + fs: float = SAMPLE_RATE, + parameters: ProcessingParameters = DEFAULT_PARAMETERS) -> Optional[float]: + """HR variability index via window-level IBI with harmonic gate. + + Pipeline: + 1. Bandpass 0.8-8.5 Hz + 2. 10s sub-windows with 50% overlap (~59 windows per 5 min) + 3. SHS autocorrelation per window → raw IBI + 4. Harmonic gate: reject/correct IBIs using trimmed-mean tracker + 5. Hampel filter on accepted IBI series + 6. Gap-aware RMSSD (skip diffs across rejected-window gaps) + 7. Range gate: 5-100 ms + + NOT clinical RMSSD — see module docstring and #221. + """ + try: + filtered = _bandpass(samples, HR_BAND[0], HR_BAND[1], fs) + sub_window = int(_HRV_WINDOW_S * fs) + step = sub_window // 2 + min_lag = int(fs * 60 / 150) + max_lag = int(fs * 60 / 40) + + # --- Pass 1: SHS autocorrelation per window → raw IBIs --- + raw_ibis: list = [] # (window_index, ibi_ms) + for idx, start in enumerate( + range(0, len(filtered) - sub_window + 1, step)): + chunk = filtered[start:start + sub_window] + acr = _compute_autocorr(chunk, fs) + if acr is None: + continue + search = acr[min_lag:min(max_lag + 1, len(acr))] + if len(search) == 0: + continue + + # SHS scoring — find peaks first, then score + peaks, _props = find_peaks(search, height=0.02, + distance=int(fs * 0.15)) + if len(peaks) == 0: + continue + + candidate_lags = peaks + min_lag + weights = [1.0, 0.8, 0.6] + best_lag, best_score = None, 0.0 + for lag in candidate_lags: + score = 0.0 + for k in range(3): + exact = lag / (k + 1) + lo_i = int(exact) + hi_i = lo_i + 1 + if hi_i >= len(acr): + continue + frac = exact - lo_i + val = acr[lo_i] * (1 - frac) + acr[hi_i] * frac + score += weights[k] * max(val, 0) + if score > best_score: + best_score = score + best_lag = lag + + if best_lag is None or best_score < 0.1: + continue + ibi_ms = best_lag / fs * 1000 + if 400 <= ibi_ms <= 1500: + raw_ibis.append((idx, ibi_ms)) + + if len(raw_ibis) < parameters.hrv_minimum_windows: + return None + + # --- Pass 2: Harmonic gate with trimmed-mean tracker --- + tracker_history: list = [] + gated: list = [] # (window_index, ibi_ms) + + for win_idx, ibi_ms in raw_ibis: + # Compute tracker IBI as trimmed mean of last 5 accepted + if len(tracker_history) >= 3: + sorted_h = sorted(tracker_history[-5:]) + # Drop min+max if we have enough + if len(sorted_h) >= 4: + trimmed = sorted_h[1:-1] + else: + trimmed = sorted_h + tracker_ibi = sum(trimmed) / len(trimmed) + elif tracker_history: + tracker_ibi = sum(tracker_history) / len(tracker_history) + else: + tracker_ibi = 0 # no prior — accept first value + + accepted = _harmonic_gate(ibi_ms, tracker_ibi, parameters.hrv_harmonic_tolerance) + if accepted is not None: + gated.append((win_idx, accepted)) + tracker_history.append(accepted) + + if len(gated) < parameters.hrv_minimum_windows: + return None + + # --- Pass 3: Hampel filter on gated IBI series --- + ibi_values = np.array([ibi for _, ibi in gated]) + win_indices = [idx for idx, _ in gated] + clean_mask = np.ones(len(ibi_values), dtype=bool) + + for i in range(len(ibi_values)): + lo_i = max(0, i - 3) + hi_i = min(len(ibi_values), i + 4) + local = ibi_values[lo_i:hi_i] + med = np.median(local) + mad = max(np.median(np.abs(local - med)), 1e-6) + if abs(ibi_values[i] - med) > 3.0 * 1.4826 * mad: + clean_mask[i] = False + + clean_ibis = ibi_values[clean_mask] + clean_indices = [win_indices[i] for i in range(len(win_indices)) + if clean_mask[i]] + + if len(clean_ibis) < parameters.hrv_minimum_windows: + return None + + # --- Pass 4: Gap-aware RMSSD --- + sq_diffs: list = [] + for i in range(1, len(clean_ibis)): + gap = clean_indices[i] - clean_indices[i - 1] + if gap >= _HRV_MAX_GAP_WINDOWS: + continue # skip diff across rejected-window gap + diff = clean_ibis[i] - clean_ibis[i - 1] + sq_diffs.append(diff ** 2) + + if len(sq_diffs) < 5: + return None + + hrv_index = float(np.sqrt(np.mean(sq_diffs))) + # Range gate: 5-100 ms. Window-level HRV above 100 ms is artifact + # even after harmonic correction (cardiologist recommendation). + return hrv_index if 5 <= hrv_index <= 100 else None + except Exception as e: + log.debug("HRV computation failed: %s", e) + return None + + +@dataclass(frozen=True) +class WindowEstimate: + timestamp_s: float + heart_rate: Optional[float] + hrv_index: Optional[float] + breathing_rate: Optional[float] + hr_raw: Optional[float] + hr_score: float + rejection_reasons: tuple + + +class WindowEstimator: + """Stateful numerical estimator used by live and historical adapters. + + Inputs are already gated, timestamped windows, in their processing order. + This layer cannot infer missing occupancy/pump/channel context and must not + be advertised as a full raw replay by itself. One instance per side and + configuration boundary prevents tracker history crossing profiles. + """ + def __init__(self, parameters: ProcessingParameters = DEFAULT_PARAMETERS): + self.parameters = parameters + self.hr_tracker = HRTracker(parameters.tracking_tolerance_bpm, + parameters.tracking_history_length) + + def estimate(self, timestamp_s: float, hr_samples: np.ndarray, + breathing_samples: np.ndarray, hrv_samples: np.ndarray, + fs: float = SAMPLE_RATE) -> WindowEstimate: + if not np.isfinite(timestamp_s): + raise ValueError("Expected a finite event timestamp") + if fs != SAMPLE_RATE: + raise ValueError("PROFILE_INCOMPATIBLE: this engine requires 500 Hz") + # Match live warmup exactly; never pad unavailable history with zeroes. + if len(hr_samples) < int(10 * fs): + return WindowEstimate(timestamp_s, None, None, None, None, 0.0, ("hr_warmup",)) + hr_raw, hr_score = subharmonic_summation_hr(hr_samples, fs) + hr = self.hr_tracker.update(hr_raw, hr_score) + br = compute_breathing_rate(breathing_samples, fs, self.parameters) + hrv = (compute_hrv(hrv_samples, fs, self.parameters) + if len(hrv_samples) >= int(300 * fs) else None) + reasons = [] + if hr is None: + reasons.append("no_hr") + if br is None: + reasons.append("no_br") + if hrv is None: + reasons.append("hrv_warmup" if len(hrv_samples) < int(300 * fs) else "no_hrv") + return WindowEstimate(timestamp_s, hr, hrv, br, hr_raw, hr_score, tuple(reasons)) diff --git a/modules/common/test_biometric_config.py b/modules/common/test_biometric_config.py new file mode 100644 index 000000000..9b678453c --- /dev/null +++ b/modules/common/test_biometric_config.py @@ -0,0 +1,66 @@ +"""Contract tests run with common's stdlib-only test environment.""" +import math + +import pytest + +from common.biometrics.config import canonical_json, config_hash, manifest, resolve_config, validate_config + + +def test_defaults_do_not_mutate_the_shipped_manifest(): + profile = resolve_config([]) + profile["resolved"]["heart_rate"]["trackingToleranceBpm"] = 25 + assert resolve_config([])["resolved"]["heart_rate"]["trackingToleranceBpm"] == 15 + assert manifest()["components"]["hrv_index"]["metricId"] == "window_ibi_variability_v1" + assert manifest()["components"]["sleep_stages"]["causal"] is False + + +def test_false_zero_and_rule_order_are_preserved(): + resolved = resolve_config([ + {"path": "sleep_stages.ruleset.smoothIsolatedEpochs", "value": False}, + {"path": "sleep_stages.ruleset.lowSignalBelow", "value": 0}]) + assert resolved["resolved"]["sleep_stages"]["ruleset"]["lowSignalBelow"] == 0 + assert resolved["resolved"]["sleep_stages"]["ruleset"]["smoothIsolatedEpochs"] is False + rules = [{"stage": "wake", "when": []}, {"stage": "light", "when": []}] + forward = resolve_config([{"path": "sleep_stages.ruleset.rules", "value": rules}]) + reverse = resolve_config([{"path": "sleep_stages.ruleset.rules", "value": rules[::-1]}]) + assert forward["hash"] != reverse["hash"] + + +@pytest.mark.parametrize("path,value", [ + ("heart_rate.trackingToleranceBpm", True), + ("heart_rate.trackingToleranceBpm", 0), + ("heart_rate.trackingHistoryLength", 1.5), + ("hrv_index.harmonicTolerance", math.nan), + ("sleep_stages.ruleset.smoothIsolatedEpochs", 0), + ("sleep_stages.ruleset.fallback", None), + ("sleep_stages.ruleset.outliers.heartRate.min", 150), + ("sleep_stages.ruleset.rules.0.stage", "wake"), + ("sleep_stages.ruleset.rules", [{"stage": "unknown", "when": []}]), + ("sleep_stages.ruleset", {}), + ("sensor_calibration.noise_floor", 0), +]) +def test_invalid_overrides_are_rejected(path, value): + with pytest.raises(ValueError): + resolve_config([{"path": path, "value": value}]) + + +def test_unknown_fields_and_bases_are_rejected(): + with pytest.raises(ValueError, match="PROFILE_INCOMPATIBLE"): + resolve_config([], "unknown") + with pytest.raises(ValueError): + resolve_config([{"path": "heart_rate.trackingToleranceBpm", "value": 15, "ignored": True}]) + value = manifest()["defaults"] + value["calibration"] = {} + with pytest.raises(ValueError): + validate_config(value) + with pytest.raises(ValueError, match="Duplicate"): + resolve_config([{"path": "heart_rate.trackingToleranceBpm", "value": 15}] * 2) + + +def test_canonical_numeric_json(): + assert canonical_json({"b": -0.0, "a": .5}) == '{"a":0.5,"b":0}' + assert canonical_json(.1) == "0.1000000000000000055511151231257827021181583404541015625" + assert config_hash({"a": 1, "b": 2}) == config_hash({"b": 2, "a": 1}) + for bad in [math.inf, math.nan, {1: "key"}, "\ud800", object(), [None] * 20001]: + with pytest.raises(ValueError): + canonical_json(bad) diff --git a/modules/common/test_biometric_routing.py b/modules/common/test_biometric_routing.py new file mode 100644 index 000000000..252bb1cb7 --- /dev/null +++ b/modules/common/test_biometric_routing.py @@ -0,0 +1,96 @@ +"""Routing parity with an experiment-only sink and recorded mode/time.""" +from dataclasses import dataclass + +from common.biometrics.routing import SingleSleeperRouter + + +@dataclass(frozen=True) +class Reading: + side: str + quality_score: float + sequence: int + + +class Mode: + home = 'left' + active = ('left',) + + def home_side(self): + return self.home + + def active_sides(self): + return self.active + + +def setup_router(): + now = [0.0] + mode = Mode() + output = [] + accepted = [True] + + def write(candidate, destination): + if not accepted[0]: + return False + output.append((candidate, destination)) + return True + + router = SingleSleeperRouter(mode, write, clock=lambda: now[0]) + return router, now, mode, output, accepted + + +def test_pairing_prefers_quality_and_preserves_first_candidate_on_ties(): + router, _, _, output, _ = setup_router() + left, right = Reading('left', 0.3, 1), Reading('right', 0.8, 1) + assert router.submit(left) + assert not output + assert router.submit(right) + assert output == [(right, 'left')] + tied_left, tied_right = Reading('left', 0.8, 2), Reading('right', 0.8, 2) + assert router.submit(tied_left) + assert router.submit(tied_right) + assert output[-1] == (tied_left, 'left') + + +def test_recorded_clock_expires_unpaired_reading_at_exact_cadence(): + router, now, _, output, _ = setup_router() + reading = Reading('right', 0.8, 1) + assert router.submit(reading) + now[0] = 59.999 + router.tick() + assert not output + now[0] = 60.0 + router.tick() + assert output == [(reading, 'left')] + router.tick() + assert len(output) == 1 + + +def test_failed_pair_cannot_be_overwritten_or_relabelled_after_mode_change(): + router, _, mode, output, accepted = setup_router() + winner = Reading('right', 0.8, 1) + router.submit(Reading('left', 0.3, 1)) + accepted[0] = False + assert router.submit(winner) + mode.home, mode.active = 'right', ('right',) + later = Reading('left', 0.9, 2) + assert not router.submit(later) + assert not output + accepted[0] = True + assert router.submit(later) + assert output == [(winner, 'left')] + assert router.flush() + assert output[-1] == (later, 'right') + + +def test_two_sleepers_and_empty_bed_use_recorded_mode_only(): + router, now, mode, output, _ = setup_router() + router.submit(Reading('left', 0.5, 1)) + mode.active = () + now[0] = 60 + router.tick() + assert router.flush() + assert not output + mode.home, mode.active = None, ('left', 'right') + reading = Reading('right', 0.4, 2) + assert router.submit(reading) + assert output == [(reading, None)] diff --git a/modules/common/test_biometric_runtime.py b/modules/common/test_biometric_runtime.py new file mode 100644 index 000000000..1227be18f --- /dev/null +++ b/modules/common/test_biometric_runtime.py @@ -0,0 +1,106 @@ +"""Runtime delivery integrity and boundary regression tests.""" +import json +from pathlib import Path +import sqlite3 + +import pytest + +from common.biometrics.config import resolve_config +from common.biometrics.runtime import BoundarySelection, RuntimeSnapshot, read_latest_snapshot + + +def payload(generation=1, boundary="next_session", **changes): + config = resolve_config([]) + value = dict(schemaVersion=1, side="left", generation=generation, + revisionId=None, manifestHash=config["manifestHash"], + resolved=config["resolved"], resolvedHash=config["hash"], boundary=boundary) + value.update(changes) + return json.dumps(value) + + +def test_snapshot_is_deeply_immutable(): + snapshot = RuntimeSnapshot.parse(payload()) + snapshot.resolved["heart_rate"]["trackingToleranceBpm"] = 25 + assert snapshot.resolved["heart_rate"]["trackingToleranceBpm"] == 15 + + +@pytest.mark.parametrize("changes", [ + {"generation": True}, {"generation": 0}, {"generation": 1.5}, + {"schemaVersion": True}, {"revisionId": "bad"}, {"side": "both"}, + {"manifestHash": "old-engine"}, {"resolvedHash": "forged"}, + {"boundary": "whenever"}, {"extra": 1}, +]) +def test_invalid_projection_is_rejected(changes): + with pytest.raises(ValueError): + RuntimeSnapshot.parse(payload(**changes)) + + +def test_duplicate_fields_and_oversized_payload_are_rejected(): + with pytest.raises(ValueError, match="Duplicate"): + RuntimeSnapshot.parse(payload()[:-1] + ', "generation": 2}') + with pytest.raises(ValueError, match="resource limit"): + RuntimeSnapshot.parse(" " * 65537) + + +def test_actual_migration_projection_envelope_and_latest_generation(): + connection = sqlite3.connect(":memory:") + connection.execute("CREATE TABLE vitals (id INTEGER)") + migration = Path(__file__).parents[2] / "src/db/biometrics-migrations/0020_black_patch.sql" + connection.executescript(migration.read_text()) + assert read_latest_snapshot(connection, "left") is None + snapshot = RuntimeSnapshot.parse(payload()) + connection.execute("INSERT INTO biometric_runtime_configs VALUES (?, ?, ?, ?, ?, ?)", + ("left", 1, snapshot.resolved_hash, payload(), "next_session", 0)) + assert read_latest_snapshot(connection, "left") == snapshot + connection.execute("INSERT INTO biometric_runtime_configs VALUES (?, ?, ?, ?, ?, ?)", + ("left", 2, snapshot.resolved_hash, payload(2), "immediate", 1)) + with pytest.raises(ValueError, match="envelope mismatch"): + read_latest_snapshot(connection, "left") + connection.execute("UPDATE biometric_runtime_configs SET payload = ? WHERE generation = 2", (" " * 65537,)) + with pytest.raises(ValueError, match="resource limit"): + read_latest_snapshot(connection, "left") + + +def test_missing_schema_is_not_silently_created(): + connection = sqlite3.connect(":memory:") + with pytest.raises(sqlite3.OperationalError): + read_latest_snapshot(connection, "left") + assert connection.execute("SELECT name FROM sqlite_master").fetchall() == [] + + +def test_next_session_waits_through_restart_and_unknown_occupancy(): + selection = BoundarySelection("left") + snapshot = RuntimeSnapshot.parse(payload()) + selection.offer(snapshot) + assert selection.ready(True) is None + assert selection.ready(None) is None + assert selection.ready(False) == snapshot + assert selection.applied is None # selection alone does not acknowledge + selection.commit(snapshot) + assert selection.applied == snapshot + selection.offer(snapshot) + assert selection.pending is None + + +def test_newer_request_supersedes_pending_and_fences_late_commit(): + selection = BoundarySelection("left") + first = RuntimeSnapshot.parse(payload()) + second = RuntimeSnapshot.parse(payload(2, "immediate")) + selection.offer(first) + selection.offer(second) + assert selection.ready(True) == second + with pytest.raises(ValueError, match="changed"): + selection.commit(first) + with pytest.raises(ValueError, match="regression"): + selection.offer(first) + with pytest.raises(ValueError, match="conflict"): + selection.offer(RuntimeSnapshot.parse(payload(2))) + selection.commit(second) + assert selection.applied == second + + +def test_side_mismatch_never_changes_selection(): + selection = BoundarySelection("right") + with pytest.raises(ValueError, match="side mismatch"): + selection.offer(RuntimeSnapshot.parse(payload())) + assert selection.pending is None diff --git a/modules/common/test_build_identity.py b/modules/common/test_build_identity.py new file mode 100644 index 000000000..6cafee1e3 --- /dev/null +++ b/modules/common/test_build_identity.py @@ -0,0 +1,57 @@ +import hashlib +import importlib.metadata +from pathlib import Path + +import pytest + +from common.biometrics import build_identity +from common.biometrics.config import config_hash + + +def test_identity_is_deterministic_and_contains_sources_and_installed_versions(): + first = build_identity.replay_build_identity() + assert first == build_identity.replay_build_identity() + assert first['hash'] == config_hash(first['identity']) + assert set(first['identity']['sources']) == set(build_identity.SOURCE_FILES) + assert 'common/biometrics/build_identity.py' in first['identity']['sources'] + assert first['identity']['dependencies']['numpy']['version'] == importlib.metadata.version('numpy') + assert first['identity']['dependencies']['scipy']['version'] == importlib.metadata.version('scipy') + root = Path(build_identity.__file__).resolve().parents[2] + assert first['identity']['sources']['piezo-processor/uv.lock'] == hashlib.sha256((root / 'piezo-processor/uv.lock').read_bytes()).hexdigest() + + +def test_changed_sources_and_dependency_metadata_change_identity(monkeypatch): + before = build_identity.replay_build_identity() + original = build_identity._digest_file + monkeypatch.setattr(build_identity, '_digest_file', lambda path: '0' * 64 if path.name == 'signal.py' else original(path)) + assert build_identity.replay_build_identity()['hash'] != before['hash'] + monkeypatch.setattr(build_identity, '_digest_file', original) + distribution = importlib.metadata.distribution + + class Changed: + version = 'changed-build' + + def read_text(self, name): + return name + '-changed' + + monkeypatch.setattr(importlib.metadata, 'distribution', lambda name: Changed() if name == 'numpy' else distribution(name)) + assert build_identity.replay_build_identity()['hash'] != before['hash'] + + +def test_missing_wheel_metadata_and_oversize_sources_fail_closed(tmp_path, monkeypatch): + source = tmp_path / 'source.py' + source.write_bytes(b'12345') + monkeypatch.setattr(build_identity, 'MAX_FILE_BYTES', 4) + with pytest.raises(ValueError, match='bound'): + build_identity._digest_file(source) + monkeypatch.setattr(build_identity, 'MAX_FILE_BYTES', 16 * 1024 * 1024) + + class Missing: + version = '1' + + def read_text(self, name): + return None + + monkeypatch.setattr(importlib.metadata, 'distribution', lambda name: Missing()) + with pytest.raises(ValueError, match='unavailable'): + build_identity.replay_build_identity() diff --git a/modules/common/test_execution_lock.py b/modules/common/test_execution_lock.py new file mode 100644 index 000000000..68543a505 --- /dev/null +++ b/modules/common/test_execution_lock.py @@ -0,0 +1,101 @@ +"""Replay and shadow must compete for one kernel-owned numerical worker slot.""" +import io +import os +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from common.biometrics import execution_lock, replay_worker, shadow_process + + +def test_replay_and_shadow_contend_for_same_inode(tmp_path, monkeypatch): + monkeypatch.setattr(sys, 'platform', 'linux') + path = tmp_path / 'execution.lock' + assert execution_lock.EXECUTION_LOCK_PATH == shadow_process.SHADOW_LOCK_PATH + first = shadow_process.acquire_shadow_lock(path) + try: + with pytest.raises(BlockingIOError): + replay_worker.acquire_execution_lock(path) + finally: + os.close(first) + second = replay_worker.acquire_execution_lock(path) + try: + with pytest.raises(BlockingIOError): + shadow_process.acquire_shadow_lock(path) + finally: + os.close(second) + + +def test_replay_contention_fails_before_limits_imports_or_input(monkeypatch): + def busy(): + raise BlockingIOError('private path') + monkeypatch.setattr(replay_worker, 'acquire_execution_lock', busy) + monkeypatch.setattr(replay_worker, 'configure_process_limits', lambda *_: pytest.fail('limits before lock')) + monkeypatch.setattr(sys, 'argv', ['worker', '--retained-directory', '/private']) + monkeypatch.setattr(sys, 'stdin', None) + stderr = io.StringIO() + monkeypatch.setattr(sys, 'stderr', stderr) + assert replay_worker.main() == 1 + assert stderr.getvalue() == 'Retained waveform worker failed\n' + + +@pytest.mark.parametrize('failed', [False, True]) +def test_replay_keeps_lock_through_processing_and_closes_on_every_exit(monkeypatch, failed): + calls = [] + monkeypatch.setattr(sys, 'argv', ['worker', '--retained-directory', '/retained']) + monkeypatch.setattr(sys, 'stdin', SimpleNamespace(buffer='source')) + monkeypatch.setattr(sys, 'stdout', SimpleNamespace(buffer='sink')) + monkeypatch.setattr(sys, 'stderr', io.StringIO()) + monkeypatch.setattr(replay_worker, 'acquire_execution_lock', lambda: (calls.append('lock'), 123)[1]) + monkeypatch.setattr(replay_worker, 'configure_process_limits', lambda *_: calls.append('limits')) + monkeypatch.setattr(replay_worker.os, 'close', lambda fd: calls.append(('close', fd))) + def run(*args): + assert calls == ['lock', 'limits'] + calls.append('run') + if failed: + raise ValueError('failed') + monkeypatch.setattr(replay_worker, 'run', run) + assert replay_worker.main() == (1 if failed else 0) + assert calls == ['lock', 'limits', 'run', ('close', 123)] + + +@pytest.mark.skipif(sys.platform != 'linux', reason='Real Linux signal/lock ownership') +def test_real_shared_lock_survives_stop_and_releases_on_kill(tmp_path): + path = tmp_path / 'execution.lock' + code = ('from common.biometrics.execution_lock import acquire_execution_lock; ' + 'import sys,time; fd=acquire_execution_lock(sys.argv[1]); ' + 'print("owned",flush=True); time.sleep(60)') + child = subprocess.Popen([sys.executable, '-c', code, str(path)], stdout=subprocess.PIPE, + env={**os.environ, 'PYTHONPATH': str(Path(__file__).resolve().parent.parent)}) + try: + import select + import signal + assert select.select([child.stdout], [], [], 5)[0] + assert child.stdout.readline() == b'owned\n' + child.send_signal(signal.SIGSTOP) + with pytest.raises(BlockingIOError): + shadow_process.acquire_shadow_lock(path) + child.kill() + child.wait(timeout=5) + fd = shadow_process.acquire_shadow_lock(path) + os.close(fd) + finally: + if child.poll() is None: + child.kill() + child.wait(timeout=5) + child.stdout.close() + + +def test_build_probe_does_not_claim_numerical_execution_slot(monkeypatch): + from common.biometrics import build_identity + monkeypatch.setattr(sys, 'argv', ['worker', '--describe-build']) + monkeypatch.setattr(replay_worker, 'acquire_execution_lock', lambda: pytest.fail('probe claimed slot')) + monkeypatch.setattr(replay_worker, 'configure_process_limits', lambda *_: None) + monkeypatch.setattr(build_identity, 'replay_build_identity', lambda: {'hash': 'fixture'}) + output = io.StringIO() + monkeypatch.setattr(sys, 'stdout', output) + assert replay_worker.main() == 0 + assert output.getvalue() == '{"hash":"fixture"}\n' diff --git a/modules/common/test_inspection_worker.py b/modules/common/test_inspection_worker.py new file mode 100644 index 000000000..abbd3a196 --- /dev/null +++ b/modules/common/test_inspection_worker.py @@ -0,0 +1,71 @@ +import gzip +from io import BytesIO +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest + +from common.biometrics.inspection_worker import run + + +def request(root, after=None, payload=None): + output = BytesIO() + run(BytesIO(payload if payload is not None else json.dumps({ + 'version': 'raw-inspection-work-v1', 'after_name': after}).encode()), output, root) + return json.loads(output.getvalue()) + + +def write_archive(root, name): + payload = b'\xa1\x64type\x61x' + (root / name).write_bytes(gzip.compress(b'\xa2\x63seq\x01\x64data\x48' + payload, mtime=0)) + + +def test_inspects_one_archive_at_a_time_without_retention(tmp_path): + write_archive(tmp_path, 'b.RAW.gz') + write_archive(tmp_path, 'a.RAW.gz') + before = {p.name: p.read_bytes() for p in tmp_path.iterdir()} + first = request(tmp_path) + assert first['status'] == 'inspected' + assert first['nextCursor'] == 'a.RAW.gz' + assert first['inspection']['retained'] is False + assert first['inspection']['inventory']['decoded_records'] == 1 + assert first['inspection']['artifact']['sha256'] == first['inspection']['inventory']['artifact_sha256'] + assert request(tmp_path, first['nextCursor'])['nextCursor'] == 'b.RAW.gz' + assert request(tmp_path, 'b.RAW.gz') == {'status': 'complete', 'nextCursor': None} + assert {p.name: p.read_bytes() for p in tmp_path.iterdir()} == before + + +def test_missing_directory_and_corruption_have_explicit_results(tmp_path): + assert request(tmp_path / 'missing') == {'status': 'archive_unavailable', 'nextCursor': None} + assert not (tmp_path / 'missing').exists() + (tmp_path / 'a.RAW.gz').write_bytes(b'not gzip') + write_archive(tmp_path, 'b.RAW.gz') + assert request(tmp_path) == {'status': 'source_unavailable', 'nextCursor': 'a.RAW.gz'} + assert request(tmp_path, 'a.RAW.gz')['status'] == 'inspected' + + +@pytest.mark.parametrize('payload', [ + b'{}', b'[]', b'x' * 1025, + b'{"version":"raw-inspection-work-v1","after_name":null,"after_name":null}', + b'{"version":"raw-inspection-work-v1","after_name":"../a.RAW.gz"}', + b'{"version":"raw-inspection-work-v1","after_name":0}', + b'{"version":"raw-inspection-work-v1","after_name":null,"path":"/tmp"}', +]) +def test_rejects_invalid_work_before_filesystem_access(tmp_path, payload): + with pytest.raises(ValueError): + request(tmp_path / 'missing', payload=payload) + + +def test_fixed_cli_returns_metadata_and_private_failure(tmp_path): + write_archive(tmp_path, 'a.RAW.gz') + command = [sys.executable, '-B', '-m', 'common.biometrics.inspection_worker', '--archive-directory', str(tmp_path)] + env = {**os.environ, 'PYTHONPATH': str(Path(__file__).resolve().parents[1])} + result = subprocess.run(command, input=json.dumps({'version': 'raw-inspection-work-v1', 'after_name': None}).encode(), capture_output=True, env=env, timeout=10) + assert result.returncode == 0 and not result.stderr + assert json.loads(result.stdout)['inspection']['source_name'] == 'a.RAW.gz' + failed = subprocess.run(command, input=b'private malformed input', capture_output=True, env=env, timeout=10) + assert failed.returncode == 1 and not failed.stdout + assert failed.stderr == b'RAW inspection worker failed\n' diff --git a/modules/common/test_raw_artifacts.py b/modules/common/test_raw_artifacts.py new file mode 100644 index 000000000..12a340485 --- /dev/null +++ b/modules/common/test_raw_artifacts.py @@ -0,0 +1,203 @@ +"""Retained RAW bytes survive rolling archive removal, with bounded reads.""" +from concurrent.futures import ThreadPoolExecutor +from dataclasses import replace +import gzip +import hashlib +import os +from pathlib import Path + +import pytest + +from common.biometrics.raw_artifacts import RawArtifactStore, RawArtifact, RawLimits + + +def fixture(tmp_path, payload=b'synthetic waveform bytes' * 100): + archive = tmp_path / 'archive' + archive.mkdir() + compressed = gzip.compress(payload, mtime=0) + (archive / '1.RAW.gz').write_bytes(compressed) + store = RawArtifactStore(tmp_path / 'retained', RawLimits(4096, 65536, 8192, 1)) + return archive, store, compressed + + +def read_all(store, descriptor): + chunks = [] + with store.open_raw(descriptor) as stream: + while True: + chunk = stream.read(1024) + if not chunk: + break + chunks.append(chunk) + return b''.join(chunks) + + +def test_retains_private_immutable_bytes_and_reuses_identical_content(tmp_path): + payload = b'original samples' * 100 + archive, store, compressed = fixture(tmp_path, payload) + descriptor = store.retain(archive, '1.RAW.gz') + assert descriptor.sha256 == hashlib.sha256(compressed).hexdigest() + assert descriptor.compressed_bytes == len(compressed) + assert descriptor.expanded_bytes == len(payload) + assert store.retain(archive, '1.RAW.gz') == descriptor + (archive / '1.RAW.gz').unlink() + assert read_all(store, descriptor) == payload + assert (store.directory / descriptor.name).stat().st_mode & 0o777 == 0o600 + assert store.directory.stat().st_mode & 0o777 == 0o700 + assert len(list(store.directory.glob('*.RAW.gz'))) == 1 + assert not list(store.directory.glob('.part-*')) + + +def test_rejects_paths_symlinks_and_non_regular_archive_inputs(tmp_path): + archive, store, _ = fixture(tmp_path) + for name in ['../1.RAW.gz', '/etc/passwd', '1.RAW', '']: + with pytest.raises(ValueError): + store.retain(archive, name) + (archive / '2.RAW.gz').symlink_to(archive / '1.RAW.gz') + with pytest.raises(OSError): + store.retain(archive, '2.RAW.gz') + os.mkfifo(archive / '3.RAW.gz') + with pytest.raises(ValueError, match='regular'): + store.retain(archive, '3.RAW.gz') + with pytest.raises(ValueError): + RawArtifact('../file', 'x', 1, 1) + + +def test_compressed_expanded_and_free_space_limits_leave_no_partial_artifact(tmp_path): + archive, store, compressed = fixture(tmp_path, b'a' * 10000) + for limits, message in [ + (RawLimits(len(compressed) - 1, 65536, 8192, 1), 'compressed'), + (RawLimits(4096, 9999, 8192, 1), 'expanded'), + (RawLimits(4096, 65536, 8192, 2**62), 'free space'), + ]: + store.limits = limits + with pytest.raises(ValueError, match=message): + store.retain(archive, '1.RAW.gz') + assert not list(store.directory.glob('*.RAW.gz')) + assert not list(store.directory.glob('.part-*')) + + +def test_corruption_and_wrong_descriptors_fail_before_reading(tmp_path): + archive, store, _ = fixture(tmp_path) + descriptor = store.retain(archive, '1.RAW.gz') + with pytest.raises(ValueError, match='size mismatch'): + read_all(store, replace(descriptor, compressed_bytes=descriptor.compressed_bytes + 1)) + with pytest.raises(ValueError, match='expanded size mismatch'): + read_all(store, replace(descriptor, expanded_bytes=descriptor.expanded_bytes + 1)) + with pytest.raises(ValueError, match='expanded byte limit'): + read_all(store, replace(descriptor, expanded_bytes=descriptor.expanded_bytes - 1)) + path = store.directory / descriptor.name + corrupted = bytearray(path.read_bytes()) + corrupted[-1] ^= 1 + path.write_bytes(corrupted) + with pytest.raises(ValueError, match='checksum'): + read_all(store, descriptor) + with pytest.raises(ValueError, match='checksum'): + store.retain(archive, '1.RAW.gz') + + +def test_serializes_quota_publication_and_recovers_crashed_staging_file(tmp_path): + archive, store, _ = fixture(tmp_path, os.urandom(1024)) + (archive / '2.RAW.gz').write_bytes(gzip.compress(os.urandom(1024), mtime=0)) + store.limits = RawLimits(4096, 65536, 1500, 1) + (store.directory / ('.part-' + 'a' * 32)).write_bytes(b'crashed copy') + def retain(name): + try: + return store.retain(archive, name) + except ValueError as error: + return str(error) + with ThreadPoolExecutor(max_workers=2) as pool: + results = list(pool.map(retain, ['1.RAW.gz', '2.RAW.gz'])) + assert sum(isinstance(row, RawArtifact) for row in results) == 1 + assert any(isinstance(row, str) and 'quota' in row for row in results) + assert len(list(store.directory.glob('*.RAW.gz'))) == 1 + assert not list(store.directory.glob('.part-*')) + + +def test_rejects_unbounded_reads_and_invalid_gzip(tmp_path): + archive, store, _ = fixture(tmp_path) + descriptor = store.retain(archive, '1.RAW.gz') + with store.open_raw(descriptor) as stream: + with pytest.raises(ValueError, match='bounded'): + stream.read() + assert stream.read(0) == b'' + (archive / '2.RAW.gz').write_bytes(b'not gzip') + with pytest.raises(OSError): + store.retain(archive, '2.RAW.gz') + assert not list(store.directory.glob('.part-*')) + + +def test_reuses_identical_content_at_exact_publication_quota(tmp_path): + archive, store, compressed = fixture(tmp_path) + store.limits = RawLimits(4096, 65536, len(compressed), 1) + first = store.retain(archive, '1.RAW.gz') + assert store.retain(archive, '1.RAW.gz') == first + assert store._usage() == (len(compressed), 1) + + +def test_changed_archive_is_not_published(tmp_path, monkeypatch): + archive, store, _ = fixture(tmp_path) + original_read = os.read + changed = False + def read(fd, count): + nonlocal changed + data = original_read(fd, count) + if data and not changed: + changed = True + (archive / '1.RAW.gz').write_bytes(gzip.compress(b'replaced source', mtime=0)) + return data + monkeypatch.setattr(os, 'read', read) + with pytest.raises(ValueError, match='changed during retention'): + store.retain(archive, '1.RAW.gz') + assert not list(store.directory.glob('*.RAW.gz')) + assert not list(store.directory.glob('.part-*')) + + +def test_count_budget_applies_even_to_tiny_files_but_allows_idempotent_retry(tmp_path): + archive, store, _ = fixture(tmp_path, b'one') + store.limits = replace(store.limits, max_artifacts=1) + first = store.retain(archive, '1.RAW.gz') + assert store.retain(archive, '1.RAW.gz') == first + (archive / '2.RAW.gz').write_bytes(gzip.compress(b'two', mtime=0)) + with pytest.raises(ValueError, match='artifact count'): + store.retain(archive, '2.RAW.gz') + assert read_all(store, first) == b'one' + assert not list(store.directory.glob('.part-*')) + + +def test_read_only_store_does_not_create_or_change_storage_and_rejects_mutations(tmp_path): + archive, store, _ = fixture(tmp_path) + descriptor = store.retain(archive, '1.RAW.gz') + os.chmod(store.directory, 0o750) + reader = RawArtifactStore(store.directory, store.limits, read_only=True) + assert store.directory.stat().st_mode & 0o777 == 0o750 + assert read_all(reader, descriptor) == b'synthetic waveform bytes' * 100 + with pytest.raises(ValueError, match='read-only'): + reader.retain(archive, '1.RAW.gz') + with pytest.raises(ValueError, match='read-only'): + reader.collect_released(None, now_ms=1000) + missing = tmp_path / 'missing' + with pytest.raises(ValueError): + RawArtifactStore(missing, read_only=True) + assert not missing.exists() + (store.directory / '.lock').unlink() + with pytest.raises(FileNotFoundError): + read_all(reader, descriptor) + assert not (store.directory / '.lock').exists() + + os.mkfifo(store.directory / '.lock') + with pytest.raises(ValueError, match='regular file'): + read_all(reader, descriptor) + + +def test_reuses_exact_retained_bytes_without_the_original_archive_and_checks_expanded_size(tmp_path): + archive, store, _ = fixture(tmp_path) + descriptor = store.retain(archive, '1.RAW.gz') + (archive / '1.RAW.gz').unlink() + archive.rmdir() + seen = [] + assert store.retain(archive, '1.RAW.gz', expected=descriptor, publish=seen.append) == descriptor + assert seen == [descriptor] + with pytest.raises(ValueError, match='expanded size mismatch'): + store.retain(archive, '1.RAW.gz', expected=replace(descriptor, expanded_bytes=descriptor.expanded_bytes + 1)) + with pytest.raises(ValueError, match='expanded byte limit'): + store.retain(archive, '1.RAW.gz', expected=replace(descriptor, expanded_bytes=descriptor.expanded_bytes - 1)) diff --git a/modules/common/test_raw_collection.py b/modules/common/test_raw_collection.py new file mode 100644 index 000000000..de75588de --- /dev/null +++ b/modules/common/test_raw_collection.py @@ -0,0 +1,269 @@ +"""Released files cannot race verified reads or reference publication.""" +from concurrent.futures import ThreadPoolExecutor +from contextlib import closing +import gzip +from pathlib import Path +import sqlite3 +from threading import Event + +import pytest + +from common.biometrics.raw_artifacts import RawArtifactStore, RawLimits + + +def fixture(tmp_path): + archive = tmp_path / 'archive' + archive.mkdir() + (archive / '1.RAW.gz').write_bytes(gzip.compress(b'samples' * 100, mtime=0)) + store = RawArtifactStore(tmp_path / 'retained', RawLimits(4096, 65536, 8192, 1)) + descriptor = store.retain(archive, '1.RAW.gz') + database = tmp_path / 'references.db' + connection = sqlite3.connect(database) + connection.execute('PRAGMA foreign_keys = ON') + connection.execute('CREATE TABLE biometric_datasets (id TEXT PRIMARY KEY)') + connection.execute("INSERT INTO biometric_datasets VALUES ('dataset')") + migration = Path(__file__).parents[2] / 'src/db/biometrics-migrations/0028_retained_raw_artifacts.sql' + connection.executescript(migration.read_text()) + connection.execute('INSERT INTO biometric_raw_artifacts VALUES (?, ?, ?, ?, ?, ?)', + (descriptor.sha256, descriptor.name, descriptor.compressed_bytes, + descriptor.expanded_bytes, 1000, 2000)) + connection.commit() + return archive, store, descriptor, database, connection + + +def test_collects_only_released_unreferenced_bytes_after_grace(tmp_path): + _, store, descriptor, _, db = fixture(tmp_path) + try: + db.execute('INSERT INTO biometric_dataset_raw_artifacts VALUES (?, ?, ?, ?, ?)', + ('link', 'dataset', descriptor.sha256, 0, None)) + db.commit() + assert store.collect_released(db, now_ms=10000, minimum_age_ms=1000) == [] + db.execute('UPDATE biometric_dataset_raw_artifacts SET released_at_ms = 2000') + db.commit() + assert store.collect_released(db, now_ms=2999, minimum_age_ms=1000) == [] + assert store.collect_released(db, now_ms=3000, minimum_age_ms=1000) == [descriptor.name] + assert not (store.directory / descriptor.name).exists() + assert store.collect_released(db, now_ms=4000, minimum_age_ms=1000) == [] + # Metadata/links remain as tombstones, including their original identity. + assert db.execute('SELECT released_at_ms FROM biometric_raw_artifacts').fetchone() == (2000,) + assert db.execute('SELECT artifact_hash FROM biometric_dataset_raw_artifacts').fetchone() == (descriptor.sha256,) + finally: + db.close() + + +def test_preserves_unknown_files_and_rejects_unbounded_or_nested_collection(tmp_path): + _, store, descriptor, _, db = fixture(tmp_path) + try: + unknown = store.directory / ('f' * 64 + '.RAW.gz') + unknown.write_bytes(b'unknown future artifact') + assert store.collect_released(db, now_ms=2000, minimum_age_ms=0) == [descriptor.name] + assert unknown.exists() + db.execute('BEGIN') + with pytest.raises(ValueError, match='independent transaction'): + store.collect_released(db, now_ms=3000) + db.rollback() + for kwargs in [{'now_ms': -1}, {'now_ms': True}, {'now_ms': 3000, 'minimum_age_ms': -1}, + {'now_ms': 3000, 'maximum_entries': 0}]: + with pytest.raises(ValueError, match='bounds'): + store.collect_released(db, **kwargs) + with pytest.raises(ValueError, match='entry limit'): + store.collect_released(db, now_ms=3000, maximum_entries=1) + finally: + db.close() + + +def test_rejects_released_symlinks_and_metadata_identity_mismatch(tmp_path): + _, store, descriptor, _, db = fixture(tmp_path) + try: + path = store.directory / descriptor.name + path.unlink() + victim = tmp_path / 'unrelated' + victim.write_text('keep') + path.symlink_to(victim) + with pytest.raises(ValueError, match='regular file'): + store.collect_released(db, now_ms=3000, minimum_age_ms=0) + assert victim.read_text() == 'keep' + assert not db.in_transaction + db.execute('UPDATE biometric_raw_artifacts SET content_hash = ?', ('f' * 64,)) + db.commit() + with pytest.raises(ValueError, match='identity mismatch'): + store.collect_released(db, now_ms=3000, minimum_age_ms=0) + finally: + db.close() + + +def test_reader_holds_file_until_stream_is_closed(tmp_path): + _, store, descriptor, database, db = fixture(tmp_path) + db.close() + started = Event() + + def collect(): + with closing(sqlite3.connect(database)) as connection: + started.set() + return store.collect_released(connection, now_ms=3000, minimum_age_ms=0) + + with ThreadPoolExecutor(max_workers=1) as pool: + with store.open_raw(descriptor) as stream: + future = pool.submit(collect) + assert started.wait(2) + assert not future.done() + assert stream.read(7) == b'samples' + assert future.result(timeout=2) == [descriptor.name] + + +def test_reference_publication_completes_before_collection_can_check(tmp_path): + archive, store, descriptor, database, db = fixture(tmp_path) + started = Event() + pending = [] + + def collect(): + with closing(sqlite3.connect(database)) as connection: + started.set() + return store.collect_released(connection, now_ms=3000, minimum_age_ms=0) + + with ThreadPoolExecutor(max_workers=1) as pool: + def publish(actual): + assert actual == descriptor + pending.append(pool.submit(collect)) + assert started.wait(2) + assert not pending[0].done() + with db: + db.execute('UPDATE biometric_raw_artifacts SET released_at_ms = NULL') + db.execute('INSERT INTO biometric_dataset_raw_artifacts VALUES (?, ?, ?, ?, ?)', + ('link', 'dataset', descriptor.sha256, 0, None)) + + assert store.retain(archive, '1.RAW.gz', expected=descriptor, publish=publish) == descriptor + assert pending[0].result(timeout=2) == [] + db.close() + assert (store.directory / descriptor.name).exists() + + +def test_failed_publisher_leaves_valid_bytes_and_releases_file_lock(tmp_path): + archive, store, descriptor, _, db = fixture(tmp_path) + try: + # Exercise fresh publication, not just the deduplicated path. + (store.directory / descriptor.name).unlink() + + def fail(actual): + assert actual == descriptor + raise RuntimeError('publication failed') + + with pytest.raises(ValueError, match='inspected descriptor'): + store.retain(archive, '1.RAW.gz', publish=fail) + with pytest.raises(RuntimeError, match='publication failed'): + store.retain(archive, '1.RAW.gz', expected=descriptor, publish=fail) + with store.open_raw(descriptor) as stream: + assert stream.read(7) == b'samples' + assert not list(store.directory.glob('.part-*')) + assert store.collect_released(db, now_ms=3000, minimum_age_ms=0) == [descriptor.name] + finally: + db.close() + + +def test_collects_only_old_unpublished_copies_and_parts(tmp_path): + import os + _, store, descriptor, _, db = fixture(tmp_path) + try: + orphan = store.directory / ('a' * 64 + '.RAW.gz') + part = store.directory / ('.part-' + 'b' * 32) + fresh = store.directory / ('c' * 64 + '.RAW.gz') + unknown = store.directory / 'unrecognized' + for path in [orphan, part, fresh, unknown]: + path.write_bytes(b'private fixture') + os.utime(path, ns=(1_000_000_000, 1_000_000_000)) + os.utime(fresh, ns=(3_000_000_000, 3_000_000_000)) + assert store.collect_orphans(db, now_ms=1999, minimum_age_ms=1000) == [] + assert store.collect_orphans(db, now_ms=2000, minimum_age_ms=1000) == sorted([orphan.name, part.name]) + assert fresh.exists() and unknown.exists() + assert (store.directory / descriptor.name).exists() + assert db.execute('SELECT count(*) FROM biometric_raw_artifacts').fetchone()[0] == 1 + finally: + db.close() + + +def test_orphan_inventory_and_symlink_errors_delete_nothing(tmp_path): + import os + _, store, _, _, db = fixture(tmp_path) + try: + orphan = store.directory / ('a' * 64 + '.RAW.gz') + orphan.write_bytes(b'old') + os.utime(orphan, ns=(0, 0)) + with pytest.raises(ValueError, match='entry limit'): + store.collect_orphans(db, now_ms=10000, maximum_entries=1) + assert orphan.exists() + symlink = store.directory / ('b' * 64 + '.RAW.gz') + symlink.symlink_to(orphan) + with pytest.raises(ValueError, match='regular file'): + store.collect_orphans(db, now_ms=10000, minimum_age_ms=0) + assert orphan.exists() and symlink.is_symlink() + assert not db.in_transaction + finally: + db.close() + + +def test_orphan_collector_preserves_metadata_even_when_unreferenced_or_misnamed(tmp_path): + import os + _, store, descriptor, _, db = fixture(tmp_path) + try: + path = store.directory / descriptor.name + os.utime(path, ns=(0, 0)) + db.execute('UPDATE biometric_raw_artifacts SET released_at_ms = NULL, name = ?', ('inconsistent-name',)) + db.commit() + assert store.collect_orphans(db, now_ms=10000, minimum_age_ms=0) == [] + assert path.exists() + finally: + db.close() + + +def test_orphan_collection_waits_for_publication_metadata_commit(tmp_path): + archive, store, descriptor, database, db = fixture(tmp_path) + db.execute('DELETE FROM biometric_raw_artifacts') + db.commit() + started = Event() + pending = [] + + def collect(): + with closing(sqlite3.connect(database)) as connection: + started.set() + return store.collect_orphans(connection, now_ms=9_000_000_000_000, minimum_age_ms=0) + + try: + with ThreadPoolExecutor(max_workers=1) as pool: + def publish(actual): + pending.append(pool.submit(collect)) + assert started.wait(2) + assert not pending[0].done() + with db: + db.execute('INSERT INTO biometric_raw_artifacts VALUES (?, ?, ?, ?, ?, ?)', + (actual.sha256, actual.name, actual.compressed_bytes, + actual.expanded_bytes, 1000, None)) + + store.retain(archive, '1.RAW.gz', expected=descriptor, publish=publish) + assert pending[0].result(timeout=2) == [] + assert (store.directory / descriptor.name).exists() + finally: + db.close() + + +def test_collection_worker_runs_both_collectors_without_exposing_filenames(tmp_path): + from io import BytesIO + import json + import os + from common.biometrics.collection_worker import run + _, store, descriptor, database, db = fixture(tmp_path) + try: + orphan = store.directory / ('a' * 64 + '.RAW.gz') + orphan.write_bytes(b'private fixture') + os.utime(orphan, ns=(0, 0)) + output = BytesIO() + run(BytesIO(b'{"version":"raw-collection-work-v1"}'), output, database, store.directory) + assert json.loads(output.getvalue()) == {'version': 'raw-collection-result-v1', 'released': 1, 'orphans': 1} + assert not orphan.exists() and not (store.directory / descriptor.name).exists() + assert db.execute('SELECT count(*) FROM biometric_raw_artifacts').fetchone()[0] == 1 + with pytest.raises(ValueError, match='envelope'): + run(BytesIO(b'{"version":"raw-collection-work-v1","path":"/tmp"}'), BytesIO(), database, store.directory) + missing = tmp_path / 'missing' + run(BytesIO(b'{"version":"raw-collection-work-v1"}'), BytesIO(), database, missing) + assert not missing.exists() + finally: + db.close() diff --git a/modules/common/test_raw_records.py b/modules/common/test_raw_records.py new file mode 100644 index 000000000..48b7e1c39 --- /dev/null +++ b/modules/common/test_raw_records.py @@ -0,0 +1,164 @@ +"""Bounded CBOR decoding against captured firmware and malformed archives.""" +import base64 +from dataclasses import replace +import gzip +import importlib.util +from pathlib import Path + +import pytest +import cbor2 + +from common.biometrics import raw_records +from common.biometrics.raw_records import decode_record, DecodeLimits, iter_retained_records +from common.biometrics.raw_artifacts import RawArtifactStore, RawLimits +from common.nats_capture_fixtures import CAPTURE_FIXTURES + +# Other follower tests stub the outer parser during collection. Use its actual +# source for this integration test, matching test_cbor_raw's isolation strategy. +spec = importlib.util.spec_from_file_location('_retained_raw_parser', Path(__file__).with_name('cbor_raw.py')) +parser = importlib.util.module_from_spec(spec) +spec.loader.exec_module(parser) + + +@pytest.fixture(autouse=True) +def real_outer_parser(monkeypatch): + monkeypatch.setattr(raw_records, 'read_raw_record', parser.read_raw_record) + + +@pytest.mark.parametrize('fixture', list(CAPTURE_FIXTURES.values())) +def test_decodes_actual_firmware_indefinite_maps(fixture): + payload = base64.b64decode(fixture['payload_b64']) + record = decode_record(payload) + assert record == cbor2.loads(payload) + assert record['type'] == fixture['type'] + assert isinstance(record['ts'], (int, float)) + if record['type'] == 'piezo-dual': + assert len(record['left1']) == len(record['right1']) == 2000 + + +def test_rejects_huge_declared_containers_without_allocating_them(): + for prefix in [b'\x9b', b'\xbb']: + with pytest.raises(ValueError, match='container budget'): + decode_record(prefix + b'\xff' * 8) + with pytest.raises(ValueError, match='string budget'): + decode_record(b'\x5b' + b'\xff' * 8) + + +@pytest.mark.parametrize('payload', [ + b'\xbf\x64type\x61x\x64type\x61y\xff', # Duplicate indefinite key. + b'\xa2\x64type\x61x\x64type\x61y', # Duplicate definite key. + b'\xa1\x01\x61x', # Non-text key. + b'\xa1\x64type\x61x\x00', # Trailing second value. + b'\xa1\x64type\xc0\x61x', # Semantic tag. + b'\xbf\x64type\x61x', # Missing break. + b'\xa1\x64type\x7f\x61\xc3\x61\xa9\xff', # Invalid UTF-8 chunks. +]) +def test_rejects_ambiguous_or_malformed_records(payload): + with pytest.raises(ValueError): + decode_record(payload) + + +def test_depth_nodes_bytes_and_indefinite_chunk_budgets(): + with pytest.raises(ValueError, match='nesting'): + decode_record(b'\x81' * 18 + b'\x00') + with pytest.raises(ValueError, match='node budget'): + decode_record(b'\x9f' + b'\x00' * 8 + b'\xff', replace(DecodeLimits(), nodes=5)) + with pytest.raises(ValueError, match='byte budget'): + decode_record(b'x' * 10, replace(DecodeLimits(), record_bytes=9)) + with pytest.raises(ValueError, match='string budget'): + decode_record(b'\x5f\x43abc\x43def\xff', replace(DecodeLimits(), string_bytes=5)) + + +def test_numeric_and_chunked_values_preserve_wire_semantics(): + payload = (b'\xbf\x64type\x61x\x61v\x9f\x20\xf5\xf4\xf6' + b'\xfa\x3f\xc0\x00\x00\xf9\x3e\x00\xff' + b'\x61b\x5f\x42ab\x41c\xff\xff') + assert decode_record(payload) == {'type': 'x', 'v': [-1, True, False, None, 1.5, 1.5], 'b': b'abc'} + + +def envelope(payload): + return b'\xa2\x63seq\x01\x64data\x5a' + len(payload).to_bytes(4, 'big') + payload + + +def retain(tmp_path, payload): + archive = tmp_path / 'archive' + archive.mkdir(exist_ok=True) + (archive / '1.RAW.gz').write_bytes(gzip.compress(payload, mtime=0)) + store = RawArtifactStore(tmp_path / 'retained', RawLimits(4096, 65536, 8192, 1)) + return store, store.retain(archive, '1.RAW.gz') + + +def test_streams_verified_records_and_counts_placeholders_against_budget(tmp_path): + payload = b'\xa1\x64type\x61x' + store, descriptor = retain(tmp_path, envelope(b'') + envelope(payload) + envelope(payload)) + assert list(iter_retained_records(store, descriptor)) == [{'type': 'x'}, {'type': 'x'}] + with pytest.raises(ValueError, match='record count'): + list(iter_retained_records(store, descriptor, max_records=2)) + + +@pytest.mark.parametrize('tail', [b'\xa2', b'\xa2\x63seq\x01', envelope(b'abc')[:-1]]) +def test_rejects_truncated_retained_outer_record_instead_of_silent_eof(tmp_path, tail): + store, descriptor = retain(tmp_path, tail) + with pytest.raises(ValueError, match='Truncated retained RAW framing'): + list(iter_retained_records(store, descriptor)) + + +def test_inventory_uses_retained_frames_without_inventing_missing_context(tmp_path): + payload = base64.b64decode(CAPTURE_FIXTURES['raw.sens.piezo']['payload_b64']) + store, descriptor = retain(tmp_path, envelope(payload) + envelope(payload)) + result = raw_records.inventory_retained(store, descriptor) + assert result['ordered_waveform_available'] is True + assert result['usable_dual_frames'] == 2 + assert result['duplicate_timestamps'] == 1 + assert result['first_event_ms'] == result['last_event_ms'] + assert result['channels']['left1']['samples'] == 1000 + assert result['channels']['right1']['samples'] == 1000 + assert result['pump_records'] == 0 + assert result['artifact_sha256'] == descriptor.sha256 + + +def test_inventory_exposes_missing_times_and_does_not_claim_replayability(tmp_path): + # Definite map: type=piezo-dual, no ts or sample channels. + payload = b'\xa1\x64type\x6apiezo-dual' + store, descriptor = retain(tmp_path, envelope(payload)) + result = raw_records.inventory_retained(store, descriptor) + assert result['invalid_timestamps'] == 1 + assert result['empty_channel_frames'] == 1 + assert result['ordered_waveform_available'] is False + assert result['first_event_ms'] is None + + +def piezo_record(timestamp, channel=b'\x01\x00\x00\x00'): + encoded_time = b'\x1a' + timestamp.to_bytes(4, 'big') + encoded_channel = b'\x59' + len(channel).to_bytes(2, 'big') + channel + return (b'\xa4\x64type\x6apiezo-dual\x62ts' + encoded_time + + b'\x65left1' + encoded_channel + b'\x66right1' + encoded_channel) + + +def test_inventory_reports_reversals_lost_markers_and_truncated_sample_bytes(tmp_path): + channel = b'\x01\x00\x00\x00\xff\xff\xff\x7f\x03\x00\x00\x00x' + store, descriptor = retain(tmp_path, envelope(piezo_record(100, channel)) + envelope(piezo_record(99, channel))) + result = raw_records.inventory_retained(store, descriptor) + assert result['timestamp_reversals'] == 1 + assert result['ordered_waveform_available'] is False + assert (result['first_event_ms'], result['last_event_ms']) == (99000, 100000) + assert result['channels']['left1'] == {'samples': 6, 'lost_markers': 2, 'truncated_tail_bytes': 2} + + +def test_inventory_rejects_channels_too_large_for_the_replay_driver(tmp_path): + store, descriptor = retain(tmp_path, envelope(piezo_record(100, bytes(20004)))) + result = raw_records.inventory_retained(store, descriptor) + assert result['oversize_channel_frames'] == 1 + assert result['ordered_waveform_available'] is False + + +def test_shared_budget_counts_empty_placeholders_across_artifact_streams(tmp_path): + store, descriptor = retain(tmp_path, envelope(b'') + envelope(b'')) + budget = raw_records.RawRecordBudget(3) + assert list(iter_retained_records(store, descriptor, budget=budget)) == [] + assert budget.consumed == 2 + with pytest.raises(ValueError, match='record count'): + list(iter_retained_records(store, descriptor, budget=budget)) + assert budget.consumed == 4 + with pytest.raises(ValueError, match='shared RAW'): + list(iter_retained_records(store, descriptor, budget=3)) diff --git a/modules/common/test_raw_sources.py b/modules/common/test_raw_sources.py new file mode 100644 index 000000000..7f04d34b4 --- /dev/null +++ b/modules/common/test_raw_sources.py @@ -0,0 +1,66 @@ +"""Inspection cannot substitute mutable archive names for frozen dataset bytes.""" +import gzip +import importlib.util +from pathlib import Path + +import pytest + +from common.biometrics import raw_records +from common.biometrics.raw_sources import ArchiveSource, archive_names +from common.biometrics.raw_artifacts import RawArtifactStore, RawLimits + +spec = importlib.util.spec_from_file_location('_source_raw_parser', Path(__file__).with_name('cbor_raw.py')) +parser = importlib.util.module_from_spec(spec) +spec.loader.exec_module(parser) + + +def archive(tmp_path, monkeypatch): + monkeypatch.setattr(raw_records, 'read_raw_record', parser.read_raw_record) + root = tmp_path / 'archive' + root.mkdir() + payload = b'\xa1\x64type\x61x' + encoded = b'\xa2\x63seq\x01\x64data\x48' + payload + (root / '1.RAW.gz').write_bytes(gzip.compress(encoded, mtime=0)) + return root + + +def test_inspects_without_retention_then_freezes_exact_selected_bytes(tmp_path, monkeypatch): + root = archive(tmp_path, monkeypatch) + source = ArchiveSource(root, '1.RAW.gz') + inspection = source.inspect() + assert inspection['retained'] is False + assert inspection['inventory']['decoded_records'] == 1 + assert sorted(p.name for p in tmp_path.iterdir()) == ['archive'] + store = RawArtifactStore(tmp_path / 'retained', RawLimits(4096, 65536, 8192, 1)) + saved = store.retain(root, '1.RAW.gz', expected=inspection['artifact']) + assert saved == inspection['artifact'] + (root / '1.RAW.gz').unlink() + assert list(raw_records.iter_retained_records(store, saved)) == [{'type': 'x'}] + + +def test_source_replacement_requires_reinspection_and_publishes_nothing(tmp_path, monkeypatch): + root = archive(tmp_path, monkeypatch) + source = ArchiveSource(root, '1.RAW.gz') + descriptor = source.inspect()['artifact'] + (root / '1.RAW.gz').write_bytes(gzip.compress(b'new archive', mtime=0)) + with pytest.raises(ValueError, match='changed since inspection'): + with source.open_raw(descriptor): + pass + store = RawArtifactStore(tmp_path / 'retained', RawLimits(4096, 65536, 8192, 1)) + with pytest.raises(ValueError, match='changed since inspection'): + store.retain(root, '1.RAW.gz', expected=descriptor) + assert not list(store.directory.glob('*.RAW.gz')) + assert not list(store.directory.glob('.part-*')) + + +def test_discovery_is_bounded_and_excludes_symlinks_and_nonarchives(tmp_path, monkeypatch): + root = archive(tmp_path, monkeypatch) + (root / '2.RAW.gz').symlink_to(root / '1.RAW.gz') + (root / 'partial.tmp').write_bytes(b'ignored') + assert archive_names(root) == ['1.RAW.gz'] + with pytest.raises(ValueError, match='entry limit'): + archive_names(root, maximum_entries=2) + with pytest.raises(ValueError): + ArchiveSource(root, '../outside.RAW.gz') + with pytest.raises(OSError): + ArchiveSource(root, '2.RAW.gz').inspect() diff --git a/modules/common/test_replay_resources.py b/modules/common/test_replay_resources.py new file mode 100644 index 000000000..3ff509354 --- /dev/null +++ b/modules/common/test_replay_resources.py @@ -0,0 +1,52 @@ +import builtins +import io +import sys +from types import SimpleNamespace + +import pytest + +from common.biometrics import replay_worker + + +def test_linux_peak_is_kernel_high_water_mark_in_bytes_without_resource(monkeypatch): + monkeypatch.setattr(sys, 'platform', 'linux') + opened = [] + + def read(path, mode, encoding): + opened.append((path, mode, encoding)) + return io.StringIO('Name:\tpython\nVmHWM:\t 12345 kB\nVmRSS:\t 100 kB\n') + + monkeypatch.setattr(builtins, 'open', read) + assert replay_worker.peak_rss_bytes() == 12345 * 1024 + assert opened == [('/proc/self/status', 'r', 'ascii')] + + +@pytest.mark.parametrize('text', ['', 'VmRSS: 12 kB\n', 'VmHWM: 0 kB\n', + 'VmHWM: 12 bytes\n', 'VmHWM: 12 kB\nVmHWM: 13 kB\n', + 'VmHWM: 99999999999999999999999 kB\n', 'x' * 16385]) +def test_missing_or_malformed_peak_is_unknown(monkeypatch, text): + monkeypatch.setattr(sys, 'platform', 'linux') + monkeypatch.setattr(builtins, 'open', lambda *args, **kwargs: io.StringIO(text)) + assert replay_worker.peak_rss_bytes() is None + + +def test_missing_procfs_is_unknown(monkeypatch): + monkeypatch.setattr(sys, 'platform', 'linux') + + def fail(*args, **kwargs): + raise FileNotFoundError() + + monkeypatch.setattr(builtins, 'open', fail) + assert replay_worker.peak_rss_bytes() is None + + +def test_darwin_peak_is_already_bytes(monkeypatch): + monkeypatch.setattr(sys, 'platform', 'darwin') + monkeypatch.setitem(sys.modules, 'resource', SimpleNamespace(RUSAGE_SELF=0, + getrusage=lambda who: SimpleNamespace(ru_maxrss=12345))) + assert replay_worker.peak_rss_bytes() == 12345 + + +def test_unknown_platform_does_not_guess_units(monkeypatch): + monkeypatch.setattr(sys, 'platform', 'other') + assert replay_worker.peak_rss_bytes() is None diff --git a/modules/common/test_shadow_build.py b/modules/common/test_shadow_build.py new file mode 100644 index 000000000..a1a4c5b9c --- /dev/null +++ b/modules/common/test_shadow_build.py @@ -0,0 +1,58 @@ +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest + +from common.biometrics import build_identity, shadow_build +from common.biometrics.config import config_hash + + +def test_shadow_identity_covers_shared_and_prospective_sources(): + result = shadow_build.shadow_build_identity() + assert result == shadow_build.shadow_build_identity() + assert result['hash'] == config_hash(result['identity']) + assert result['hash'] != build_identity.replay_build_identity()['hash'] + assert result['identity']['version'] == 'shadow-python-build-v1' + root = Path(shadow_build.__file__).resolve().parents[2] + assert set(result['identity']['sources']) == set(build_identity.SOURCE_FILES + shadow_build.SHADOW_SOURCE_FILES) + for name in shadow_build.SHADOW_SOURCE_FILES: + assert result['identity']['sources'][name] == hashlib.sha256((root / name).read_bytes()).hexdigest() + + +@pytest.mark.parametrize('name', shadow_build.SHADOW_SOURCE_FILES) +def test_each_shadow_source_change_invalidates_identity(name, monkeypatch): + before = shadow_build.shadow_build_identity()['hash'] + original = build_identity._digest_file + monkeypatch.setattr(build_identity, '_digest_file', lambda path: 'a' * 64 if str(path).endswith(name) else original(path)) + assert shadow_build.shadow_build_identity()['hash'] != before + + +def test_probe_emits_only_identity_json_and_never_imports_processor(): + env = {**os.environ, 'PYTHONPATH': str(Path(shadow_build.__file__).resolve().parents[2])} + result = subprocess.run([sys.executable, '-B', '-m', 'common.biometrics.shadow_build'], + capture_output=True, text=True, check=True, timeout=10, env=env) + assert json.loads(result.stdout) == shadow_build.shadow_build_identity() + assert result.stderr == '' + code = "from common.biometrics.shadow_build import shadow_build_identity; import sys; shadow_build_identity(); assert 'common.biometrics.shadow' not in sys.modules; assert 'numpy' not in sys.modules" + subprocess.run([sys.executable, '-B', '-c', code], check=True, timeout=10, env=env) + + +def test_missing_dependency_file_listing_fails_closed(monkeypatch): + original = shadow_build.importlib.metadata.distribution + + class WithoutFiles: + files = [] + + def __init__(self, name): + self.original = original(name) + + def __getattr__(self, name): + return getattr(self.original, name) + + monkeypatch.setattr(shadow_build.importlib.metadata, 'distribution', WithoutFiles) + with pytest.raises(ValueError, match='paths unavailable'): + shadow_build.shadow_build_identity() diff --git a/modules/common/test_shadow_process.py b/modules/common/test_shadow_process.py new file mode 100644 index 000000000..e52b0dfec --- /dev/null +++ b/modules/common/test_shadow_process.py @@ -0,0 +1,329 @@ +"""Kernel-limit startup must fail closed before numerical imports or input.""" +import builtins +import io +import subprocess +import sys +from types import SimpleNamespace + +import pytest + +from common.biometrics import shadow_process + + +def test_installs_address_limit_after_shared_cpu_limits(monkeypatch): + calls = [] + monkeypatch.setattr(sys, 'platform', 'linux') + monkeypatch.setattr(shadow_process, 'configure_process_limits', lambda cpu: calls.append(('cpu', cpu))) + monkeypatch.setitem(sys.modules, 'resource', SimpleNamespace(RLIMIT_AS=9, + setrlimit=lambda kind, values: calls.append(('as', kind, values)))) + shadow_process.configure_shadow_limits(31, 123456789) + assert calls == [('cpu', 31), ('as', 9, (123456789, 123456789))] + + +@pytest.mark.parametrize('budget', [0, -1, True, 1.5, '123', 9_007_199_254_740_992]) +def test_invalid_address_budget_cannot_start(monkeypatch, budget): + monkeypatch.setattr(sys, 'platform', 'linux') + monkeypatch.setattr(shadow_process, 'configure_process_limits', lambda cpu: pytest.fail('limits already changed')) + with pytest.raises(ValueError): + shadow_process.configure_shadow_limits(1, budget) + + +def test_unsupported_host_does_not_claim_enforcement(monkeypatch): + monkeypatch.setattr(sys, 'platform', 'darwin') + with pytest.raises(RuntimeError, match='Linux'): + shadow_process.configure_shadow_limits(1, 123) + + +def test_yocto_uses_fixed_prlimit_and_propagates_refusal(monkeypatch): + monkeypatch.setattr(sys, 'platform', 'linux') + monkeypatch.setattr(shadow_process, 'configure_process_limits', lambda cpu: None) + original = builtins.__import__ + def without_resource(name, *args, **kwargs): + if name == 'resource': + raise ModuleNotFoundError(name='resource') + return original(name, *args, **kwargs) + monkeypatch.setattr(builtins, '__import__', without_resource) + calls = [] + def execute(command, **kwargs): + calls.append((command, kwargs)) + monkeypatch.setattr(subprocess, 'run', execute) + shadow_process.configure_shadow_limits(1, 123456789) + assert calls == [(['/usr/bin/prlimit', '--pid', str(shadow_process.os.getpid()), '--as=123456789:123456789'], + dict(check=True, timeout=5, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL))] + def refuse(*args, **kwargs): + raise subprocess.CalledProcessError(1, args[0]) + monkeypatch.setattr(subprocess, 'run', refuse) + with pytest.raises(subprocess.CalledProcessError): + shadow_process.configure_shadow_limits(1, 123456789) + + +def test_failed_limits_never_import_processor_or_read_input(monkeypatch): + original = builtins.__import__ + def guard(name, *args, **kwargs): + if name.endswith('shadow_worker'): + pytest.fail('numerical processor imported before limits succeeded') + return original(name, *args, **kwargs) + monkeypatch.setattr(builtins, '__import__', guard) + def fail(*args): + raise OSError('private-path-or-frame') + monkeypatch.setattr(shadow_process, 'configure_parent_death', lambda pid: None) + monkeypatch.setattr(shadow_process, 'acquire_shadow_lock', lambda: 123) + monkeypatch.setattr(shadow_process, 'configure_wall_deadline', lambda ms: None) + monkeypatch.setattr(shadow_process.os, 'close', lambda fd: None) + monkeypatch.setattr(shadow_process, 'configure_shadow_limits', fail) + stderr = io.StringIO() + monkeypatch.setattr(sys, 'stderr', stderr) + monkeypatch.setattr(sys, 'stdin', None) + assert shadow_process.main(['--parent-pid', '123', '--wall-ms', '10000', '--cpu-seconds', '1', '--address-space-bytes', '123']) == 1 + assert stderr.getvalue() == 'Shadow process failed\n' + + +def test_successful_limits_precede_processor_import_and_run(monkeypatch): + calls = [] + monkeypatch.setattr(shadow_process, 'configure_parent_death', lambda pid: calls.append('parent')) + monkeypatch.setattr(shadow_process, 'acquire_shadow_lock', lambda: (calls.append('lock'), 123)[1]) + monkeypatch.setattr(shadow_process, 'configure_wall_deadline', lambda ms: calls.append(('wall', ms))) + monkeypatch.setattr(shadow_process.os, 'close', lambda fd: calls.append(('close', fd))) + monkeypatch.setattr(shadow_process, 'configure_shadow_limits', lambda *args: calls.append('limits')) + original = builtins.__import__ + def guarded(name, *args, **kwargs): + if name == 'shadow_worker': + assert calls == ['parent', 'lock', ('wall', 10000), 'limits'] + calls.append('import') + return SimpleNamespace(run=lambda source, destination: calls.append((source, destination))) + return original(name, *args, **kwargs) + monkeypatch.setattr(builtins, '__import__', guarded) + monkeypatch.setattr(sys, 'stdin', SimpleNamespace(buffer='source')) + monkeypatch.setattr(sys, 'stdout', SimpleNamespace(buffer='destination')) + assert shadow_process.main(['--parent-pid', '123', '--wall-ms', '10000', '--cpu-seconds', '1', '--address-space-bytes', '123']) == 0 + assert calls == ['parent', 'lock', ('wall', 10000), 'limits', 'import', ('source', 'destination'), ('close', 123)] + + +@pytest.mark.skipif(sys.platform != 'linux', reason='Kernel enforcement is Linux-only') +def test_real_linux_kernel_refuses_allocation_and_terminates_busy_worker(): + import os + from pathlib import Path + import signal + environment = {**os.environ, 'PYTHONPATH': str(Path(__file__).resolve().parent.parent)} + prefix = 'from common.biometrics.shadow_process import configure_shadow_limits\nconfigure_shadow_limits(1, 96 * 1024 * 1024)\n' + memory = subprocess.run([sys.executable, '-B', '-c', prefix + ''' +import resource +assert resource.getrlimit(resource.RLIMIT_AS) == (96 * 1024 * 1024,) * 2 +try: + bytearray(128 * 1024 * 1024) +except MemoryError: + print('allocation refused') +else: + raise AssertionError('allocation was not bounded') +'''], env=environment, capture_output=True, timeout=10) + assert memory.returncode == 0, memory.stderr + assert memory.stdout == b'allocation refused\n' + cpu = subprocess.run([sys.executable, '-B', '-c', prefix + 'while True: pass\n'], + env=environment, capture_output=True, timeout=10) + assert cpu.returncode == -signal.SIGXCPU + + +@pytest.mark.parametrize('parent', [0, -1, True, 1.5, '123', 2_147_483_648]) +def test_invalid_parent_pid_refused(monkeypatch, parent): + monkeypatch.setattr(sys, 'platform', 'linux') + with pytest.raises(ValueError): + shadow_process.configure_parent_death(parent) + + +@pytest.mark.parametrize('parents, result', [([122], 0), ([123, 122], 0), ([123], -1)]) +def test_parent_loss_and_kernel_refusal_fail_closed(monkeypatch, parents, result): + import ctypes + monkeypatch.setattr(sys, 'platform', 'linux') + values = iter(parents) + monkeypatch.setattr(shadow_process.os, 'getppid', lambda: next(values)) + def prctl(*args): + assert args == (1, shadow_process.signal.SIGKILL, 0, 0, 0) + return result + monkeypatch.setattr(ctypes, 'CDLL', lambda *args, **kwargs: SimpleNamespace(prctl=prctl)) + with pytest.raises((RuntimeError, OSError)): + shadow_process.configure_parent_death(123) + + +@pytest.mark.skipif(sys.platform != 'linux', reason='Parent-death signal is Linux-only') +def test_real_parent_death_kills_detached_shadow_worker(): + import os + import signal + import time + from pathlib import Path + environment = {**os.environ, 'PYTHONPATH': str(Path(__file__).resolve().parent.parent)} + child_code = "from common.biometrics.shadow_process import configure_parent_death; import os,sys,time; configure_parent_death(int(sys.argv[1])); print(os.getpid(), flush=True); time.sleep(60)" + parent_code = "import subprocess,sys,os,time; subprocess.Popen([sys.executable,'-B','-c',sys.argv[1],str(os.getpid())],start_new_session=True); time.sleep(60)" + parent = subprocess.Popen([sys.executable, '-B', '-c', parent_code, child_code], + env=environment, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + worker_pid = None + try: + import select + ready, _, _ = select.select([parent.stdout], [], [], 10) + assert ready, 'Worker did not arm parent-death protection' + worker_pid = int(parent.stdout.readline()) + parent.kill() + parent.wait(timeout=5) + deadline = time.monotonic() + 5 + while time.monotonic() < deadline: + try: + state = Path(f'/proc/{worker_pid}/stat').read_text().rsplit(')', 1)[1].split()[0] + except (FileNotFoundError, ProcessLookupError): + # The task can vanish after procfs open but before read (ESRCH). + break + if state == 'Z': + break # dead; reaping belongs to the container init/subreaper + time.sleep(0.02) + else: + raise AssertionError('Worker survived host death') + worker_pid = None + finally: + if parent.poll() is None: + parent.kill() + parent.wait(timeout=5) + if worker_pid is not None: + try: + os.kill(worker_pid, signal.SIGKILL) + except ProcessLookupError: + pass + parent.stdout.close() + parent.stderr.close() + + +def test_parent_protection_failure_prevents_limits_and_processing(monkeypatch): + def fail(pid): + raise OSError('private parent protection detail') + monkeypatch.setattr(shadow_process, 'configure_parent_death', fail) + monkeypatch.setattr(shadow_process, 'configure_shadow_limits', lambda *args: pytest.fail('limits reached after parent protection failed')) + monkeypatch.setattr(sys, 'stdin', None) + stderr = io.StringIO() + monkeypatch.setattr(sys, 'stderr', stderr) + assert shadow_process.main(['--parent-pid', '123', '--wall-ms', '10000', '--cpu-seconds', '1', '--address-space-bytes', '123']) == 1 + assert stderr.getvalue() == 'Shadow process failed\n' + + +@pytest.mark.parametrize('kind', ['symlink', 'hardlink', 'public', 'content']) +def test_unsafe_lock_files_refused(tmp_path, monkeypatch, kind): + import os + monkeypatch.setattr(sys, 'platform', 'linux') + path = tmp_path / 'worker.lock' + if kind == 'symlink': + target = tmp_path / 'target' + target.touch(mode=0o600) + path.symlink_to(target) + else: + path.touch(mode=0o600) + if kind == 'hardlink': + os.link(path, tmp_path / 'alias') + if kind == 'public': + path.chmod(0o644) + if kind == 'content': + path.write_text('not a lock') + with pytest.raises((OSError, RuntimeError)): + shadow_process.acquire_shadow_lock(path) + + +def test_lock_contention_and_release_do_not_delete_runtime_file(tmp_path, monkeypatch): + import os + monkeypatch.setattr(sys, 'platform', 'linux') + path = tmp_path / 'worker.lock' + first = shadow_process.acquire_shadow_lock(path) + try: + with pytest.raises(BlockingIOError): + shadow_process.acquire_shadow_lock(path) + finally: + os.close(first) + before = path.stat().st_ino + second = shadow_process.acquire_shadow_lock(path) + os.close(second) + assert path.stat().st_ino == before + assert path.read_bytes() == b'' + + +@pytest.mark.skipif(sys.platform != 'linux', reason='Kernel execution lock is Linux-only') +def test_real_linux_lock_survives_pause_and_releases_after_worker_death(): + import os + import select + import signal + import tempfile + from pathlib import Path + from common.biometrics.shadow_process import acquire_shadow_lock + environment = {**os.environ, 'PYTHONPATH': str(Path(__file__).resolve().parent.parent)} + with tempfile.TemporaryDirectory() as directory: + path = str(Path(directory) / 'worker.lock') + code = "from common.biometrics.shadow_process import acquire_shadow_lock; import sys,time; fd=acquire_shadow_lock(sys.argv[1]); print('locked',flush=True); time.sleep(60)" + worker = subprocess.Popen([sys.executable, '-B', '-c', code, path], env=environment, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + try: + ready, _, _ = select.select([worker.stdout], [], [], 10) + assert ready and worker.stdout.readline() == 'locked\n' + os.kill(worker.pid, signal.SIGSTOP) + try: + fd = acquire_shadow_lock(path) + except BlockingIOError: + pass + else: + os.close(fd) + raise AssertionError('Paused worker lost exclusive execution lock') + worker.kill() + worker.wait(timeout=5) + fd = acquire_shadow_lock(path) + os.close(fd) + finally: + if worker.poll() is None: + worker.kill() + worker.wait(timeout=5) + worker.stdout.close() + worker.stderr.close() + + +def test_lock_contention_prevents_numerical_startup(monkeypatch): + monkeypatch.setattr(shadow_process, 'configure_parent_death', lambda pid: None) + def busy(): + raise BlockingIOError('another worker owns the slot') + monkeypatch.setattr(shadow_process, 'acquire_shadow_lock', busy) + monkeypatch.setattr(shadow_process, 'configure_shadow_limits', lambda *args: pytest.fail('limits reached without execution slot')) + monkeypatch.setattr(sys, 'stdin', None) + stderr = io.StringIO() + monkeypatch.setattr(sys, 'stderr', stderr) + assert shadow_process.main(['--parent-pid', '123', '--wall-ms', '10000', '--cpu-seconds', '1', '--address-space-bytes', '123']) == 1 + assert stderr.getvalue() == 'Shadow process failed\n' + + +@pytest.mark.parametrize('budget', [0, -1, True, 1.5, '1000', 7 * 86_400_000 + 300_001]) +def test_invalid_worker_wall_budget_refused(monkeypatch, budget): + monkeypatch.setattr(sys, 'platform', 'linux') + with pytest.raises(ValueError): + shadow_process.configure_wall_deadline(budget) + + +def test_worker_wall_timer_uses_default_termination_signal(monkeypatch): + monkeypatch.setattr(sys, 'platform', 'linux') + calls = [] + monkeypatch.setattr(shadow_process.signal, 'signal', lambda *args: calls.append(args)) + monkeypatch.setattr(shadow_process.signal, 'pthread_sigmask', lambda *args: calls.append(args)) + monkeypatch.setattr(shadow_process.signal, 'setitimer', lambda *args: calls.append(args)) + shadow_process.configure_wall_deadline(1234) + assert calls == [(shadow_process.signal.SIGALRM, shadow_process.signal.SIG_DFL), + (shadow_process.signal.SIG_UNBLOCK, {shadow_process.signal.SIGALRM}), + (shadow_process.signal.ITIMER_REAL, 1.234)] + + +@pytest.mark.skipif(sys.platform != 'linux', reason='Worker wall timer is Linux-only') +def test_real_worker_wall_timer_terminates_idle_process_and_releases_slot(): + import os + import signal + import tempfile + import time + from pathlib import Path + from common.biometrics.shadow_process import acquire_shadow_lock + environment = {**os.environ, 'PYTHONPATH': str(Path(__file__).resolve().parent.parent)} + with tempfile.TemporaryDirectory() as directory: + path = str(Path(directory) / 'worker.lock') + code = "from common.biometrics.shadow_process import acquire_shadow_lock,configure_wall_deadline; import sys,time,signal; fd=acquire_shadow_lock(sys.argv[1]); signal.pthread_sigmask(signal.SIG_BLOCK,{signal.SIGALRM}); configure_wall_deadline(100); time.sleep(60)" + started = time.monotonic() + result = subprocess.run([sys.executable, '-B', '-c', code, path], env=environment, + capture_output=True, timeout=5) + assert result.returncode == -signal.SIGALRM + assert time.monotonic() - started < 5 + fd = acquire_shadow_lock(path) + os.close(fd) diff --git a/modules/common/test_worker_limits.py b/modules/common/test_worker_limits.py new file mode 100644 index 000000000..e922c5856 --- /dev/null +++ b/modules/common/test_worker_limits.py @@ -0,0 +1,81 @@ +import builtins +import subprocess +import sys +from types import SimpleNamespace + +import pytest + +from common.biometrics import replay_worker + + +def without_resource(monkeypatch): + original = builtins.__import__ + + def load(name, *args, **kwargs): + if name == 'resource': + raise ModuleNotFoundError("No module named 'resource'", name='resource') + return original(name, *args, **kwargs) + + monkeypatch.setattr(builtins, '__import__', load) + monkeypatch.setattr(replay_worker.sys, 'platform', 'linux') + monkeypatch.setattr(replay_worker.os, 'nice', lambda value: None) + + +def test_yocto_worker_sets_same_limits_on_its_own_pid(monkeypatch): + without_resource(monkeypatch) + calls = [] + monkeypatch.setattr(replay_worker.subprocess, 'run', lambda *args, **kwargs: calls.append((args, kwargs))) + replay_worker.configure_process_limits() + assert calls == [((['/usr/bin/prlimit', '--pid', str(replay_worker.os.getpid()), + '--core=0:0', '--cpu=120:121', '--nofile=64:64'],), { + 'check': True, 'timeout': 5, 'stdin': subprocess.DEVNULL, + 'stdout': subprocess.DEVNULL, 'stderr': subprocess.DEVNULL})] + + +@pytest.mark.parametrize('failure', [FileNotFoundError(), + subprocess.CalledProcessError(1, 'prlimit'), subprocess.TimeoutExpired('prlimit', 5)]) +def test_missing_or_failed_limit_utility_fails_closed(monkeypatch, failure): + without_resource(monkeypatch) + + def fail(*args, **kwargs): + raise failure + + monkeypatch.setattr(replay_worker.subprocess, 'run', fail) + with pytest.raises(type(failure)): + replay_worker.configure_process_limits() + + +def test_missing_resource_on_other_platform_does_not_use_linux_utility(monkeypatch): + without_resource(monkeypatch) + monkeypatch.setattr(replay_worker.sys, 'platform', 'darwin') + with pytest.raises(ModuleNotFoundError): + replay_worker.configure_process_limits() + + +@pytest.mark.parametrize('cpu_seconds', [120, 1800, 3600]) +def test_standard_resource_extension_sets_limits_without_external_command(monkeypatch, cpu_seconds): + limits = [] + monkeypatch.setitem(sys.modules, 'resource', SimpleNamespace(RLIMIT_CORE=4, + RLIMIT_CPU=0, RLIMIT_NOFILE=7, setrlimit=lambda key, value: limits.append((key, value)))) + monkeypatch.setattr(replay_worker.os, 'nice', lambda value: None) + + def unexpected(*args, **kwargs): + pytest.fail('The standard resource path must not spawn a utility') + + monkeypatch.setattr(replay_worker.subprocess, 'run', unexpected) + replay_worker.configure_process_limits(cpu_seconds) + assert limits == [(4, (0, 0)), (0, (cpu_seconds, cpu_seconds + 1)), (7, (64, 64))] + + +def test_configured_cpu_limit_reaches_yocto_utility(monkeypatch): + without_resource(monkeypatch) + calls = [] + monkeypatch.setattr(replay_worker.subprocess, 'run', lambda args, **kwargs: calls.append(args)) + replay_worker.configure_process_limits(1800) + assert '--cpu=1800:1801' in calls[0] + + +@pytest.mark.parametrize('value', [True, 0, -1, 1.5, 3601, float('inf'), None]) +def test_invalid_cpu_limit_fails_before_changing_process_limits(value): + with pytest.raises(ValueError, match='CPU budget'): + replay_worker.configure_process_limits(value) diff --git a/modules/piezo-processor/fixtures/default-processing-v1.json b/modules/piezo-processor/fixtures/default-processing-v1.json new file mode 100644 index 000000000..37f13a3aa --- /dev/null +++ b/modules/piezo-processor/fixtures/default-processing-v1.json @@ -0,0 +1,27 @@ +{ + "sourceCommit": "3e9027ed", + "numpy": "2.2.6", + "scipy": "1.15.3", + "seed": 20261012, + "sampleRateHz": 500, + "hr": [ + 66.51884700665188, + 0.9522735382682044 + ], + "breathing": 14.936519790888722, + "hrvIndex": 12.937515617181978, + "tracker": [ + 60, + 61, + 60.0, + 62, + 65.0, + 65, + 50.0, + 50.0, + 50.0, + 50.5, + null, + 50 + ] +} diff --git a/modules/piezo-processor/main.py b/modules/piezo-processor/main.py index 325d460a3..7a55ada15 100644 --- a/modules/piezo-processor/main.py +++ b/modules/piezo-processor/main.py @@ -36,7 +36,6 @@ import threading from pathlib import Path from datetime import datetime, timezone -from collections import deque from dataclasses import dataclass from typing import Callable, Optional @@ -48,7 +47,23 @@ from common.side_mode import SingleSleeperMode from common.bed_presence import BedPresence import numpy as np -from scipy.signal import butter, sosfiltfilt, hilbert, find_peaks +from runtime_adapter import RuntimeCoordinator, RunContext +from common.biometrics.frames import decode_int32_samples as _int32_samples, LOST_SAMPLE +from common.biometrics.processing import ( + StatefulSideProcessor, HR_WINDOW_S, BREATHING_WINDOW_S, HRV_WINDOW_S, +) +from common.biometrics.routing import SingleSleeperRouter, VITALS_INTERVAL_S +from common.biometrics.gates import ( + FrzHealthPumpState as _FrzHealthPumpState, PumpGate as _PumpGate, + PresenceDetector, PUMP_ENERGY_MULTIPLIER, PUMP_CORRELATION_MIN, + PUMP_GUARD_S, PUMP_ACTIVE_RPM_MIN, PUMP_OFF_GUARD_S, + PUMP_COUPLING_STD_FACTOR, PUMP_COUPLING_ACR_THRESHOLD, +) +from common.biometrics.signal import ( + _bandpass, _autocorr_quality, _compute_autocorr, subharmonic_summation_hr, + HRTracker, compute_breathing_rate, compute_hrv, + ProcessingParameters, DEFAULT_PARAMETERS, WindowEstimator, +) # --------------------------------------------------------------------------- # Configuration @@ -62,26 +77,6 @@ "SLEEP_DETECTOR_STATE_PATH", str(BIOMETRICS_DB.parent / "sleep-detector-state.json"))) SAMPLE_RATE = 500 # Hz — piezo sensor sample rate -VITALS_INTERVAL_S = 60 # write a vitals row every N seconds -HR_WINDOW_S = 30 # seconds of data for heart rate calculation -BREATHING_WINDOW_S = 60 # seconds of data for breathing rate calculation -HRV_WINDOW_S = 300 # seconds of data for HRV (5-minute RMSSD) - -# Pump gating -PUMP_ENERGY_MULTIPLIER = 10.0 -PUMP_CORRELATION_MIN = 0.5 -PUMP_GUARD_S = 5.0 - -# Per-side pump activity threshold (frzHealth pump RPM). PumpGate catches -# broadband-symmetric energy spikes but misses two steady-state coupling -# modes: asymmetric (one pump on, observed 2026-05-02 Pod 3 — pump@2000RPM -# on right with left idle drove phantom HR/HRV/BR) and symmetric (both -# pumps on, observed 2026-05-05 Pod 5 — 1940/2004 rpm beat at 64/min lands -# in the cardiac band). Both are caught by the guard below. -PUMP_ACTIVE_RPM_MIN = 50 # pump considered "running" above this -PUMP_OFF_GUARD_S = 5.0 # trailing guard after own-side pump-off -PUMP_COUPLING_STD_FACTOR = 4.0 # raise enter_threshold by this factor -PUMP_COUPLING_ACR_THRESHOLD = 0.6 # require strong autocorr to enter # HR band: 0.8 Hz preserves fundamental of 48+ BPM; 8.5 Hz per PMC7582983 HR_BAND = (0.8, 8.5) @@ -172,7 +167,8 @@ def write_vitals(holder: "DBHolder", side: str, ts: datetime, breathing_rate: Optional[float], quality_score: float, flags: Optional[list] = None, - hr_raw: Optional[float] = None) -> bool: + hr_raw: Optional[float] = None, + run_context: Optional[RunContext] = None) -> bool: """Insert one vitals row + paired vitals_quality row. Logs and swallows sqlite3 errors so the main loop survives transient WAL/disk issues (#325). After _DB_RECONNECT_THRESHOLD consecutive failures, the holder's @@ -187,10 +183,21 @@ def write_vitals(holder: "DBHolder", side: str, ts: datetime, flags_json = json.dumps(flags) if flags else None try: with conn: - cur = conn.execute( - "INSERT INTO vitals (side, timestamp, heart_rate, hrv, breathing_rate) VALUES (?, ?, ?, ?, ?)", - (side, ts_unix, heart_rate, hrv, breathing_rate), - ) + if run_context is not None: + # Keep the profile and source channel that computed the value, + # even if single-sleeper mode changed while this row was held. + if side != run_context.side: + raise sqlite3.IntegrityError("Vitals routing disagrees with processing provenance") + run_context.ensure(conn) + cur = conn.execute( + "INSERT INTO vitals (side, timestamp, heart_rate, hrv, breathing_rate, processing_run_id) VALUES (?, ?, ?, ?, ?, ?)", + (side, ts_unix, heart_rate, hrv, breathing_rate, run_context.id), + ) + else: + cur = conn.execute( + "INSERT INTO vitals (side, timestamp, heart_rate, hrv, breathing_rate) VALUES (?, ?, ?, ?, ?)", + (side, ts_unix, heart_rate, hrv, breathing_rate), + ) conn.execute( "INSERT INTO vitals_quality (vitals_id, side, timestamp, quality_score, flags, hr_raw, created_at) " "VALUES (?, ?, ?, ?, ?, ?, ?)", @@ -219,91 +226,25 @@ class VitalsCandidate: quality_score: float flags: Optional[list] hr_raw: Optional[float] + run_context: Optional[RunContext] = None def write(self, holder: "DBHolder", side: Optional[str] = None) -> bool: - return write_vitals(holder, side or self.side, self.ts, self.heart_rate, + # Provenance captures the logical side at computation, not at retry. + destination = self.run_context.side if self.run_context else side or self.side + kwargs = {"run_context": self.run_context} if self.run_context else {} + return write_vitals(holder, destination, self.ts, self.heart_rate, self.hrv, self.breathing_rate, quality_score=self.quality_score, flags=self.flags, - hr_raw=self.hr_raw) - + hr_raw=self.hr_raw, **kwargs) -class SingleSleeperVitals: - """Routes vitals rows when one side is in away mode (single sleeper). - - A solo sleeper who rolls onto the away side is picked up by that side's - piezo too — the same heartbeat, within a few bpm. Each cycle's - candidates from the two sides are paired and - only the higher-quality one is written, under the home side, so the - home session shows one heart-rate series with no gaps or duplicates. - A candidate whose partner doesn't arrive within VITALS_INTERVAL_S (the - other side saw no one) is written alone under the home side. - - Outside single-sleeper mode rows are written under their own side. - """ +class SingleSleeperVitals(SingleSleeperRouter[VitalsCandidate]): + """Bind shared arbitration to the live database/provenance writer.""" def __init__(self, holder: "DBHolder", mode: SingleSleeperMode, clock: Callable[[], float] = time.monotonic): self._holder = holder - self._mode = mode - self._clock = clock - self._pending: Optional[VitalsCandidate] = None - self._retry: Optional[VitalsCandidate] = None - self._retry_home: Optional[str] = None - self._pending_home: Optional[str] = None - self._pending_at = 0.0 - - def submit(self, cand: VitalsCandidate) -> bool: - """Accept one side's candidate. True means the side may advance its - write cursor: the row was written, or is held (for pairing or a - write retry). A held candidate is only dropped once written.""" - # Completed intervals must never pair with a later candidate. - if self._retry is not None and not self.flush(): - return False - home = self._mode.home_side() - if not self._mode.active_sides(): - self._pending = self._retry = None - return True - if home is None: - if not self.flush(): - return False - return cand.write(self._holder) - pending = self._pending - if pending is not None and pending.side != cand.side and self._pending_home == home: - best = cand if cand.quality_score > pending.quality_score else pending - self._pending = best - self.flush() - return True - # A same-side repeat means its partner never arrived: write the - # older one first. If that fails, refuse the new candidate so its - # side keeps it and resubmits, rather than dropping the older row. - if not self.flush(): - return False - self._pending, self._pending_home, self._pending_at = cand, home, self._clock() - return True - - def tick(self) -> None: - """Write a held candidate whose partner didn't arrive in time.""" - if not self._mode.active_sides(): - self._pending = self._retry = None - return - held = self._pending is not None or self._retry is not None - if held and self._clock() - self._pending_at >= VITALS_INTERVAL_S: - self.flush() - - def flush(self) -> bool: - """Write the held candidate, if any. On failure it stays held and - idle retries wait VITALS_INTERVAL_S; a new submission retries first. - True when nothing is left held.""" - if self._retry is None: - self._retry, self._retry_home = self._pending, self._pending_home - self._pending = None - if self._retry is None: - return True - if self._retry.write(self._holder, side=self._retry_home): - self._retry = None - return True - self._pending_at = self._clock() - return False + super().__init__(mode, lambda candidate, side: candidate.write(holder, side=side), + clock=clock) def report_health(status: str, message: str) -> None: @@ -326,790 +267,37 @@ def report_health(status: str, message: str) -> None: except Exception as e: log.warning("Could not write health status: %s", e) -# --------------------------------------------------------------------------- -# Signal processing — filters -# --------------------------------------------------------------------------- - -def _bandpass(sig: np.ndarray, lo: float, hi: float, fs: float, - order: int = 4) -> np.ndarray: - """Bandpass filter using SOS form for numerical stability.""" - sos = butter(order, [lo, hi], btype='band', fs=fs, output='sos') - return sosfiltfilt(sos, sig) - # --------------------------------------------------------------------------- # Pump gating — streaming adaptation (Shin et al. IEEE TBME 2009) # --------------------------------------------------------------------------- -class FrzHealthPumpState: - """Tracks per-side pump RPM from frzHealth records. - - Used by SideProcessor to suppress phantom presence detection when - only the same side's pump is running (asymmetric pump-coupling - artifact). Trailing guard period mirrors the post-pump-off ringing - in the piezo channel. - """ - - def __init__(self): - self._pump_rpm = {"left": 0.0, "right": 0.0} - # monotonic seconds when each side's pump last turned off; -inf = - # never (0.0 would falsely assert the guard for the first - # PUMP_OFF_GUARD_S when time.monotonic() has a near-zero origin) - self._pump_off_at = {"left": float("-inf"), "right": float("-inf")} - self._was_active = {"left": False, "right": False} - - def update(self, record: dict) -> None: - if record.get("type") != "frzHealth": - return - for side in ("left", "right"): - data = record.get(side) - if not isinstance(data, dict): - continue - rpm = 0.0 - for key in ("pumpRpm", "pump_rpm", "pumpRPM", "rpm"): - val = data.get(key) - if val is not None: - try: - rpm = float(val) - except (TypeError, ValueError): - rpm = 0.0 - break - # New NATS firmware nests the same reading at side.pump.rpm. - pump = data.get("pump") - if rpm == 0 and isinstance(pump, dict): - val = pump.get("rpm") - if val is not None: - try: - rpm = float(val) - except (TypeError, ValueError): - rpm = 0.0 - if rpm == 0: - duty = next((data.get(key) for key in - ("pumpDuty", "pump_duty", "duty") - if data.get(key) is not None), None) - if duty is None and isinstance(pump, dict): - duty = next((pump.get(key) for key in ("duty", "power") - if pump.get(key) is not None), None) - if duty is not None: - try: - rpm = PUMP_ACTIVE_RPM_MIN + 1.0 if float(duty) > 0 else 0.0 - except (TypeError, ValueError): - rpm = 0.0 - - now_active = rpm >= PUMP_ACTIVE_RPM_MIN - if self._was_active[side] and not now_active: - self._pump_off_at[side] = time.monotonic() - self._pump_rpm[side] = rpm - self._was_active[side] = now_active - - def is_side_pump_active(self, side: str) -> bool: - """Pump active = currently running OR within the trailing guard window.""" - if self._pump_rpm[side] >= PUMP_ACTIVE_RPM_MIN: - return True - return time.monotonic() - self._pump_off_at[side] < PUMP_OFF_GUARD_S - - def is_asymmetric_for(self, side: str) -> bool: - """Own pump active, opposite pump idle (and outside its own guard). - This is the configuration that produces phantom presence: pump - vibration on the powered side without competing signal on the - other side to balance/reject.""" - other = "right" if side == "left" else "left" - return self.is_side_pump_active(side) and not self.is_side_pump_active(other) - - def is_symmetric_active(self) -> bool: - """Both pumps active. RPM mismatch (Pod 5 live: 1940/2004) produces - a beat frequency in the cardiac band, generating bilateral phantom - vitals. PumpGate catches symmetric spikes but not steady-state.""" - return self.is_side_pump_active("left") and self.is_side_pump_active("right") - - -class PumpGate: - """Detects pump activity from dual-channel energy spikes. - - In streaming mode, each incoming record's L+R chunks are checked. - If a pump is detected, a guard period is set during which all - subsequent records are dropped (not ingested into side buffers). - """ - - def __init__(self, fs: float = SAMPLE_RATE): - self._fs = fs - self._baseline: Optional[float] = None - self._pump_until: float = 0.0 # monotonic seconds when guard expires - - def is_pump_active(self) -> bool: - return time.monotonic() < self._pump_until - - def check(self, left_chunk: np.ndarray, right_chunk: np.ndarray) -> bool: - """Check a single record's L+R chunks for pump activity. - - Returns True if the record should be DROPPED (pump detected or - guard period active), False if the record is clean. - """ - # If we're still in the guard period, drop - if self.is_pump_active(): - return True - - # Compute energy for both channels - le = float(np.mean(left_chunk.astype(np.float64) ** 2)) - re = float(np.mean(right_chunk.astype(np.float64) ** 2)) - - if le == 0 or re == 0: - return False - - ratio = min(le, re) / max(le, re) - - if self._baseline is not None and self._baseline > 0: - spike = max(le, re) / self._baseline - if ratio > PUMP_CORRELATION_MIN and spike > PUMP_ENERGY_MULTIPLIER: - # Pump detected — set guard period - self._pump_until = time.monotonic() + PUMP_GUARD_S - log.debug("Pump detected (spike=%.1f, ratio=%.2f), " - "guard until +%.0fs", spike, ratio, PUMP_GUARD_S) - return True - - # Update baseline with exponential moving average (only on clean data). - # Guard: if first record has unusually high energy (module started during - # pump), don't initialize baseline from it — wait for a quieter sample. - avg = (le + re) / 2 - if self._baseline is None: - self._baseline = avg - self._baseline_samples = 1 - elif avg < self._baseline * 3: - self._baseline = 0.95 * self._baseline + 0.05 * avg - self._baseline_samples = getattr(self, '_baseline_samples', 0) + 1 - # If early samples are settling, allow baseline to drop faster - if self._baseline_samples < 10: - self._baseline = min(self._baseline, avg) - - return False - -# --------------------------------------------------------------------------- -# Presence detection — hysteresis state machine (Paalasmaa et al. 2012) -# --------------------------------------------------------------------------- - -def _autocorr_quality(sig: np.ndarray, fs: float = SAMPLE_RATE) -> float: - """Peak height of best autocorrelation peak in cardiac range. - - Person on bed produces periodic signal with high peak. - Empty bed produces noise with low peak. - """ - try: - filtered = _bandpass(sig, 0.8, 8.5, fs) - seg = filtered[:int(min(len(filtered), 10 * fs))] - seg = seg - np.mean(seg) - norm = np.std(seg) - if norm < 1e-10: - return 0.0 - seg = seg / norm - - n = len(seg) - fft_size = 1 - while fft_size < 2 * n: - fft_size *= 2 - fft_f = np.fft.rfft(seg, n=fft_size) - acr = np.fft.irfft(fft_f * np.conj(fft_f), n=fft_size)[:n] - if acr[0] == 0: - return 0.0 - acr = acr / acr[0] - - min_lag = int(fs * 60 / 150) - max_lag = min(int(fs * 60 / 40), len(acr) - 1) - if max_lag <= min_lag: - return 0.0 - return float(max(np.max(acr[min_lag:max_lag + 1]), 0.0)) - except Exception: - return 0.0 - - -class PresenceDetector: - """Hysteresis-based presence detection with autocorrelation quality. - - Uses median std of 1-10 Hz filtered signal in 5-second windows as - the primary feature, with autocorrelation quality as secondary. - Requires 3 consecutive low windows to exit PRESENT state. - - The enter/exit gap protects against an empty bed that vibrates (pump, - prime). When the capacitance sensor already confirms a body is in bed - (`relaxed=True`), that protection costs an hour of missed vitals per - night: a sleeper who rolls into a weak-signal position exits after three - windows and then has to move hard to re-enter. Relaxed re-entry uses the - exit criteria, so presence resumes as soon as the signal stops being weak. - """ - ABSENT = 0 - PRESENT = 1 - - def __init__(self): - self.state = self.ABSENT - self.consecutive_low = 0 - self.enter_threshold = 400_000 - self.exit_threshold = 150_000 - self.exit_count = 3 # 3 consecutive low windows to declare absent - self.acr_threshold = 0.45 - - def _is_weak(self, window_std: float, acr_qual: float) -> bool: - return window_std < self.exit_threshold and acr_qual < self.acr_threshold * 0.5 - - def update(self, window_std: float, acr_qual: float, relaxed: bool = False) -> bool: - """Update state and return True if user is present. - - `relaxed`: the bed is known occupied by another sensor; re-enter on - anything that would not count as an exit window.""" - if self.state == self.ABSENT: - if relaxed: - entered = not self._is_weak(window_std, acr_qual) - else: - entered = window_std > self.enter_threshold or acr_qual > self.acr_threshold - if entered: - self.state = self.PRESENT - self.consecutive_low = 0 - return True - return False - else: - if self._is_weak(window_std, acr_qual): - self.consecutive_low += 1 - if self.consecutive_low >= self.exit_count: - self.state = self.ABSENT - self.consecutive_low = 0 - return False - return True # still present (hysteresis) - else: - self.consecutive_low = 0 - return True - -# --------------------------------------------------------------------------- -# Heart rate — subharmonic summation autocorrelation (Hermes 1988; Bruser 2011) -# --------------------------------------------------------------------------- - -def _compute_autocorr(filtered: np.ndarray, fs: float) -> Optional[np.ndarray]: - """Compute normalised autocorrelation via FFT.""" - n = len(filtered) - fft_size = 1 - while fft_size < 2 * n: - fft_size *= 2 - fft_f = np.fft.rfft(filtered, n=fft_size) - acr = np.fft.irfft(fft_f * np.conj(fft_f), n=fft_size)[:n] - if acr[0] == 0: - return None - return acr / acr[0] - - -def subharmonic_summation_hr(samples: np.ndarray, fs: float = SAMPLE_RATE, - bpm_range: tuple = (45, 120), - n_harmonics: int = 3): - """Subharmonic summation for robust fundamental HR detection. - - Only scores autocorrelation PEAKS (not every lag) to avoid edge effects - where long lags accumulate noise. For each candidate peak lag L, - score = weighted sum of autocorr at L, L/2, L/3. - - Hermes 1988 (JASA), adapted for BCG by Bruser et al. 2011. - - Returns (heart_rate_bpm, score) or (None, 0.0) on failure. - """ - filtered = _bandpass(samples, HR_BAND[0], HR_BAND[1], fs) - acr = _compute_autocorr(filtered, fs) - if acr is None: - return None, 0.0 - - min_lag = int(fs * 60 / bpm_range[1]) - max_lag = int(fs * 60 / bpm_range[0]) - max_lag = min(max_lag, len(acr) - 1) - - if max_lag <= min_lag: - return None, 0.0 - - # Find actual peaks in autocorrelation (not every lag) - search = acr[min_lag:max_lag + 1] - peaks, _props = find_peaks(search, height=0.02, distance=int(fs * 0.15)) - if len(peaks) == 0: - return None, 0.0 - - candidate_lags = peaks + min_lag - weights = [1.0, 0.8, 0.6][:n_harmonics] - - best_lag = None - best_score = 0.0 - - for lag in candidate_lags: - score = 0.0 - for k in range(n_harmonics): - exact_lag = lag / (k + 1) - lag_lo = int(exact_lag) - lag_hi = lag_lo + 1 - if lag_hi >= len(acr): - continue - frac = exact_lag - lag_lo - val = acr[lag_lo] * (1 - frac) + acr[lag_hi] * frac - score += weights[k] * max(val, 0) - if score > best_score: - best_score = score - best_lag = lag - - if best_lag is None or best_score < 0.1: - return None, best_score - - hr = 60.0 * fs / best_lag - return hr, best_score - - -class HRTracker: - """Inter-window HR tracking with Gaussian consistency constraint. - - HR cannot jump > max_delta BPM between consecutive windows at rest. - Catches residual harmonic escapes via half/double correction. - Bruser et al. 2011. - """ - - # After this many consecutive uncorrectable readings, re-seed history at - # the new level. A lone outlier must not poison the median, but a - # sustained shift (e.g. different occupant) previously froze the tracker: - # the stale median rejected every new reading forever. - FALLBACK_RESET_STREAK = 3 - - def __init__(self, max_delta: float = 15.0, history_len: int = 5): - # Bounded deque: only the last history_len entries are ever consulted, - # so retaining the full list would leak ~1440 floats/day (#325). - self.history: deque = deque(maxlen=history_len) - self.max_delta = max_delta - self.history_len = history_len - self._fallback_streak = 0 - - def update(self, hr_candidate: Optional[float], - score: float) -> Optional[float]: - if hr_candidate is None: - return None - - if not self.history: - self.history.append(hr_candidate) - return hr_candidate - - recent = float(np.median(list(self.history))) - delta = abs(hr_candidate - recent) - - # Gaussian consistency weight - consistency = np.exp(-0.5 * (delta / self.max_delta) ** 2) - if consistency > 0.3: - self._fallback_streak = 0 - self.history.append(hr_candidate) - return hr_candidate - - # Try half (harmonic correction) - half_hr = hr_candidate / 2 - if 40 <= half_hr <= 120: - half_delta = abs(half_hr - recent) - half_consistency = np.exp(-0.5 * (half_delta / self.max_delta) ** 2) - if half_consistency > 0.3: - self._fallback_streak = 0 - self.history.append(half_hr) - return half_hr - - # Try double (sub-harmonic correction) - double_hr = hr_candidate * 2 - if 40 <= double_hr <= 180: - double_delta = abs(double_hr - recent) - double_consistency = np.exp( - -0.5 * (double_delta / self.max_delta) ** 2) - if double_consistency > 0.3: - self._fallback_streak = 0 - self.history.append(double_hr) - return double_hr - - # Accept anyway but keep a lone outlier out of history. A sustained - # streak of uncorrectable readings means the tracked level itself is - # stale (e.g. different occupant) — previously the frozen median then - # rejected every reading forever, so re-seed at the new level. - self._fallback_streak += 1 - if self._fallback_streak >= self.FALLBACK_RESET_STREAK: - self.history.clear() - self.history.append(hr_candidate) - self._fallback_streak = 0 - return hr_candidate - -# --------------------------------------------------------------------------- -# Breathing rate — Hilbert envelope (PMC9354426) -# --------------------------------------------------------------------------- - -def compute_breathing_rate(samples: np.ndarray, - fs: float = SAMPLE_RATE) -> Optional[float]: - """Breathing rate via Hilbert envelope of the cardiac band. - - Pipeline: bandpass 0.8-10 Hz -> Hilbert transform -> amplitude envelope - -> bandpass 0.1-0.7 Hz -> peak counting. - """ - try: - cardiac = _bandpass(samples, 0.8, 10.0, fs) - envelope = np.abs(hilbert(cardiac)) - resp = _bandpass(envelope, 0.1, 0.7, fs) - peaks, _ = find_peaks(resp, distance=int(2.0 * fs)) - if len(peaks) < 3: - return None - intervals = np.diff(peaks) / fs - med = np.median(intervals) - valid = intervals[(intervals > med * 0.5) & (intervals < med * 1.5)] - if len(valid) < 2: - return None - br = 60.0 / float(np.mean(valid)) - return br if 6 <= br <= 30 else None - except Exception as e: - log.debug("Breathing rate computation failed: %s", e) - return None - -# --------------------------------------------------------------------------- -# HR variability index — window-level IBI with harmonic gate -# -# NOTE: This is NOT clinical RMSSD. It computes successive differences of -# window-level IBI estimates (~59 per 5 min with 10s windows), not beat-to-beat -# intervals (~300+ per 5 min). The output measures HR stability across windows, -# not vagal beat-to-beat modulation. See #221 for the beat-level upgrade path. -# --------------------------------------------------------------------------- - -# Harmonic tolerance for IBI gating (18% per DSP expert recommendation) -_HRV_HARMONIC_TOL = 0.18 -# Sub-window length for IBI estimation (10s per DSP expert recommendation — -# gives ~59 windows per 5 min vs ~19 with 30s windows) -_HRV_WINDOW_S = 10 -# Minimum accepted IBI estimates after filtering -_HRV_MIN_IBIS = 10 -# Maximum gap between consecutive accepted windows before skipping the diff -_HRV_MAX_GAP_WINDOWS = 3 - - -def _harmonic_gate(ibi_ms: float, tracker_ibi_ms: float) -> Optional[float]: - """Reject or correct IBIs that are harmonic multiples of the expected value. - - Returns the accepted/corrected IBI in ms, or None to discard. - """ - if tracker_ibi_ms <= 0: - return ibi_ms # no prior yet — accept as-is - - for multiplier in [1.0, 2.0, 0.5, 3.0, 1.0 / 3.0]: - candidate = tracker_ibi_ms * multiplier - if candidate > 0 and abs(ibi_ms - candidate) / candidate < _HRV_HARMONIC_TOL: - if multiplier == 1.0: - return ibi_ms # fundamental — accept - else: - return tracker_ibi_ms # harmonic — substitute tracker value - return None # outside all harmonic relationships — discard - - -def compute_hrv(samples: np.ndarray, - fs: float = SAMPLE_RATE) -> Optional[float]: - """HR variability index via window-level IBI with harmonic gate. - - Pipeline: - 1. Bandpass 0.8-8.5 Hz - 2. 10s sub-windows with 50% overlap (~59 windows per 5 min) - 3. SHS autocorrelation per window → raw IBI - 4. Harmonic gate: reject/correct IBIs using trimmed-mean tracker - 5. Hampel filter on accepted IBI series - 6. Gap-aware RMSSD (skip diffs across rejected-window gaps) - 7. Range gate: 5-100 ms - - NOT clinical RMSSD — see module docstring and #221. - """ - try: - filtered = _bandpass(samples, HR_BAND[0], HR_BAND[1], fs) - sub_window = int(_HRV_WINDOW_S * fs) - step = sub_window // 2 - min_lag = int(fs * 60 / 150) - max_lag = int(fs * 60 / 40) - - # --- Pass 1: SHS autocorrelation per window → raw IBIs --- - raw_ibis: list = [] # (window_index, ibi_ms) - for idx, start in enumerate( - range(0, len(filtered) - sub_window + 1, step)): - chunk = filtered[start:start + sub_window] - acr = _compute_autocorr(chunk, fs) - if acr is None: - continue - search = acr[min_lag:min(max_lag + 1, len(acr))] - if len(search) == 0: - continue - - # SHS scoring — find peaks first, then score - peaks, _props = find_peaks(search, height=0.02, - distance=int(fs * 0.15)) - if len(peaks) == 0: - continue - - candidate_lags = peaks + min_lag - weights = [1.0, 0.8, 0.6] - best_lag, best_score = None, 0.0 - for lag in candidate_lags: - score = 0.0 - for k in range(3): - exact = lag / (k + 1) - lo_i = int(exact) - hi_i = lo_i + 1 - if hi_i >= len(acr): - continue - frac = exact - lo_i - val = acr[lo_i] * (1 - frac) + acr[hi_i] * frac - score += weights[k] * max(val, 0) - if score > best_score: - best_score = score - best_lag = lag - - if best_lag is None or best_score < 0.1: - continue - ibi_ms = best_lag / fs * 1000 - if 400 <= ibi_ms <= 1500: - raw_ibis.append((idx, ibi_ms)) - - if len(raw_ibis) < _HRV_MIN_IBIS: - return None - - # --- Pass 2: Harmonic gate with trimmed-mean tracker --- - tracker_history: list = [] - gated: list = [] # (window_index, ibi_ms) - - for win_idx, ibi_ms in raw_ibis: - # Compute tracker IBI as trimmed mean of last 5 accepted - if len(tracker_history) >= 3: - sorted_h = sorted(tracker_history[-5:]) - # Drop min+max if we have enough - if len(sorted_h) >= 4: - trimmed = sorted_h[1:-1] - else: - trimmed = sorted_h - tracker_ibi = sum(trimmed) / len(trimmed) - elif tracker_history: - tracker_ibi = sum(tracker_history) / len(tracker_history) - else: - tracker_ibi = 0 # no prior — accept first value - - accepted = _harmonic_gate(ibi_ms, tracker_ibi) - if accepted is not None: - gated.append((win_idx, accepted)) - tracker_history.append(accepted) - - if len(gated) < _HRV_MIN_IBIS: - return None - - # --- Pass 3: Hampel filter on gated IBI series --- - ibi_values = np.array([ibi for _, ibi in gated]) - win_indices = [idx for idx, _ in gated] - clean_mask = np.ones(len(ibi_values), dtype=bool) - - for i in range(len(ibi_values)): - lo_i = max(0, i - 3) - hi_i = min(len(ibi_values), i + 4) - local = ibi_values[lo_i:hi_i] - med = np.median(local) - mad = max(np.median(np.abs(local - med)), 1e-6) - if abs(ibi_values[i] - med) > 3.0 * 1.4826 * mad: - clean_mask[i] = False - - clean_ibis = ibi_values[clean_mask] - clean_indices = [win_indices[i] for i in range(len(win_indices)) - if clean_mask[i]] - - if len(clean_ibis) < _HRV_MIN_IBIS: - return None +class FrzHealthPumpState(_FrzHealthPumpState): + """Live adapter; replay imports the shared gate with its event clock.""" + def __init__(self, *, clock=None): + super().__init__(clock=clock if clock is not None else lambda: time.monotonic()) - # --- Pass 4: Gap-aware RMSSD --- - sq_diffs: list = [] - for i in range(1, len(clean_ibis)): - gap = clean_indices[i] - clean_indices[i - 1] - if gap >= _HRV_MAX_GAP_WINDOWS: - continue # skip diff across rejected-window gap - diff = clean_ibis[i] - clean_ibis[i - 1] - sq_diffs.append(diff ** 2) - if len(sq_diffs) < 5: - return None +class PumpGate(_PumpGate): + """Live adapter retaining the historical no-argument constructor.""" + def __init__(self, fs: float = SAMPLE_RATE, *, clock=None): + super().__init__(fs, clock=clock if clock is not None else lambda: time.monotonic()) - hrv_index = float(np.sqrt(np.mean(sq_diffs))) - # Range gate: 5-100 ms. Window-level HRV above 100 ms is artifact - # even after harmonic correction (cardiologist recommendation). - return hrv_index if 5 <= hrv_index <= 100 else None - except Exception as e: - log.debug("HRV computation failed: %s", e) - return None # --------------------------------------------------------------------------- # Per-side processor # --------------------------------------------------------------------------- -class SideProcessor: +class SideProcessor(StatefulSideProcessor): + """Live adapter binds shared state to wall time and canonical DB writes.""" def __init__(self, side: str, db_holder: DBHolder, - pump_state: Optional[FrzHealthPumpState] = None): - self.side = side + pump_state: Optional[FrzHealthPumpState] = None, + parameters: ProcessingParameters = DEFAULT_PARAMETERS): self.db_holder = db_holder - self._hr_buf: deque = deque(maxlen=HR_WINDOW_S * SAMPLE_RATE) - self._hrv_buf: deque = deque(maxlen=HRV_WINDOW_S * SAMPLE_RATE) - self._br_buf: deque = deque(maxlen=BREATHING_WINDOW_S * SAMPLE_RATE) - self._last_write = 0.0 - self._presence = PresenceDetector() - self._hr_tracker = HRTracker() - self._other: Optional['SideProcessor'] = None # set after both sides created - self._last_med_std: float = 0.0 # cached for cross-channel comparison - self._last_acr_qual: float = 0.0 - self._pump_state = pump_state - # Where computed vitals go; None writes them under this side. - # main() routes both sides through SingleSleeperVitals.submit. - self.sink: Optional[Callable[[VitalsCandidate], bool]] = None - # The bed's own answer to "is anyone here" (sleep-detector presence): - # False vetoes piezo presence, None (unknown) leaves it alone. - self.bed_occupied: Optional[Callable[[], Optional[bool]]] = None - # Single-sleeper home side (None in per-side mode). The sleep-detector - # merges both zones into the home side, so only the home side's - # processor may treat `bed_occupied() is True` as its own evidence. - self.home_side: Optional[Callable[[], Optional[str]]] = None - - def ingest(self, samples: np.ndarray) -> None: - self._hr_buf.extend(samples) - self._hrv_buf.extend(samples) - self._br_buf.extend(samples) - self._maybe_write() - - def _maybe_write(self) -> None: - now = time.time() - if now - self._last_write < VITALS_INTERVAL_S: - return - - hr_arr = np.array(self._hr_buf) - if len(hr_arr) < int(10 * SAMPLE_RATE): - return # Not enough data yet - - # --- Presence detection (hysteresis + autocorrelation quality) --- - filt = _bandpass(hr_arr, 1.0, 10.0, SAMPLE_RATE) - w = int(5 * SAMPLE_RATE) - stds = [np.std(filt[j:j + w]) - for j in range(0, len(filt) - w + 1, w)] - med_std = float(np.median(stds)) if stds else 0.0 - acr_qual = _autocorr_quality(hr_arr) - cross_channel_rejected = False - - # Cross-channel rejection: if the other side has stronger or similar - # signal AND strong autocorrelation (someone is actually there), this - # side's signal is likely vibration coupling, not a person. - # A real person produces ~3-5x stronger signal on their side. - # - # Note: due to sequential processing in main() (left before right), - # left sees right's values from the previous cycle (~60s stale). - # This is acceptable — coupling produces similar energy regardless - # of which side checks first. - if self._other is not None and self._other._last_med_std > 0: - other_std = self._other._last_med_std - other_acr = self._other._last_acr_qual - # Only suppress if: other side has real periodicity (someone there), - # other side's energy is >= ours, and our energy is below threshold - if (other_acr > 0.3 - and other_std > med_std * 0.7 - and med_std < self._presence.enter_threshold): - acr_qual = 0.0 - cross_channel_rejected = True - - # Cache AFTER suppression so the other side sees post-suppression values - self._last_med_std = med_std - self._last_acr_qual = acr_qual - - # The bed's own answer to "is anyone here" (sleep-detector capacitance - # presence), read once per cycle: True relaxes re-entry below, False - # vetoes piezo presence, None (unknown) leaves the piezo alone. - bed = self.bed_occupied() if self.bed_occupied is not None else None - # Only occupancy measured on this side may relax the piezo. With one - # side away the reading is merged under the home side, so the away - # side inherits a True it cannot attribute to its own zone and keeps - # the strict path. A side whose energy cross-channel rejection has - # already attributed to the other sleeper keeps it too. - own_side = self.home_side is None or self.home_side() in (None, self.side) - relaxed = bed is True and own_side and not cross_channel_rejected - - # Pump-coupling guard. PumpGate handles symmetric spikes (ramps) - # but misses two steady-state modes: asymmetric (own-pump-on with - # opposite idle — vibration couples into same-side piezo, no - # competing signal for cross-channel rejection) and symmetric - # (both pumps on at mismatched RPMs — beat frequency lands in the - # cardiac band, both sides look "real"). In both, require elevated - # energy AND strong autocorrelation to enter PRESENT; coupling - # rarely produces both, only a real person does. - # - # The guard answers "is this vibration a person or the pump". When - # the capacitance sensor says a person is in bed on this side and - # the energy is not the other sleeper's, that question is settled; - # on Pod 5 both pumps run all night, so without this bypass a - # sleeper who drifted to a weak-signal position stayed ABSENT for - # an hour at a time (05:57-07:01 on 2026-10-11) until a hard - # movement cleared the raised bar. - if (self._pump_state is not None and not relaxed - and self._presence.state == PresenceDetector.ABSENT): - symmetric = self._pump_state.is_symmetric_active() - asymmetric = self._pump_state.is_asymmetric_for(self.side) - if symmetric or asymmetric: - std_threshold = self._presence.enter_threshold * PUMP_COUPLING_STD_FACTOR - if not (med_std > std_threshold and acr_qual > PUMP_COUPLING_ACR_THRESHOLD): - log.debug( - "%s: pump-coupling guard suppressed presence " - "(med_std=%.0f, acr=%.2f, mode=%s)", - self.side, med_std, acr_qual, - "symmetric" if symmetric else "asymmetric", - ) - return - - if relaxed and self._presence.state == PresenceDetector.ABSENT: - log.debug("%s: bed reads occupied, relaxed piezo re-entry (med_std=%.0f, acr=%.2f)", - self.side, med_std, acr_qual) - if cross_channel_rejected: - # The other side owns this energy. Zeroing acr alone cannot end - # presence (exit needs weak energy too), so count the window as - # weak and let the normal hysteresis exit instead of latching, - # and never submit vitals computed from it meanwhile. - self._presence.update(0.0, 0.0) - present = False - else: - present = self._presence.update(med_std, acr_qual, relaxed=relaxed) - # Piezo presence is vibration energy and rhythm; a prime or the pump - # shaking an empty bed passes it. Trust the capacitance-based - # presence when the sleep-detector says nobody is there. - if present and bed is False: - present = False - - if not present: - # Reset the interval cursor so a return from extended absence - # doesn't trigger burst processing on every ingest until the - # first write completes (#325). - self._last_write = now - return # No user detected — skip - - # --- Heart rate (subharmonic summation + tracking) --- - hr_raw, hr_score = subharmonic_summation_hr(hr_arr) - hr = self._hr_tracker.update(hr_raw, hr_score) - - # --- Breathing rate (Hilbert envelope) --- - br = compute_breathing_rate(np.array(self._br_buf)) - - # --- HRV (sub-window autocorrelation IBI) --- - hrv_arr = np.array(self._hrv_buf) - hrv = compute_hrv(hrv_arr) if len(hrv_arr) >= int( - HRV_WINDOW_S * SAMPLE_RATE) else None - - if hr is not None or hrv is not None or br is not None: - ts = datetime.now(timezone.utc) - snr = max(0.0, min(1.0, acr_qual)) - hr_conf = max(0.0, min(1.0, hr_score)) if hr is not None else 0.0 - quality = round(0.55 * snr + 0.45 * hr_conf, 3) - flags = [] - if hr is None: - flags.append("no_hr") - if hrv is None: - flags.append("no_hrv") - if br is None: - flags.append("no_br") - if med_std < self._presence.enter_threshold: - flags.append("low_signal") - cand = VitalsCandidate(self.side, ts, hr, hrv, br, quality, flags or None, hr_raw) - wrote = self.sink(cand) if self.sink is not None else cand.write(self.db_holder) - log.info("vitals %s — HR=%.1f HRV=%.1f BR=%.1f q=%.2f", self.side, - hr or 0, hrv or 0, br or 0, quality) - # Only advance the downsample cursor when the write actually - # committed — otherwise a transient sqlite error would skip - # VITALS_INTERVAL_S of valid samples. - if wrote: - self._last_write = now - else: - # No metric to write — advance so we don't recompute on every ingest. - self._last_write = now + super().__init__(side, clock=lambda: time.time(), + writer=lambda candidate: candidate.write(db_holder), + candidate_factory=VitalsCandidate, pump_state=pump_state, + parameters=parameters, + autocorr_quality=lambda samples: _autocorr_quality(samples)) # --------------------------------------------------------------------------- # Main loop @@ -1120,35 +308,6 @@ def _maybe_write(self) -> None: # in a few percent of records. Taken as data it is a spike ~450x any real # signal: the pump gate drops the record (and 5 s after it), and the beat # tracker restarts. -LOST_SAMPLE = np.iinfo(np.int32).max - - -def _int32_samples(buf) -> np.ndarray: - """Decode an int32 sample buffer, tolerating truncated tails and - repairing lost-sample markers. - - A RAW record cut mid-write can carry a payload whose length is not a - multiple of 4; np.frombuffer would raise ValueError and kill the module. - Truncate to whole samples instead. A LOST_SAMPLE marker is replaced by - linear interpolation between its neighbours (the nearest valid sample at - a record edge); a record with no valid sample decodes as empty. - """ - if not isinstance(buf, (bytes, bytearray, memoryview)): - return np.empty(0, dtype=np.int32) - usable = len(buf) - (len(buf) % 4) - if usable == 0: - return np.empty(0, dtype=np.int32) - samples = np.frombuffer(buf[:usable], dtype=np.int32) - lost = samples == LOST_SAMPLE - if not lost.any(): - return samples - if lost.all(): - return np.empty(0, dtype=np.int32) - idx = np.arange(samples.size) - repaired = samples.astype(np.float64) - repaired[lost] = np.interp(idx[lost], idx[~lost], repaired[~lost]) - return np.round(repaired).astype(np.int32) - def main() -> None: log.info("Starting piezo-processor v2 (biometrics.db=%s)", BIOMETRICS_DB) @@ -1175,6 +334,8 @@ def main() -> None: left.bed_occupied = lambda: bed.occupied(bed_mode.home_side() or "left") right.bed_occupied = lambda: bed.occupied(bed_mode.home_side() or "right") left.home_side = right.home_side = bed_mode.home_side + runtime = RuntimeCoordinator(db_holder, {"left": left, "right": right}) + runtime_problem = None # Source selected once at startup: NatsFollower on new-firmware pods (NATS # reachable), else the unchanged .RAW tailer. Same decoded-record contract. follower = create_follower(RAW_DATA_DIR, _shutdown, poll_interval=0.01) @@ -1213,8 +374,16 @@ def main() -> None: if pump_gate.check(l_samples, r_samples): continue + frame_time_ms = int(time.time() * 1000) + runtime.before_frame(bed_mode.home_side(), bed.occupied, frame_time_ms) left.ingest(l_samples) right.ingest(r_samples) + runtime.after_frame(frame_time_ms) + if runtime.problem != runtime_problem: + runtime_problem = runtime.problem + report_health("degraded" if runtime_problem else "healthy", + "Biometric configuration fallback: " + runtime_problem if runtime_problem + else "Biometric runtime configuration available") except Exception as e: log.exception("Fatal error in main loop: %s", e) diff --git a/modules/piezo-processor/manifest.json b/modules/piezo-processor/manifest.json index e44925d26..5c3b3e651 100644 --- a/modules/piezo-processor/manifest.json +++ b/modules/piezo-processor/manifest.json @@ -2,8 +2,17 @@ "name": "piezo-processor", "version": "1.0.0", "description": "Heart rate, HRV, and breathing rate from piezoelectric sensors", - "provides": ["vitals.heartRate", "vitals.hrv", "vitals.breathingRate"], - "writes": ["vitals"], + "provides": [ + "vitals.heartRate", + "vitals.hrv", + "vitals.breathingRate" + ], + "writes": [ + "vitals", + "vitals_quality", + "biometric_processing_runs", + "biometric_runtime_receipts" + ], "service": "sleepypod-piezo-processor.service", "language": "python", "minVersion": "1.0.0" diff --git a/modules/piezo-processor/runtime_adapter.py b/modules/piezo-processor/runtime_adapter.py new file mode 100644 index 000000000..aa0276cec --- /dev/null +++ b/modules/piezo-processor/runtime_adapter.py @@ -0,0 +1,185 @@ +"""Live configuration adoption; no hardware control or desired-state writes.""" +from dataclasses import dataclass +import json +import logging +import sqlite3 +import time +from uuid import uuid4 + +from common.biometrics.config import config_hash, manifest +from common.biometrics.runtime import BoundarySelection, RuntimeSnapshot, read_latest_snapshot +from common.biometrics.signal import ProcessingParameters + +log = logging.getLogger(__name__) +COMPONENTS = ("heart_rate", "breathing_rate", "hrv_index") + + +@dataclass(frozen=True) +class RunContext: + id: str + side: str + source_side: str + bundle_json: str + config_hash: str + started_at_ms: int + + def ensure(self, connection): + """Called inside the observation transaction, including after DB reset.""" + connection.execute( + "INSERT OR IGNORE INTO biometric_processing_runs " + "(id, side, kind, bundle, config_hash, calibration_hash, input_identity, started_at_ms) " + "VALUES (?, ?, 'live', ?, ?, NULL, ?, ?)", + (self.id, self.side, self.bundle_json, self.config_hash, + json.dumps({"sourceSide": self.source_side, "timestampBasis": "processing_clock", + "rawIdentity": "not_retained", "sensorCalibration": "not_consumed"}), + self.started_at_ms)) + + +def make_context(side, source_side, snapshot, now_ms): + shipped = manifest() + resolved = snapshot.resolved if snapshot else shipped["defaults"] + bundle = {"schemaVersion": 1, "manifestHash": config_hash(shipped), + "resolved": resolved, "generation": snapshot.generation if snapshot else 0, + "revisionId": snapshot.revision_id if snapshot else None, + "components": shipped["components"], "sourceHashes": shipped["sourceHashes"]} + return RunContext(str(uuid4()), side, source_side, json.dumps(bundle), + config_hash(resolved), now_ms) + + +def restore_applied(connection, side): + """Only restore a projection acknowledged by all three numerical consumers.""" + rows = connection.execute( + "SELECT generation, applied_hash FROM biometric_runtime_receipts " + "WHERE side = ? AND state = 'applied' AND effective_sample_time_ms IS NOT NULL " + "AND component IN ('heart_rate', 'breathing_rate', 'hrv_index') " + "GROUP BY generation, applied_hash HAVING COUNT(DISTINCT component) = 3 " + "ORDER BY generation DESC LIMIT 1", (side,)).fetchone() + if rows is None: + return None + row = connection.execute( + "SELECT side, generation, resolved_hash, boundary, " + "CASE WHEN length(CAST(payload AS BLOB)) <= 65536 THEN payload ELSE NULL END " + "FROM biometric_runtime_configs WHERE side = ? AND generation = ?", + (side, rows[0])).fetchone() + if row is None: + raise ValueError("Applied runtime projection is missing") + snapshot = RuntimeSnapshot.parse(row[4]) + if (snapshot.side, snapshot.generation, snapshot.resolved_hash, snapshot.boundary) != tuple(row[:4]): + raise ValueError("Applied runtime projection envelope mismatch") + if (snapshot.side, snapshot.generation, snapshot.resolved_hash) != (side, rows[0], rows[1]): + raise ValueError("Applied runtime projection disagrees with receipts") + return snapshot + + +class RuntimeCoordinator: + def __init__(self, holder, processors, clock=time.monotonic, wall=time.time): + self.holder = holder + self.processors = processors + self.clock, self.wall = clock, wall + self.selections = {side: BoundarySelection(side) for side in ("left", "right")} + self.initialized = False + self.enabled = False + self.last_poll = float("-inf") + self.problem = None + self.effective_times = {} + self._frame_sides = set() + self._receipted = {} + + def _problem(self, error): + problem = str(error) + if problem != self.problem: + log.warning("Biometric runtime configuration unavailable; retaining actual compatible configuration: %s", problem) + self.problem = problem + + def _poll(self): + if self.clock() - self.last_poll < 5: + return + self.last_poll = self.clock() + try: + conn = self.holder.conn + # Activation requires both acknowledgment and provenance schema. + conn.execute("SELECT processing_run_id FROM vitals LIMIT 0") + conn.execute("SELECT id FROM biometric_processing_runs LIMIT 0") + conn.execute("SELECT component FROM biometric_runtime_receipts LIMIT 0") + problems = [] + for side, selection in self.selections.items(): + try: + if not self.initialized: + restored = restore_applied(conn, side) + selection = BoundarySelection(side, restored) + self.selections[side] = selection + snapshot = read_latest_snapshot(conn, side) + if snapshot: + selection.offer(snapshot) + except ValueError as error: + selection.pending = None + problems.append(f"{side}: {error}") + self.initialized = True + self.enabled = True + if problems: + self._problem("; ".join(problems)) + else: + self.problem = None + except sqlite3.Error as error: + for selection in self.selections.values(): + selection.pending = None + self._problem(error) + + def before_frame(self, home, occupied, now_ms): + """Install before accepting either physical channel's first new sample.""" + self._poll() + self._frame_sides = set() + if not self.enabled: + return + for side, selection in self.selections.items(): + if home is not None and side != home: + continue + ready = selection.ready(occupied(side)) + snapshot = ready or selection.applied + for source_side, processor in self.processors.items(): + logical_side = home or source_side + if logical_side != side: + continue + key = (side, snapshot) + if getattr(processor, "runtime_key", None) != key: + parameters = ProcessingParameters.from_config(snapshot.resolved if snapshot else manifest()["defaults"]) + processor.install_runtime(parameters, make_context(side, source_side, snapshot, now_ms), + reset_side=getattr(processor, "runtime_key", (None,))[0] != side) + processor.runtime_key = key + if ready: + selection.commit(ready) + self.effective_times[side] = now_ms + elif side not in self.effective_times: + # Restored config starts a new warmup after daemon restart. + self.effective_times[side] = now_ms + self._frame_sides.add(side) + + def after_frame(self, now_ms): + """Receipt only after both channels have accepted this frame successfully.""" + if not self.enabled: + return + try: + completed = {} + with self.holder.conn: + for side in self._frame_sides: + snapshot = self.selections[side].applied + if snapshot is None: + continue + prior = self._receipted.get(side) + identity = (snapshot.generation, snapshot.resolved_hash, self.effective_times[side]) + if prior and prior[0] == identity and self.clock() - prior[1] < 30: + continue + for component in COMPONENTS: + self.holder.conn.execute( + "INSERT INTO biometric_runtime_receipts " + "(side, component, generation, applied_hash, effective_sample_time_ms, state, error, heartbeat_at_ms) " + "VALUES (?, ?, ?, ?, ?, 'applied', NULL, ?) " + "ON CONFLICT(side, component, generation) DO UPDATE SET " + "applied_hash=excluded.applied_hash, effective_sample_time_ms=excluded.effective_sample_time_ms, " + "state='applied', error=NULL, heartbeat_at_ms=excluded.heartbeat_at_ms", + (side, component, snapshot.generation, snapshot.resolved_hash, + self.effective_times[side], now_ms)) + completed[side] = (identity, self.clock()) + self._receipted.update(completed) + except sqlite3.Error as error: + self._problem(error) diff --git a/modules/piezo-processor/test_default_golden.py b/modules/piezo-processor/test_default_golden.py new file mode 100644 index 000000000..785c0b383 --- /dev/null +++ b/modules/piezo-processor/test_default_golden.py @@ -0,0 +1,29 @@ +"""Pre-refactor baseline captured from dev 3e9027ed with the locked environment. + +A synthetic fixture establishes deterministic numerical parity, not clinical +accuracy. Existing artifact/presence/pump fixtures cover gating separately. +""" +import json +from pathlib import Path + +import numpy as np +import pytest + +from main import HRTracker, compute_breathing_rate, compute_hrv, subharmonic_summation_hr + + +def test_default_processing_golden(): + fixture = json.loads(Path(__file__).with_name("fixtures").joinpath("default-processing-v1.json").read_text()) + fs = fixture["sampleRateHz"] + t = np.arange(fs * 300) / fs + rng = np.random.default_rng(fixture["seed"]) + phase = 2 * np.pi * (1.1 * t + .2 * np.sin(2 * np.pi * t / 40)) + samples = ((1 + .2 * np.sin(2 * np.pi * .25 * t)) + * (np.sin(phase) + .5 * np.sin(2 * phase) + .25 * np.sin(3 * phase)) + + rng.normal(0, .08, len(t))) + assert subharmonic_summation_hr(samples[-30 * fs:]) == pytest.approx(fixture["hr"], abs=1e-9) + assert compute_breathing_rate(samples[-60 * fs:]) == pytest.approx(fixture["breathing"], abs=1e-9) + assert compute_hrv(samples) == pytest.approx(fixture["hrvIndex"], abs=1e-9) + tracker = HRTracker() + actual = [tracker.update(hr, 1.0) for hr in [60, 61, 120, 62, 130, 65, 100, 100, 100, 101, None, 50]] + assert actual == fixture["tracker"] diff --git a/modules/piezo-processor/test_main.py b/modules/piezo-processor/test_main.py index 6dad60520..4b4f63430 100644 --- a/modules/piezo-processor/test_main.py +++ b/modules/piezo-processor/test_main.py @@ -40,6 +40,7 @@ "common.bed_presence", Path(__file__).resolve().parent.parent / "common" / "bed_presence.py") _stubs["common.bed_presence"] = importlib.util.module_from_spec(_spec) _spec.loader.exec_module(_stubs["common.bed_presence"]) +_stubs["common"].__path__ = [str(Path(__file__).resolve().parent.parent / "common")] sys.modules.update(_stubs) from main import ( # noqa: E402 diff --git a/modules/piezo-processor/test_raw_publication.py b/modules/piezo-processor/test_raw_publication.py new file mode 100644 index 000000000..725d53bef --- /dev/null +++ b/modules/piezo-processor/test_raw_publication.py @@ -0,0 +1,238 @@ +"""Real migration/SQLite tests for complete-batch waveform publication.""" +from concurrent.futures import ThreadPoolExecutor +from copy import deepcopy +import gzip +import json +from pathlib import Path +import sqlite3 +import sys +from threading import Event +import uuid + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from common.biometrics.raw_artifacts import RawArtifactStore +from common.biometrics.raw_publication import publish_waveform_dataset +from test_retained_replay import fixture + + +def prepare(tmp_path): + source, inputs, _ = fixture(tmp_path) + archive = tmp_path / 'archive' + selections = [] + for index, item in enumerate(inputs['artifacts']): + name = f'{index}.RAW.gz' + (archive / name).write_bytes((source.directory / item['artifact']['name']).read_bytes()) + selections.append({'source_name': name, 'artifact': item['artifact']}) + store = RawArtifactStore(tmp_path / 'published', source.limits) + database = tmp_path / 'biometrics.db' + db = sqlite3.connect(database) + db.execute('PRAGMA foreign_keys = ON') + migrations = Path(__file__).resolve().parents[2] / 'src/db/biometrics-migrations' + for migration in sorted(migrations.glob('*.sql')): + db.executescript(migration.read_text()) + prepared = {'id': str(uuid.uuid4()), 'side': 'left', 'subjectId': 'fixture', 'idempotencyKey': 'waveform-one', + 'requestHash': 'a' * 64, 'contentHash': 'b' * 64, 'createdAtMs': 1000, + 'manifest': {'version': 'waveform-dataset-manifest-v1', 'splitProtocol': {'assignments': [ + {'cohortId': 'night', 'startMs': inputs['score_start_ms'], 'endMs': inputs['score_end_ms']}]}}, + 'snapshot': {'version': 'waveform-dataset-v1', 'side': 'left', 'subjectId': 'fixture', + 'sessions': [], 'references': [], 'processingRuns': [], 'waveformInputs': [{'cohortId': 'night', 'input': inputs}]}} + return store, db, database, archive, selections, prepared + + +def counts(db): + return [db.execute(f'SELECT count(*) FROM {name}').fetchone()[0] for name in + ('biometric_datasets', 'biometric_raw_artifacts', 'biometric_dataset_raw_artifacts', 'vitals')] + + +def test_publishes_exact_snapshot_with_all_ordered_references_and_idempotent_retry(tmp_path): + store, db, _, archive, selections, prepared = prepare(tmp_path) + try: + result = publish_waveform_dataset(store, db, archive, selections, prepared) + assert result['id'] == prepared['id'] and result['available'] + assert counts(db) == [1, 2, 2, 0] + snapshot = db.execute('SELECT snapshot FROM biometric_datasets').fetchone()[0] + assert json.loads(snapshot) == prepared['snapshot'] + assert [row[0] for row in db.execute('SELECT artifact_hash FROM biometric_dataset_raw_artifacts ORDER BY position')] == [item['artifact']['sha256'] for item in selections] + for name in archive.iterdir(): + name.unlink() + retry = {**prepared, 'id': str(uuid.uuid4()), 'createdAtMs': 2000} + assert publish_waveform_dataset(store, db, archive, selections, retry) == result + with pytest.raises(ValueError, match='idempotency key changed'): + publish_waveform_dataset(store, db, archive, selections, {**retry, 'requestHash': 'c' * 64}) + assert counts(db) == [1, 2, 2, 0] + finally: + db.close() + + +def test_failure_on_last_reference_rolls_back_every_database_write(tmp_path): + store, db, _, archive, selections, prepared = prepare(tmp_path) + try: + db.executescript("CREATE TRIGGER fail_last BEFORE INSERT ON biometric_dataset_raw_artifacts WHEN NEW.position = 1 BEGIN SELECT RAISE(ABORT, 'injected failure'); END;") + with pytest.raises(sqlite3.IntegrityError, match='injected failure'): + publish_waveform_dataset(store, db, archive, selections, prepared) + assert counts(db) == [0, 0, 0, 0] + assert len(list(store.directory.glob('*.RAW.gz'))) == 2 + assert not db.in_transaction + db.execute('DROP TRIGGER fail_last') + assert publish_waveform_dataset(store, db, archive, selections, prepared)['available'] + assert counts(db) == [1, 2, 2, 0] + finally: + db.close() + + +def test_changed_second_archive_never_publishes_a_partial_dataset(tmp_path): + store, db, _, archive, selections, prepared = prepare(tmp_path) + try: + (archive / selections[1]['source_name']).write_bytes(gzip.compress(b'changed archive')) + with pytest.raises(ValueError, match='changed since inspection'): + publish_waveform_dataset(store, db, archive, selections, prepared) + assert counts(db) == [0, 0, 0, 0] + assert len(list(store.directory.glob('*.RAW.gz'))) == 1 + assert not list(store.directory.glob('.part-*')) + finally: + db.close() + + +def test_released_retry_stays_released_and_a_new_dataset_can_retain_shared_bytes(tmp_path): + store, db, _, archive, selections, prepared = prepare(tmp_path) + try: + first = publish_waveform_dataset(store, db, archive, selections, prepared) + with db: + db.execute('UPDATE biometric_datasets SET released_at_ms = 2000, snapshot = ?', ('{"released":true}',)) + db.execute('UPDATE biometric_dataset_raw_artifacts SET released_at_ms = 2000') + db.execute('UPDATE biometric_raw_artifacts SET released_at_ms = 2000') + assert not publish_waveform_dataset(store, db, archive, selections, prepared)['available'] + assert db.execute('SELECT count(*) FROM biometric_raw_artifacts WHERE released_at_ms IS NULL').fetchone()[0] == 0 + for file in archive.iterdir(): + file.unlink() + archive.rmdir() + second = publish_waveform_dataset(store, db, archive, selections, {**prepared, + 'id': str(uuid.uuid4()), 'idempotencyKey': 'waveform-two', 'createdAtMs': 3000}) + assert second['id'] != first['id'] and second['available'] + assert counts(db) == [2, 2, 4, 0] + assert db.execute('SELECT count(*) FROM biometric_raw_artifacts WHERE released_at_ms IS NULL AND created_at_ms = 1000').fetchone()[0] == 2 + assert db.execute('SELECT count(*) FROM biometric_dataset_raw_artifacts WHERE released_at_ms = 2000').fetchone()[0] == 2 + finally: + db.close() + + +def test_rejects_mismatched_cohorts_and_sources_without_copies(tmp_path): + store, db, _, archive, selections, prepared = prepare(tmp_path) + try: + bad = deepcopy(prepared) + bad['snapshot']['waveformInputs'][0]['input']['score_start_ms'] += 1 + with pytest.raises(ValueError, match='whole-night cohort'): + publish_waveform_dataset(store, db, archive, selections, bad) + with pytest.raises(ValueError, match='match frozen'): + publish_waveform_dataset(store, db, archive, selections[:1], prepared) + db.execute('BEGIN') + with pytest.raises(ValueError, match='independent transaction'): + publish_waveform_dataset(store, db, archive, selections, prepared) + db.rollback() + assert counts(db) == [0, 0, 0, 0] + assert not list(store.directory.glob('*.RAW.gz')) + finally: + db.close() + + +def test_copying_does_not_hold_sqlite_write_lock_or_expose_partial_rows(tmp_path, monkeypatch): + store, db, database, archive, selections, prepared = prepare(tmp_path) + reached, proceed = Event(), Event() + original = store._retain_locked + + def delayed(root, name, expected): + if name == selections[1]['source_name']: + reached.set() + assert proceed.wait(5) + return original(root, name, expected) + + monkeypatch.setattr(store, '_retain_locked', delayed) + + def publish(): + connection = sqlite3.connect(database) + try: + connection.execute('PRAGMA foreign_keys = ON') + return publish_waveform_dataset(store, connection, archive, selections, prepared) + finally: + connection.close() + + try: + with ThreadPoolExecutor(max_workers=1) as pool: + future = pool.submit(publish) + try: + assert reached.wait(5) + assert counts(db) == [0, 0, 0, 0] + db.execute('BEGIN IMMEDIATE') + db.rollback() + finally: + proceed.set() + assert future.result(timeout=5)['available'] + assert counts(db) == [1, 2, 2, 0] + finally: + db.close() + + +def test_real_publication_entrypoint_commits_and_recovers_a_lost_reply_by_retry(tmp_path): + import subprocess + store, db, database, archive, selections, prepared = prepare(tmp_path) + payload = json.dumps({'version': 'waveform-publication-work-v1', 'prepared': prepared, 'selections': selections}).encode() + command = [sys.executable, '-B', '-m', 'common.biometrics.publication_worker', '--database', str(database), + '--archive-directory', str(archive), '--retained-directory', str(store.directory)] + env = {'PYTHONPATH': str(Path(__file__).resolve().parent.parent), 'LANG': 'C.UTF-8', 'TZ': 'UTC'} + try: + first = subprocess.run(command, input=payload, capture_output=True, env=env, cwd=tmp_path, timeout=20) + assert first.returncode == 0, first.stderr + assert first.stderr == b'' + assert json.loads(first.stdout) == {'type': 'result', 'id': prepared['id'], 'contentHash': prepared['contentHash'], 'available': True} + assert counts(db) == [1, 2, 2, 0] + for file in archive.iterdir(): + file.unlink() + retry = subprocess.run(command, input=payload, capture_output=True, env=env, cwd=tmp_path, timeout=20) + assert retry.returncode == 0 and retry.stdout == first.stdout + assert counts(db) == [1, 2, 2, 0] + finally: + db.close() + + +def test_sigkill_inside_reference_transaction_recovers_without_partial_publication(tmp_path): + import select + import subprocess + store, db, database, archive, selections, prepared = prepare(tmp_path) + request = tmp_path / 'request.json' + request.write_text(json.dumps({'prepared': prepared, 'selections': selections})) + db.executescript('CREATE TRIGGER pause_publication BEFORE INSERT ON biometric_dataset_raw_artifacts BEGIN SELECT pause_ref(); END;') + script = ''' +import json, sqlite3, sys, time +from common.biometrics.raw_artifacts import RawArtifactStore +from common.biometrics.raw_publication import publish_waveform_dataset +work=json.load(open(sys.argv[1])) +db=sqlite3.connect(sys.argv[2]); db.execute('PRAGMA foreign_keys = ON') +def pause(): + print('transaction-open', flush=True) + time.sleep(60) + return 0 +db.create_function('pause_ref', 0, pause) +publish_waveform_dataset(RawArtifactStore(sys.argv[4]), db, sys.argv[3], work['selections'], work['prepared']) +''' + child = subprocess.Popen([sys.executable, '-B', '-c', script, str(request), str(database), str(archive), str(store.directory)], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + env={'PYTHONPATH': str(Path(__file__).resolve().parent.parent), 'OPENBLAS_NUM_THREADS': '1'}) + try: + assert select.select([child.stdout], [], [], 10)[0] + assert child.stdout.readline() == b'transaction-open\n' + assert counts(db) == [0, 0, 0, 0] + child.kill() + child.communicate(timeout=5) + assert child.returncode < 0 + assert counts(db) == [0, 0, 0, 0] + assert len(list(store.directory.glob('*.RAW.gz'))) == 2 + db.execute('DROP TRIGGER pause_publication') + assert publish_waveform_dataset(store, db, archive, selections, prepared)['available'] + assert counts(db) == [1, 2, 2, 0] + finally: + if child.poll() is None: + child.kill() + child.communicate(timeout=5) + db.close() diff --git a/modules/piezo-processor/test_replay_worker.py b/modules/piezo-processor/test_replay_worker.py new file mode 100644 index 000000000..96cde2246 --- /dev/null +++ b/modules/piezo-processor/test_replay_worker.py @@ -0,0 +1,128 @@ +"""Real subprocess protocol for shipped retained waveform processing.""" +from copy import deepcopy +from io import BytesIO +import json +from pathlib import Path +import subprocess +import sys + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from common.biometrics.config import manifest +from common.biometrics import replay_worker +from test_retained_replay import fixture, run + + +def work(snapshot): + return {'version': 'retained-waveform-work-v1', 'input': snapshot, + 'configs': {side: deepcopy(manifest()['defaults']) for side in ('left', 'right')}} + + +def child(tmp_path, directory, payload): + return subprocess.run([sys.executable, '-c', + 'from common.biometrics import replay_worker; ' + 'from common.biometrics.execution_lock import acquire_execution_lock; ' + f'replay_worker.acquire_execution_lock=lambda: acquire_execution_lock({str(tmp_path / "execution.lock")!r}); ' + 'raise SystemExit(replay_worker.main())', + '--retained-directory', str(directory)], input=payload, capture_output=True, + cwd=tmp_path, env={'PYTHONPATH': str(Path(__file__).resolve().parent.parent), + 'LANG': 'C.UTF-8', 'TZ': 'UTC'}, timeout=20) + + +def test_real_worker_streams_provisional_readings_then_one_success_result(tmp_path): + store, snapshot, _ = fixture(tmp_path) + before = {entry.name: (entry.stat().st_mtime_ns, entry.stat().st_mode) for entry in store.directory.iterdir()} + expected_rows = [] + expected = run(store, snapshot, expected_rows.append) + completed = child(tmp_path, store.directory, json.dumps(work(snapshot)).encode()) + assert completed.returncode == 0, completed.stderr + assert completed.stderr == b'' + events = [json.loads(line) for line in completed.stdout.splitlines()] + assert len(events) == len(expected_rows) + 1 + from common.biometrics.build_identity import replay_build_identity + resources = events[-1]['result'].pop('resources') + assert resources['version'] == 'waveform-process-resources-v1' + assert resources['wallMs'] > 0 + assert resources['cpuMs'] > 0 + assert resources['peakRssBytes'] > 0 + assert resources['scope'] == 'python_process_until_result' + assert resources['memoryScope'] == 'process_lifetime_high_water_mark' + assert events[-1] == {'type': 'result', 'result': {**expected, 'build_hash': replay_build_identity()['hash']}} + for event, row in zip(events[:-1], expected_rows): + assert event['type'] == 'reading' + value = event['reading'] + assert value['heartRate'] == row.candidate.heart_rate + assert value['timestampMs'] == row.timestamp_ms + assert value['configHash'] == row.candidate.run_context.config_hash + assert value['hrvMetric'] == 'window_ibi_variability_v1' + assert value['side'] == row.side + assert {entry.name: (entry.stat().st_mtime_ns, entry.stat().st_mode) for entry in store.directory.iterdir()} == before + + +def test_late_missing_artifact_has_no_success_event_or_sensitive_diagnostics(tmp_path): + store, snapshot, _ = fixture(tmp_path, split_at=136) + (store.directory / snapshot['artifacts'][1]['artifact']['name']).unlink() + completed = child(tmp_path, store.directory, json.dumps(work(snapshot)).encode()) + assert completed.returncode == 1 + assert completed.stderr == b'Retained waveform worker failed\n' + assert all(json.loads(line)['type'] == 'reading' for line in completed.stdout.splitlines()) + assert b'configHash' in completed.stdout # Partial output must not be mistaken for success. + + +@pytest.mark.parametrize('payload', [b'{"version":1,"version":2}', b'{"value":NaN}', b'{"value":Infinity}', b'null', b'[]']) +def test_malformed_envelopes_fail_before_numerical_execution(tmp_path, payload): + output = BytesIO() + with pytest.raises(ValueError): + replay_worker.run(BytesIO(payload), output, tmp_path / 'missing') + assert output.getvalue() == b'' + assert not (tmp_path / 'missing').exists() + + +def test_worker_byte_limits_do_not_emit_an_oversize_input_or_result(tmp_path, monkeypatch): + store, snapshot, _ = fixture(tmp_path) + payload = json.dumps(work(snapshot)).encode() + monkeypatch.setattr(replay_worker, 'MAX_INPUT_BYTES', len(payload) - 1) + output = BytesIO() + with pytest.raises(ValueError, match='input exceeds'): + replay_worker.run(BytesIO(payload), output, store.directory) + assert output.getvalue() == b'' + monkeypatch.setattr(replay_worker, 'MAX_INPUT_BYTES', len(payload)) + monkeypatch.setattr(replay_worker, 'MAX_OUTPUT_BYTES', 10) + with pytest.raises(ValueError, match='output exceeds'): + replay_worker.run(BytesIO(payload), output, store.directory) + assert output.getvalue() == b'' + + +def test_build_probe_and_pinned_execution_reject_runtime_changes_before_reading(tmp_path): + from common.biometrics.build_identity import replay_build_identity + build = replay_build_identity() + probe = subprocess.run([sys.executable, '-B', '-m', 'common.biometrics.replay_worker', '--describe-build'], + capture_output=True, cwd=tmp_path, + env={'PYTHONPATH': str(Path(__file__).resolve().parent.parent), 'LANG': 'C.UTF-8', 'TZ': 'UTC'}, timeout=20) + assert probe.returncode == 0 and not probe.stderr + assert json.loads(probe.stdout) == build + store, snapshot, _ = fixture(tmp_path) + payload = {**work(snapshot), 'expected_build_hash': build['hash']} + success = child(tmp_path, store.directory, json.dumps(payload).encode()) + assert success.returncode == 0 + assert json.loads(success.stdout.splitlines()[-1])['result']['build_hash'] == build['hash'] + for bad in ('0' * 64, None, 42, 'bad'): + output = BytesIO() + with pytest.raises(ValueError, match='build identity changed'): + replay_worker.run(BytesIO(json.dumps({**payload, 'expected_build_hash': bad}).encode()), output, tmp_path / 'missing') + assert output.getvalue() == b'' + assert not (tmp_path / 'missing').exists() + + +def test_changed_build_during_execution_never_emits_success(tmp_path, monkeypatch): + from common.biometrics import build_identity + store, snapshot, _ = fixture(tmp_path) + before = build_identity.replay_build_identity() + identities = iter([before, {**before, 'hash': '0' * 64}]) + monkeypatch.setattr(build_identity, 'replay_build_identity', lambda: next(identities)) + output = BytesIO() + with pytest.raises(ValueError, match='changed during execution'): + replay_worker.run(BytesIO(json.dumps(work(snapshot)).encode()), output, store.directory) + events = [json.loads(line) for line in output.getvalue().splitlines()] + assert events and all(event['type'] == 'reading' for event in events) diff --git a/modules/piezo-processor/test_retained_replay.py b/modules/piezo-processor/test_retained_replay.py new file mode 100644 index 000000000..09feddcd7 --- /dev/null +++ b/modules/piezo-processor/test_retained_replay.py @@ -0,0 +1,154 @@ +"""Retained gzip -> bounded CBOR -> shared numerical replay integration.""" +from copy import deepcopy +from dataclasses import asdict +import gzip +from pathlib import Path +import struct +import sys + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from common.biometrics.config import manifest +from common.biometrics.raw_artifacts import RawArtifactStore, RawLimits +from common.biometrics.retained_replay import RetainedReplayInput, replay_retained +from test_waveform_replay import frames, START, KNOWN, make_replay + + +# Encode the small fixture subset without importing cbor2, which existing live +# adapter tests intentionally stub globally during collection. +def encode(value): + if isinstance(value, dict): + return bytes([0xa0 + len(value)]) + b''.join(encode(key) + encode(item) for key, item in value.items()) + if isinstance(value, str): + return bytes([0x60 + len(value)]) + value.encode() + if isinstance(value, bytes): + return b'\x5a' + struct.pack('>I', len(value)) + value + return b'\xfb' + struct.pack('>d', value) + + +def fixture(tmp_path, records=None, split_at=76): + records = records if records is not None else [ + (START, {'type': 'frzHealth', 'left': {'pumpRpm': 0}, 'right': {'pumpRpm': 0}}), *frames()] + archive = tmp_path / 'archive' + archive.mkdir() + store = RawArtifactStore(tmp_path / 'retained', RawLimits(1_000_000, 2_000_000, 4_000_000, 1)) + selected = [] + for index, group in enumerate([records[:split_at], records[split_at:]]): + if not group: + continue + data = b'' + for timestamp, record in group: + payload = encode({**record, 'ts': timestamp / 1000}) + data += b'\xa2\x63seq\x01\x64data\x5a' + struct.pack('>I', len(payload)) + payload + name = f'{index}.RAW.gz' + (archive / name).write_bytes(gzip.compress(data, mtime=0)) + artifact = store.retain(archive, name) + selected.append({'artifact': asdict(artifact), 'first_event_ms': group[0][0], 'last_event_ms': group[-1][0]}) + (archive / name).unlink() + snapshot = {'version': 'retained-waveform-input-v1', 'retained_start_ms': START, + 'score_start_ms': START + 60000, 'score_end_ms': START + 150000, 'artifacts': selected, + 'contexts': [{'start_ms': START, 'end_ms': START + 150000, + 'active_sides': ['left'], 'left_occupied': True, 'right_occupied': False, 'pump_context_known': True}]} + return store, snapshot, records + + +def run(store, snapshot, sink, **options): + return replay_retained(store, snapshot, + {side: deepcopy(manifest()['defaults']) for side in ('left', 'right')}, sink, **options) + + +def test_retained_files_match_direct_numerical_driver_after_original_archives_are_deleted(tmp_path): + store, snapshot, records = fixture(tmp_path) + expected, actual = [], [] + replay = make_replay(expected.append) + for timestamp, record in records: + replay.feed(timestamp, record, KNOWN) + replay.close() + result = run(store, snapshot, actual.append) + assert actual == expected + assert actual + assert result['counts'] == replay.counts + assert result['config_hashes'] == replay.hashes + assert result['decoded_records'] == 151 + assert result['artifact_hashes'] == [item['artifact']['sha256'] for item in snapshot['artifacts']] + assert result['limitations'] == sorted(replay.limitations) + again = [] + assert run(store, snapshot, again.append) == result + assert again == actual + + +def test_context_gaps_are_unknown_and_caller_mutation_cannot_change_frozen_timeline(tmp_path): + store, snapshot, _ = fixture(tmp_path) + snapshot['contexts'][0]['end_ms'] = START + 90000 + frozen = RetainedReplayInput.parse(snapshot) + snapshot['contexts'][0]['active_sides'].append('right') + assert frozen.contexts[0].context.active_sides == ('left',) + snapshot['contexts'][0]['active_sides'] = ['left'] + output = [] + result = run(store, snapshot, output.append) + assert {'missing_sleeper_mode', 'missing_occupancy_context', 'missing_pump_context'} <= set(result['limitations']) + assert any('missing_occupancy_context' in row.limitations for row in output) + + +def test_rejects_overlaps_duplicates_unknown_versions_and_bad_context_before_outputs(tmp_path): + store, snapshot, _ = fixture(tmp_path) + bad_inputs = [] + bad = deepcopy(snapshot) + bad['artifacts'][1]['first_event_ms'] = bad['artifacts'][0]['last_event_ms'] + bad_inputs.append(bad) + bad = deepcopy(snapshot) + bad['artifacts'].append(bad['artifacts'][0]) + bad_inputs.append(bad) + bad = deepcopy(snapshot) + bad['contexts'].append(bad['contexts'][0]) + bad_inputs.append(bad) + bad = deepcopy(snapshot) + bad['contexts'][0]['left_occupied'] = 1 + bad_inputs.append(bad) + bad_inputs.extend([{**snapshot, 'version': 'future'}, {**snapshot, 'extra': True}, {**snapshot, 'artifacts': []}]) + for bad in bad_inputs: + output = [] + with pytest.raises(ValueError): + run(store, bad, output.append) + assert not output + + +def test_rejects_incorrect_inspection_bounds_and_record_reversals(tmp_path): + store, snapshot, _ = fixture(tmp_path) + snapshot['artifacts'][0]['first_event_ms'] -= 1 + with pytest.raises(ValueError, match='differ from inspected'): + run(store, snapshot, lambda _: None) + snapshot['artifacts'][0]['first_event_ms'] += 2 + with pytest.raises(ValueError, match='outside inspected'): + run(store, snapshot, lambda _: None) + other = tmp_path / 'reversed' + other.mkdir() + store, snapshot, _ = fixture(other, [(START + offset, {'type': 'frzHealth'}) for offset in (0, 2000, 1000, 3000)]) + with pytest.raises(ValueError, match='reversed'): + run(store, snapshot, lambda _: None) + + +def test_cancellation_output_and_decode_budgets_fail_without_successful_result(tmp_path): + store, snapshot, _ = fixture(tmp_path) + with pytest.raises(ValueError, match='cancelled'): + run(store, snapshot, lambda _: None, cancelled=lambda: True) + with pytest.raises(ValueError, match='record.*budget|record count'): + run(store, snapshot, lambda _: None, maximum_records=100) + with pytest.raises(ValueError, match='output budget'): + run(store, snapshot, lambda _: None, maximum_readings=1) + cancelled = [False] + output = [] + def sink(row): + output.append(row) + cancelled[0] = True + with pytest.raises(ValueError, match='cancelled'): + run(store, snapshot, sink, cancelled=lambda: cancelled[0]) + assert len(output) == 1 + + +def test_missing_artifact_never_falls_back_to_a_rolling_archive(tmp_path): + store, snapshot, _ = fixture(tmp_path) + (store.directory / snapshot['artifacts'][0]['artifact']['name']).unlink() + with pytest.raises(FileNotFoundError): + run(store, snapshot, lambda _: None) diff --git a/modules/piezo-processor/test_runtime_adapter.py b/modules/piezo-processor/test_runtime_adapter.py new file mode 100644 index 000000000..8b0573ad0 --- /dev/null +++ b/modules/piezo-processor/test_runtime_adapter.py @@ -0,0 +1,201 @@ +"""Real SQLite, estimator and routing checks for live profile adoption.""" +from datetime import datetime, timezone +import json +from pathlib import Path +import sqlite3 + +import numpy as np +import pytest + +import main +from common.biometrics.config import resolve_config +from runtime_adapter import RuntimeCoordinator, restore_applied + + +class Clock: + value = 0 + def __call__(self): + return self.value + + +@pytest.fixture +def rig(): + conn = sqlite3.connect(":memory:") + conn.executescript(""" + CREATE TABLE vitals (id INTEGER PRIMARY KEY, side TEXT, timestamp INTEGER, + heart_rate REAL, hrv REAL, breathing_rate REAL); + CREATE TABLE vitals_quality (vitals_id INTEGER, side TEXT, timestamp INTEGER, + quality_score REAL, flags TEXT, hr_raw REAL, created_at INTEGER); + """) + migration = Path(__file__).parents[2] / "src/db/biometrics-migrations/0020_black_patch.sql" + conn.executescript(migration.read_text()) + holder = main.DBHolder(conn) + processors = {side: main.SideProcessor(side, holder) for side in ("left", "right")} + clock = Clock() + coordinator = RuntimeCoordinator(holder, processors, clock=clock) + return conn, holder, processors, clock, coordinator + + +def project(conn, generation=1, side="left", boundary="immediate", overrides=None): + config = resolve_config(overrides or [{"path": "heart_rate.trackingToleranceBpm", "value": 20}]) + payload = dict(schemaVersion=1, side=side, generation=generation, revisionId=None, + resolved=config["resolved"], resolvedHash=config["hash"], + manifestHash=config["manifestHash"], boundary=boundary) + with conn: + conn.execute("INSERT INTO biometric_runtime_configs VALUES (?, ?, ?, ?, ?, ?)", + (side, generation, config["hash"], json.dumps(payload), boundary, 0)) + return config + + +def test_actual_parameters_reset_before_samples_and_ack_after(rig): + conn, _, processors, _, runtime = rig + project(conn) + left = processors["left"] + left._hr_buf.extend([123] * 20) + runtime.before_frame(None, lambda _: True, 1000) + assert left.parameters.tracking_tolerance_bpm == 20 + assert len(left._hr_buf) == 0 + assert processors["right"].parameters.tracking_tolerance_bpm == 15 + assert conn.execute("SELECT * FROM biometric_runtime_receipts").fetchall() == [] + left.ingest(np.array([1, 2])) + runtime.after_frame(1000) + receipts = conn.execute("SELECT component, generation, effective_sample_time_ms, state FROM biometric_runtime_receipts").fetchall() + assert len(receipts) == 3 + assert {r[0] for r in receipts} == {"heart_rate", "breathing_rate", "hrv_index"} + assert all(r[1:] == (1, 1000, "applied") for r in receipts) + + +def test_pending_next_session_and_restart_restore(rig): + conn, holder, processors, clock, runtime = rig + project(conn, boundary="next_session") + runtime.before_frame(None, lambda _: None, 1000) + runtime.after_frame(1000) + assert processors["left"].parameters.tracking_tolerance_bpm == 15 + assert restore_applied(conn, "left") is None + runtime.before_frame(None, lambda _: False, 2000) + runtime.after_frame(2000) + assert restore_applied(conn, "left").generation == 1 + restarted = RuntimeCoordinator(holder, {s: main.SideProcessor(s, holder) for s in processors}, clock=clock) + restarted.before_frame(None, lambda _: True, 3000) + assert restarted.processors["left"].parameters.tracking_tolerance_bpm == 20 + assert restarted.processors["left"].run_context.id != processors["left"].run_context.id + + +def test_single_sleeper_uses_home_profile_on_both_channels_and_retains_source(rig): + conn, holder, processors, _, runtime = rig + config = project(conn) + runtime.before_frame("left", lambda _: True, 1000) + assert processors["right"].parameters.tracking_tolerance_bpm == 20 + context = processors["right"].run_context + candidate = main.VitalsCandidate("right", datetime.fromtimestamp(5, timezone.utc), + 70, 30, 12, .8, None, 70, context) + # A delayed retry after a mode change must preserve its original destination. + assert candidate.write(holder, side="right") + row = conn.execute("SELECT side, processing_run_id FROM vitals").fetchone() + assert row == ("left", context.id) + run = conn.execute("SELECT side, config_hash, input_identity FROM biometric_processing_runs").fetchone() + assert run[:2] == ("left", config["hash"]) + assert json.loads(run[2])["sourceSide"] == "right" + runtime.before_frame(None, lambda _: True, 2000) + assert processors["right"].parameters.tracking_tolerance_bpm == 15 + assert processors["right"].run_context.side == "right" + + +def test_stage_only_change_preserves_numerical_state(rig): + conn, _, processors, clock, runtime = rig + project(conn) + runtime.before_frame(None, lambda _: True, 1000) + left = processors["left"] + left._hr_buf.extend([7, 8]) + estimator = left._estimator + project(conn, 2, overrides=[{"path": "heart_rate.trackingToleranceBpm", "value": 20}, + {"path": "sleep_stages.ruleset.smoothIsolatedEpochs", "value": False}]) + clock.value = 5 + runtime.before_frame(None, lambda _: True, 2000) + assert left._estimator is estimator + assert list(left._hr_buf) == [7, 8] + assert json.loads(left.run_context.bundle_json)["generation"] == 2 + + +def test_invalid_new_projection_retains_actual_profile_and_reports_error(rig): + conn, _, processors, clock, runtime = rig + project(conn) + runtime.before_frame(None, lambda _: True, 1000) + runtime.after_frame(1000) + project(conn, 2) + conn.execute("UPDATE biometric_runtime_configs SET payload='{}' WHERE generation=2") + clock.value = 5 + runtime.before_frame(None, lambda _: True, 2000) + runtime.after_frame(2000) + assert processors["left"].parameters.tracking_tolerance_bpm == 20 + assert runtime.problem is not None + assert conn.execute("SELECT DISTINCT generation FROM biometric_runtime_receipts").fetchall() == [(1,)] + + +def test_observation_and_provenance_rollback_together_then_recover(rig): + conn, holder, processors, _, runtime = rig + runtime.before_frame(None, lambda _: True, 1000) + context = processors["left"].run_context + conn.execute("CREATE TRIGGER reject_quality BEFORE INSERT ON vitals_quality BEGIN SELECT RAISE(ABORT, 'full'); END") + candidate = main.VitalsCandidate("left", datetime.fromtimestamp(5, timezone.utc), 70, 30, 12, .8, None, 70, context) + assert not candidate.write(holder) + assert conn.execute("SELECT * FROM vitals").fetchall() == [] + assert conn.execute("SELECT * FROM biometric_processing_runs").fetchall() == [] + conn.execute("DROP TRIGGER reject_quality") + assert candidate.write(holder) + assert conn.execute("SELECT processing_run_id FROM vitals").fetchone()[0] == context.id + # History reset: a retained context can recreate its exact bundle. + conn.execute("DELETE FROM vitals") + conn.execute("DELETE FROM biometric_processing_runs") + conn.commit() + assert candidate.write(holder) + assert conn.execute("SELECT id FROM biometric_processing_runs").fetchone()[0] == context.id + + +def test_receipt_transaction_failure_retries_without_false_applied_state(rig): + conn, _, _, _, runtime = rig + project(conn) + runtime.before_frame(None, lambda _: True, 1000) + conn.execute("CREATE TRIGGER reject_receipt BEFORE INSERT ON biometric_runtime_receipts WHEN NEW.component='hrv_index' BEGIN SELECT RAISE(ABORT, 'full'); END") + runtime.after_frame(1000) + assert conn.execute("SELECT * FROM biometric_runtime_receipts").fetchall() == [] + assert runtime.problem is not None + conn.execute("DROP TRIGGER reject_receipt") + runtime.after_frame(2000) + assert restore_applied(conn, "left").generation == 1 + + +def test_old_schema_cannot_enable_custom_processing(rig): + conn, _, processors, _, runtime = rig + project(conn) + conn.execute("DROP TABLE biometric_runtime_receipts") + runtime.before_frame(None, lambda _: True, 1000) + assert not runtime.enabled + assert processors["left"].parameters.tracking_tolerance_bpm == 15 + assert runtime.problem is not None + + +def test_restore_rejects_corrupt_projection_envelope(rig): + conn, _, _, _, runtime = rig + project(conn) + runtime.before_frame(None, lambda _: True, 1000) + runtime.after_frame(1000) + conn.execute("UPDATE biometric_runtime_configs SET resolved_hash='corrupt'") + with pytest.raises(ValueError, match="envelope mismatch"): + restore_applied(conn, "left") + + +def test_heartbeat_is_bounded_and_repairs_missing_receipts(rig): + conn, _, _, clock, runtime = rig + project(conn) + runtime.before_frame(None, lambda _: True, 1000) + runtime.after_frame(1000) + runtime.after_frame(2000) + assert conn.execute("SELECT DISTINCT heartbeat_at_ms FROM biometric_runtime_receipts").fetchall() == [(1000,)] + conn.execute("DELETE FROM biometric_runtime_receipts") + conn.commit() + clock.value = 30 + runtime.before_frame(None, lambda _: True, 31000) + runtime.after_frame(31000) + assert restore_applied(conn, "left").generation == 1 + assert conn.execute("SELECT DISTINCT heartbeat_at_ms FROM biometric_runtime_receipts").fetchall() == [(31000,)] diff --git a/modules/piezo-processor/test_shadow.py b/modules/piezo-processor/test_shadow.py new file mode 100644 index 000000000..abb522cba --- /dev/null +++ b/modules/piezo-processor/test_shadow.py @@ -0,0 +1,146 @@ +"""Prospective adapter parity and failure boundaries using locked signal deps.""" +from copy import deepcopy +from pathlib import Path +import sys + +import numpy as np +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from common.biometrics.config import manifest +from common.biometrics.replay import RecordedContext, WaveformReplay +from common.biometrics.shadow import ProspectiveShadow + +START = 1_700_000_000_000 +CONTEXT = RecordedContext(('left',), True, False, True) + + +def configs(): + return {side: deepcopy(manifest()['defaults']) for side in ('left', 'right')} + + +def frames(): + for second in range(150): + values = (np.sin((np.arange(500) + second * 500) / 500 * 2 * np.pi * 1.2) * 1_000_000).astype(' 0 diff --git a/modules/piezo-processor/test_waveform_replay.py b/modules/piezo-processor/test_waveform_replay.py new file mode 100644 index 000000000..d9403da0c --- /dev/null +++ b/modules/piezo-processor/test_waveform_replay.py @@ -0,0 +1,173 @@ +"""Raw-frame replay checks using locked numerical dependencies and real estimates.""" +from copy import deepcopy +from dataclasses import FrozenInstanceError +from pathlib import Path +import sys + +import numpy as np +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from common.biometrics.config import manifest, config_hash +from common.biometrics.replay import WaveformReplay, RecordedContext, MAX_CHANNEL_BYTES +from common.biometrics.frames import decode_int32_samples, LOST_SAMPLE + +START = 1_700_000_000_000 +KNOWN = RecordedContext(('left',), True, False, True) + + +def make_replay(sink, start=START + 60_000, end=START + 150_000): + configs = {side: deepcopy(manifest()['defaults']) for side in ('left', 'right')} + return WaveformReplay(configs, retained_start_ms=START, + score_start_ms=start, score_end_ms=end, sink=sink) + + +def frames(): + for second in range(150): + samples = (np.sin((np.arange(500) + second * 500) / 500 * 2 * np.pi * 1.2) + * 1_000_000).astype(' 0 + assert replay.counts['scored_outputs'] == len(first) + assert replay.counts['accepted_frames'] == 150 + assert 'cold_start_without_serialized_state' in first[0].limitations + assert 'missing_pump_context' not in replay.limitations + with pytest.raises(FrozenInstanceError): + first[0].side = 'right' + with pytest.raises(ValueError, match='closed'): + replay.feed(START, {}, KNOWN) + + +def test_recorded_pump_guard_drops_frames_until_exact_event_time_boundary(): + replay = make_replay(lambda _: None, start=START, end=START + 20_000) + context = RecordedContext(('left', 'right'), False, False, True) + quiet = np.ones(500, dtype=' /// import "./.next/types/routes.d.ts"; +import "./.next/types/root-params.d.ts"; // NOTE: This file should not be edited // see https://nextjs.org/docs/app/api-reference/config/typescript for more information. diff --git a/package.json b/package.json index b8890e91f..ee79c37d4 100644 --- a/package.json +++ b/package.json @@ -3,8 +3,8 @@ "type": "module", "private": true, "scripts": { - "prebuild": "node scripts/generate-git-info.mjs && lingui compile", - "build": "next build && node scripts/strip-turbopack-external-hashes.mjs && cp -r .next/static .next/standalone/.next/static && { [ ! -d public ] || cp -r public .next/standalone/public || true; }", + "prebuild": "node scripts/generate-git-info.mjs && lingui compile && node scripts/build/demo-biometric-worker.mjs", + "build": "next build && node scripts/strip-turbopack-external-hashes.mjs && cp -r .next/static .next/standalone/.next/static && { [ ! -d public ] || cp -r public .next/standalone/public || true; } && node scripts/build/biometrics-worker.mjs", "dev": "next dev", "db:generate": "drizzle-kit generate", "db:migrate": "drizzle-kit migrate", @@ -27,7 +27,10 @@ "test:mutation": "stryker run", "test:mutation:incremental": "stryker run --incremental", "tsc": "tsc --noEmit", - "prepare": "git rev-parse --git-dir >/dev/null 2>&1 && git config core.hooksPath .githooks || true" + "prepare": "git rev-parse --git-dir >/dev/null 2>&1 && git config core.hooksPath .githooks || true", + "build:biometrics-worker": "node scripts/build/biometrics-worker.mjs", + "biometrics:experiment": "tsx scripts/algorithm-experiments/cli.ts", + "test:browser": "node tests/browser/run.mjs" }, "lint-staged": { "*.{ts,tsx}": "eslint --fix", @@ -43,6 +46,7 @@ "@lingui/react": "^6.0.0", "@modelcontextprotocol/sdk": "^1.31.0", "@nats-io/transport-node": "^3.4.0", + "@noble/hashes": "2.4.0", "@tanstack/react-query": "^5.90.21", "@trpc/client": "^11.10.0", "@trpc/next": "^11.10.0", @@ -88,6 +92,7 @@ "@lingui/loader": "^6.0.0", "@lingui/swc-plugin": "^6.0.0", "@lingui/vite-plugin": "^6.0.0", + "@playwright/test": "1.58.2", "@rolldown/plugin-babel": "^0.2.4", "@semantic-release/commit-analyzer": "^13.0.1", "@semantic-release/git": "^11.0.0", @@ -114,6 +119,7 @@ "@vitest/coverage-v8": "4.1.11", "conventional-changelog-conventionalcommits": "^9.1.0", "drizzle-kit": "^0.31.9", + "esbuild": "0.28.2", "eslint": "^9.39.3", "eslint-config-next": "^16.1.6", "eslint-plugin-react": "^7.37.5", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c779f161c..f70c531ac 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -19,7 +19,7 @@ importers: dependencies: '@ajayche/trpc-panel': specifier: ^2.0.4 - version: 2.1.0(@trpc/server@11.19.0(typescript@6.0.3))(@types/react@19.3.0)(next@16.4.0(@babel/core@7.29.7)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0))(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(zod@4.6.5) + version: 2.1.0(@trpc/server@11.19.0(typescript@6.0.3))(@types/react@19.3.0)(next@16.4.0(@babel/core@7.29.7)(@playwright/test@1.58.2)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0))(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(zod@4.6.5) '@base-ui/react': specifier: ^1.2.0 version: 1.9.0(@types/react@19.3.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0) @@ -41,6 +41,9 @@ importers: '@nats-io/transport-node': specifier: ^3.4.0 version: 3.4.0 + '@noble/hashes': + specifier: 2.4.0 + version: 2.4.0 '@tanstack/react-query': specifier: ^5.90.21 version: 5.104.1(react@19.3.0) @@ -49,7 +52,7 @@ importers: version: 11.19.0(@trpc/server@11.19.0(typescript@6.0.3))(typescript@6.0.3) '@trpc/next': specifier: ^11.10.0 - version: 11.19.0(@tanstack/react-query@5.104.1(react@19.3.0))(@trpc/client@11.19.0(@trpc/server@11.19.0(typescript@6.0.3))(typescript@6.0.3))(@trpc/react-query@11.19.0(@tanstack/react-query@5.104.1(react@19.3.0))(@trpc/client@11.19.0(@trpc/server@11.19.0(typescript@6.0.3))(typescript@6.0.3))(@trpc/server@11.19.0(typescript@6.0.3))(react@19.3.0)(typescript@6.0.3))(@trpc/server@11.19.0(typescript@6.0.3))(next@16.4.0(@babel/core@7.29.7)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0))(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(typescript@6.0.3) + version: 11.19.0(@tanstack/react-query@5.104.1(react@19.3.0))(@trpc/client@11.19.0(@trpc/server@11.19.0(typescript@6.0.3))(typescript@6.0.3))(@trpc/react-query@11.19.0(@tanstack/react-query@5.104.1(react@19.3.0))(@trpc/client@11.19.0(@trpc/server@11.19.0(typescript@6.0.3))(typescript@6.0.3))(@trpc/server@11.19.0(typescript@6.0.3))(react@19.3.0)(typescript@6.0.3))(@trpc/server@11.19.0(typescript@6.0.3))(next@16.4.0(@babel/core@7.29.7)(@playwright/test@1.58.2)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0))(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(typescript@6.0.3) '@trpc/react-query': specifier: ^11.10.0 version: 11.19.0(@tanstack/react-query@5.104.1(react@19.3.0))(@trpc/client@11.19.0(@trpc/server@11.19.0(typescript@6.0.3))(typescript@6.0.3))(@trpc/server@11.19.0(typescript@6.0.3))(react@19.3.0)(typescript@6.0.3) @@ -58,7 +61,7 @@ importers: version: 11.19.0(typescript@6.0.3) '@vercel/analytics': specifier: ^2.0.1 - version: 2.0.1(next@16.4.0(@babel/core@7.29.7)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0))(react@19.3.0) + version: 2.0.1(next@16.4.0(@babel/core@7.29.7)(@playwright/test@1.58.2)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0))(react@19.3.0) better-sqlite3: specifier: ^12.6.2 version: 12.11.1 @@ -103,7 +106,7 @@ importers: version: 1.1.0 next: specifier: ^16.1.6 - version: 16.4.0(@babel/core@7.29.7)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0) + version: 16.4.0(@babel/core@7.29.7)(@playwright/test@1.58.2)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0) node-schedule: specifier: ^2.1.1 version: 2.1.1 @@ -146,7 +149,7 @@ importers: devDependencies: '@codecov/nextjs-webpack-plugin': specifier: ^2.0.1 - version: 2.0.1(next@16.4.0(@babel/core@7.29.7)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0))(webpack@5.111.1(esbuild@0.28.2)) + version: 2.0.1(next@16.4.0(@babel/core@7.29.7)(@playwright/test@1.58.2)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0))(webpack@5.111.1(esbuild@0.28.2)) '@eslint/css': specifier: ^2.0.0 version: 2.0.0 @@ -171,6 +174,9 @@ importers: '@lingui/vite-plugin': specifier: ^6.0.0 version: 6.9.1(@babel/core@7.29.7)(@lingui/babel-plugin-lingui-macro@6.9.1(@babel/core@7.29.7))(@rolldown/plugin-babel@0.2.4(@babel/core@7.29.7)(@babel/runtime@7.29.7)(rolldown@1.2.13)(vite@8.3.4(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)))(esbuild@0.28.2)(rolldown@1.2.13)(vite@8.3.4(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)) + '@playwright/test': + specifier: 1.58.2 + version: 1.58.2 '@rolldown/plugin-babel': specifier: ^0.2.4 version: 0.2.4(@babel/core@7.29.7)(@babel/runtime@7.29.7)(rolldown@1.2.13)(vite@8.3.4(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)) @@ -249,6 +255,9 @@ importers: drizzle-kit: specifier: ^0.31.9 version: 0.31.11 + esbuild: + specifier: 0.28.2 + version: 0.28.2 eslint: specifier: ^9.39.3 version: 9.39.5(jiti@2.7.0) @@ -266,7 +275,7 @@ importers: version: 2.7.0 jsdom: specifier: ^29.0.0 - version: 29.1.1 + version: 29.1.1(@noble/hashes@2.4.0) lint-staged: specifier: ^16.4.0 version: 16.4.0 @@ -296,7 +305,7 @@ importers: version: 6.1.1(typescript@6.0.3)(vite@8.3.4(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)) vitest: specifier: ^4.0.18 - version: 4.1.11(@types/node@25.9.9)(@vitest/coverage-v8@4.1.11)(jsdom@29.1.1)(vite@8.3.4(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)) + version: 4.1.11(@types/node@25.9.9)(@vitest/coverage-v8@4.1.11)(jsdom@29.1.1(@noble/hashes@2.4.0))(vite@8.3.4(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)) packages: @@ -1868,6 +1877,10 @@ packages: cpu: [x64] os: [win32] + '@noble/hashes@2.4.0': + resolution: {integrity: sha512-X5XaVWZIBCT7HHZGm5I7ZQXDwLG+bGXuSrMQAW+7Zvl87h1kmc1ZB1VSRJcpUfoUrGQp4Fkoxm5kZ+Ms+aW+eA==} + engines: {node: '>= 20.19.0'} + '@nodelib/fs.scandir@2.1.5': resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==} engines: {node: '>= 8'} @@ -1963,6 +1976,11 @@ packages: '@oxc-project/types@0.153.0': resolution: {integrity: sha512-jSNrRIyvPVLL5E1z/tS5zLAprosGz/FuI1PWwEOz59j3E4TTTzSpqKb/RDGqJhhW0QHYISGXxq582wGx6T11Tw==} + '@playwright/test@1.58.2': + resolution: {integrity: sha512-akea+6bHYBBfA9uQqSYmlJXn61cTa+jbO87xVLCWbTqbWadRVmhxlXATaOjOgcBaWU4ePo0wB41KMFv3o35IXA==} + engines: {node: '>=18'} + hasBin: true + '@pnpm/config.env-replace@1.1.0': resolution: {integrity: sha512-htyl8TWnKL7K/ESFa1oW2UB5lVDxuF5DpM7tBi6Hu2LNL3mWkIzNLG6N4zoCUP1lCKNxWy/3iu8mS8MvToGd6w==} engines: {node: '>=12.22.0'} @@ -4210,6 +4228,11 @@ packages: resolution: {integrity: sha512-KYAb4c9BJQI6QqGKthV68OHe0badztdXJWKo0WtBA9IuCFPTKvE5ZdUBglP833aMjhaSPNO4A5j/EkzZtGlKjA==} engines: {node: '>=14.14'} + fsevents@2.3.2: + resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + fsevents@2.3.3: resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} @@ -5933,6 +5956,16 @@ packages: resolution: {integrity: sha512-nDywThFk1i4BQK4twPQ6TA4RT8bDY96yeuCVBWL3ePARCiEKDRSrNGbFIgUJpLp+XeIR65v8ra7WuJOFUBtkMA==} engines: {node: '>=8'} + playwright-core@1.58.2: + resolution: {integrity: sha512-yZkEtftgwS8CsfYo7nm0KE8jsvm6i/PTgVtB8DL726wNf6H2IMsDuxCpJj59KDaxCtSnrWan2AeDqM7JBaultg==} + engines: {node: '>=18'} + hasBin: true + + playwright@1.58.2: + resolution: {integrity: sha512-vA30H8Nvkq/cPBnNw4Q8TWz1EJyqgpuinBcHET0YVJVFldr8JDNiU9LaWAE1KqSkRYazuaBhTpB5ZzShOezQ6A==} + engines: {node: '>=18'} + hasBin: true + pngjs@5.0.0: resolution: {integrity: sha512-40QW5YalBNfQo5yRYmiw7Yz6TKKVr3h6970B2YE+3fQpsWcrbj1PzJgxeJ19DRQjhMbKPIuMY8rFaXc8moolVw==} engines: {node: '>=10.13.0'} @@ -7428,13 +7461,13 @@ snapshots: '@actions/io@3.0.2': {} - '@ajayche/trpc-panel@2.1.0(@trpc/server@11.19.0(typescript@6.0.3))(@types/react@19.3.0)(next@16.4.0(@babel/core@7.29.7)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0))(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(zod@4.6.5)': + '@ajayche/trpc-panel@2.1.0(@trpc/server@11.19.0(typescript@6.0.3))(@types/react@19.3.0)(next@16.4.0(@babel/core@7.29.7)(@playwright/test@1.58.2)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0))(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(zod@4.6.5)': dependencies: '@microlink/react-json-view': 1.31.38(@types/react@19.3.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0) '@trpc/server': 11.19.0(typescript@6.0.3) clsx: 2.1.1 fuzzysort: 2.0.4 - nuqs: 2.10.1(next@16.4.0(@babel/core@7.29.7)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0))(react@19.3.0) + nuqs: 2.10.1(next@16.4.0(@babel/core@7.29.7)(@playwright/test@1.58.2)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0))(react@19.3.0) path: 0.12.7 pretty-bytes: 6.1.1 pretty-ms: 8.0.0 @@ -7762,11 +7795,11 @@ snapshots: unplugin: 1.16.1 zod: 3.25.76 - '@codecov/nextjs-webpack-plugin@2.0.1(next@16.4.0(@babel/core@7.29.7)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0))(webpack@5.111.1(esbuild@0.28.2))': + '@codecov/nextjs-webpack-plugin@2.0.1(next@16.4.0(@babel/core@7.29.7)(@playwright/test@1.58.2)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0))(webpack@5.111.1(esbuild@0.28.2))': dependencies: '@codecov/bundler-plugin-core': 2.0.1 '@codecov/webpack-plugin': 2.0.1(webpack@5.111.1(esbuild@0.28.2)) - next: 16.4.0(@babel/core@7.29.7)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0) + next: 16.4.0(@babel/core@7.29.7)(@playwright/test@1.58.2)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0) unplugin: 1.16.1 webpack: 5.111.1(esbuild@0.28.2) @@ -8159,7 +8192,9 @@ snapshots: '@eslint/core': 1.2.1 levn: 0.4.1 - '@exodus/bytes@1.16.0': {} + '@exodus/bytes@1.16.0(@noble/hashes@2.4.0)': + optionalDependencies: + '@noble/hashes': 2.4.0 '@floating-ui/core@1.8.0': dependencies: @@ -8797,6 +8832,8 @@ snapshots: '@next/swc-win32-x64-msvc@16.4.0': optional: true + '@noble/hashes@2.4.0': {} + '@nodelib/fs.scandir@2.1.5': dependencies: '@nodelib/fs.stat': 2.0.5 @@ -8897,6 +8934,10 @@ snapshots: '@oxc-project/types@0.153.0': {} + '@playwright/test@1.58.2': + dependencies: + playwright: 1.58.2 + '@pnpm/config.env-replace@1.1.0': {} '@pnpm/network.ca-file@1.0.2': @@ -9184,7 +9225,7 @@ snapshots: '@stryker-mutator/util': 9.6.1 semver: 7.8.5 tslib: 2.8.1 - vitest: 4.1.11(@types/node@25.9.9)(@vitest/coverage-v8@4.1.11)(jsdom@29.1.1)(vite@8.3.4(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)) + vitest: 4.1.11(@types/node@25.9.9)(@vitest/coverage-v8@4.1.11)(jsdom@29.1.1(@noble/hashes@2.4.0))(vite@8.3.4(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)) '@stylistic/eslint-plugin@5.10.0(eslint@9.39.5(jiti@2.7.0))': dependencies: @@ -9308,11 +9349,11 @@ snapshots: '@trpc/server': 11.19.0(typescript@6.0.3) typescript: 6.0.3 - '@trpc/next@11.19.0(@tanstack/react-query@5.104.1(react@19.3.0))(@trpc/client@11.19.0(@trpc/server@11.19.0(typescript@6.0.3))(typescript@6.0.3))(@trpc/react-query@11.19.0(@tanstack/react-query@5.104.1(react@19.3.0))(@trpc/client@11.19.0(@trpc/server@11.19.0(typescript@6.0.3))(typescript@6.0.3))(@trpc/server@11.19.0(typescript@6.0.3))(react@19.3.0)(typescript@6.0.3))(@trpc/server@11.19.0(typescript@6.0.3))(next@16.4.0(@babel/core@7.29.7)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0))(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(typescript@6.0.3)': + '@trpc/next@11.19.0(@tanstack/react-query@5.104.1(react@19.3.0))(@trpc/client@11.19.0(@trpc/server@11.19.0(typescript@6.0.3))(typescript@6.0.3))(@trpc/react-query@11.19.0(@tanstack/react-query@5.104.1(react@19.3.0))(@trpc/client@11.19.0(@trpc/server@11.19.0(typescript@6.0.3))(typescript@6.0.3))(@trpc/server@11.19.0(typescript@6.0.3))(react@19.3.0)(typescript@6.0.3))(@trpc/server@11.19.0(typescript@6.0.3))(next@16.4.0(@babel/core@7.29.7)(@playwright/test@1.58.2)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0))(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(typescript@6.0.3)': dependencies: '@trpc/client': 11.19.0(@trpc/server@11.19.0(typescript@6.0.3))(typescript@6.0.3) '@trpc/server': 11.19.0(typescript@6.0.3) - next: 16.4.0(@babel/core@7.29.7)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0) + next: 16.4.0(@babel/core@7.29.7)(@playwright/test@1.58.2)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0) react: 19.3.0 react-dom: 19.3.0(react@19.3.0) typescript: 6.0.3 @@ -9661,9 +9702,9 @@ snapshots: '@unrs/resolver-binding-win32-x64-msvc@1.12.2': optional: true - '@vercel/analytics@2.0.1(next@16.4.0(@babel/core@7.29.7)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0))(react@19.3.0)': + '@vercel/analytics@2.0.1(next@16.4.0(@babel/core@7.29.7)(@playwright/test@1.58.2)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0))(react@19.3.0)': optionalDependencies: - next: 16.4.0(@babel/core@7.29.7)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0) + next: 16.4.0(@babel/core@7.29.7)(@playwright/test@1.58.2)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0) react: 19.3.0 '@vitejs/plugin-react@6.1.2(@rolldown/plugin-babel@0.2.4(@babel/core@7.29.7)(@babel/runtime@7.29.7)(rolldown@1.2.13)(vite@8.3.4(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)))(vite@8.3.4(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1))': @@ -9685,7 +9726,7 @@ snapshots: obug: 2.2.1 std-env: 4.3.0 tinyrainbow: 3.1.1 - vitest: 4.1.11(@types/node@25.9.9)(@vitest/coverage-v8@4.1.11)(jsdom@29.1.1)(vite@8.3.4(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)) + vitest: 4.1.11(@types/node@25.9.9)(@vitest/coverage-v8@4.1.11)(jsdom@29.1.1(@noble/hashes@2.4.0))(vite@8.3.4(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)) '@vitest/expect@4.1.11': dependencies: @@ -10456,10 +10497,10 @@ snapshots: damerau-levenshtein@1.0.8: {} - data-urls@7.0.0: + data-urls@7.0.0(@noble/hashes@2.4.0): dependencies: whatwg-mimetype: 5.0.0 - whatwg-url: 16.0.1 + whatwg-url: 16.0.1(@noble/hashes@2.4.0) transitivePeerDependencies: - '@noble/hashes' @@ -10877,8 +10918,8 @@ snapshots: '@next/eslint-plugin-next': 16.4.0(eslint@9.39.5(jiti@2.7.0)) eslint: 9.39.5(jiti@2.7.0) eslint-import-resolver-node: 0.3.10 - eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.71.0(eslint@9.39.5(jiti@2.7.0))(typescript@6.0.3))(eslint@9.39.5(jiti@2.7.0)))(eslint@9.39.5(jiti@2.7.0)) - eslint-plugin-import: 2.32.0(@typescript-eslint/parser@8.71.0(eslint@9.39.5(jiti@2.7.0))(typescript@6.0.3))(eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.71.0(eslint@9.39.5(jiti@2.7.0))(typescript@6.0.3))(eslint@9.39.5(jiti@2.7.0)))(eslint@9.39.5(jiti@2.7.0)))(eslint@9.39.5(jiti@2.7.0)) + eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0)(eslint@9.39.5(jiti@2.7.0)) + eslint-plugin-import: 2.32.0(@typescript-eslint/parser@8.71.0(eslint@9.39.5(jiti@2.7.0))(typescript@6.0.3))(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.5(jiti@2.7.0)) eslint-plugin-jsx-a11y: 6.10.2(eslint@9.39.5(jiti@2.7.0)) eslint-plugin-react: 7.37.5(eslint@9.39.5(jiti@2.7.0)) eslint-plugin-react-hooks: 7.1.1(eslint@9.39.5(jiti@2.7.0)) @@ -10900,7 +10941,7 @@ snapshots: transitivePeerDependencies: - supports-color - eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.71.0(eslint@9.39.5(jiti@2.7.0))(typescript@6.0.3))(eslint@9.39.5(jiti@2.7.0)))(eslint@9.39.5(jiti@2.7.0)): + eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0)(eslint@9.39.5(jiti@2.7.0)): dependencies: '@nolyfill/is-core-module': 1.0.39 debug: 4.4.3 @@ -10911,22 +10952,22 @@ snapshots: tinyglobby: 0.2.17 unrs-resolver: 1.12.2 optionalDependencies: - eslint-plugin-import: 2.32.0(@typescript-eslint/parser@8.71.0(eslint@9.39.5(jiti@2.7.0))(typescript@6.0.3))(eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.71.0(eslint@9.39.5(jiti@2.7.0))(typescript@6.0.3))(eslint@9.39.5(jiti@2.7.0)))(eslint@9.39.5(jiti@2.7.0)))(eslint@9.39.5(jiti@2.7.0)) + eslint-plugin-import: 2.32.0(@typescript-eslint/parser@8.71.0(eslint@9.39.5(jiti@2.7.0))(typescript@6.0.3))(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.5(jiti@2.7.0)) transitivePeerDependencies: - supports-color - eslint-module-utils@2.14.0(@typescript-eslint/parser@8.71.0(eslint@9.39.5(jiti@2.7.0))(typescript@6.0.3))(eslint-import-resolver-node@0.3.10)(eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.71.0(eslint@9.39.5(jiti@2.7.0))(typescript@6.0.3))(eslint@9.39.5(jiti@2.7.0)))(eslint@9.39.5(jiti@2.7.0)))(eslint@9.39.5(jiti@2.7.0)): + eslint-module-utils@2.14.0(@typescript-eslint/parser@8.71.0(eslint@9.39.5(jiti@2.7.0))(typescript@6.0.3))(eslint-import-resolver-node@0.3.10)(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.5(jiti@2.7.0)): dependencies: debug: 3.2.7 optionalDependencies: '@typescript-eslint/parser': 8.71.0(eslint@9.39.5(jiti@2.7.0))(typescript@6.0.3) eslint: 9.39.5(jiti@2.7.0) eslint-import-resolver-node: 0.3.10 - eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.71.0(eslint@9.39.5(jiti@2.7.0))(typescript@6.0.3))(eslint@9.39.5(jiti@2.7.0)))(eslint@9.39.5(jiti@2.7.0)) + eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0)(eslint@9.39.5(jiti@2.7.0)) transitivePeerDependencies: - supports-color - eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.71.0(eslint@9.39.5(jiti@2.7.0))(typescript@6.0.3))(eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.71.0(eslint@9.39.5(jiti@2.7.0))(typescript@6.0.3))(eslint@9.39.5(jiti@2.7.0)))(eslint@9.39.5(jiti@2.7.0)))(eslint@9.39.5(jiti@2.7.0)): + eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.71.0(eslint@9.39.5(jiti@2.7.0))(typescript@6.0.3))(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.5(jiti@2.7.0)): dependencies: '@rtsao/scc': 1.1.0 array-includes: 3.2.0 @@ -10937,7 +10978,7 @@ snapshots: doctrine: 2.1.0 eslint: 9.39.5(jiti@2.7.0) eslint-import-resolver-node: 0.3.10 - eslint-module-utils: 2.14.0(@typescript-eslint/parser@8.71.0(eslint@9.39.5(jiti@2.7.0))(typescript@6.0.3))(eslint-import-resolver-node@0.3.10)(eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.71.0(eslint@9.39.5(jiti@2.7.0))(typescript@6.0.3))(eslint@9.39.5(jiti@2.7.0)))(eslint@9.39.5(jiti@2.7.0)))(eslint@9.39.5(jiti@2.7.0)) + eslint-module-utils: 2.14.0(@typescript-eslint/parser@8.71.0(eslint@9.39.5(jiti@2.7.0))(typescript@6.0.3))(eslint-import-resolver-node@0.3.10)(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.5(jiti@2.7.0)) hasown: 2.0.4 is-core-module: 2.17.0 is-glob: 4.0.3 @@ -11350,6 +11391,9 @@ snapshots: jsonfile: 6.2.1 universalify: 2.0.1 + fsevents@2.3.2: + optional: true + fsevents@2.3.3: optional: true @@ -11572,9 +11616,9 @@ snapshots: dependencies: lru-cache: 11.5.3 - html-encoding-sniffer@6.0.0: + html-encoding-sniffer@6.0.0(@noble/hashes@2.4.0): dependencies: - '@exodus/bytes': 1.16.0 + '@exodus/bytes': 1.16.0(@noble/hashes@2.4.0) transitivePeerDependencies: - '@noble/hashes' @@ -11928,17 +11972,17 @@ snapshots: jsbi@2.0.5: {} - jsdom@29.1.1: + jsdom@29.1.1(@noble/hashes@2.4.0): dependencies: '@asamuzakjp/css-color': 5.1.11 '@asamuzakjp/dom-selector': 7.1.1 '@bramus/specificity': 2.4.2 '@csstools/css-syntax-patches-for-csstree': 1.1.14(css-tree@3.2.1) - '@exodus/bytes': 1.16.0 + '@exodus/bytes': 1.16.0(@noble/hashes@2.4.0) css-tree: 3.2.1 - data-urls: 7.0.0 + data-urls: 7.0.0(@noble/hashes@2.4.0) decimal.js: 10.6.0 - html-encoding-sniffer: 6.0.0 + html-encoding-sniffer: 6.0.0(@noble/hashes@2.4.0) is-potential-custom-element-name: 1.0.1 lru-cache: 11.5.3 parse5: 8.0.1 @@ -11949,7 +11993,7 @@ snapshots: w3c-xmlserializer: 5.0.0 webidl-conversions: 8.0.1 whatwg-mimetype: 5.0.0 - whatwg-url: 16.0.1 + whatwg-url: 16.0.1(@noble/hashes@2.4.0) xml-name-validator: 5.0.0 transitivePeerDependencies: - '@noble/hashes' @@ -12648,7 +12692,7 @@ snapshots: nerf-dart@1.0.0: {} - next@16.4.0(@babel/core@7.29.7)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0): + next@16.4.0(@babel/core@7.29.7)(@playwright/test@1.58.2)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0): dependencies: '@next/env': 16.4.0 '@swc/helpers': 0.5.23 @@ -12667,6 +12711,7 @@ snapshots: '@next/swc-linux-x64-musl': 16.4.0 '@next/swc-win32-arm64-msvc': 16.4.0 '@next/swc-win32-x64-msvc': 16.4.0 + '@playwright/test': 1.58.2 sharp: 0.35.5(@types/node@25.9.9) transitivePeerDependencies: - '@babel/core' @@ -12742,12 +12787,12 @@ snapshots: npm@11.20.0: {} - nuqs@2.10.1(next@16.4.0(@babel/core@7.29.7)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0))(react@19.3.0): + nuqs@2.10.1(next@16.4.0(@babel/core@7.29.7)(@playwright/test@1.58.2)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0))(react@19.3.0): dependencies: '@standard-schema/spec': 1.1.0 react: 19.3.0 optionalDependencies: - next: 16.4.0(@babel/core@7.29.7)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0) + next: 16.4.0(@babel/core@7.29.7)(@playwright/test@1.58.2)(@types/node@25.9.9)(react-dom@19.3.0(react@19.3.0))(react@19.3.0) object-assign@4.1.1: {} @@ -13017,6 +13062,14 @@ snapshots: dependencies: find-up: 3.0.0 + playwright-core@1.58.2: {} + + playwright@1.58.2: + dependencies: + playwright-core: 1.58.2 + optionalDependencies: + fsevents: 2.3.2 + pngjs@5.0.0: {} pofile-ts@4.0.3: {} @@ -14419,7 +14472,7 @@ snapshots: tsx: 4.23.15 yaml: 2.9.1 - vitest@4.1.11(@types/node@25.9.9)(@vitest/coverage-v8@4.1.11)(jsdom@29.1.1)(vite@8.3.4(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)): + vitest@4.1.11(@types/node@25.9.9)(@vitest/coverage-v8@4.1.11)(jsdom@29.1.1(@noble/hashes@2.4.0))(vite@8.3.4(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)): dependencies: '@vitest/expect': 4.1.11 '@vitest/mocker': 4.1.11(vite@8.3.4(@types/node@25.9.9)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)) @@ -14444,7 +14497,7 @@ snapshots: optionalDependencies: '@types/node': 25.9.9 '@vitest/coverage-v8': 4.1.11(vitest@4.1.11) - jsdom: 29.1.1 + jsdom: 29.1.1(@noble/hashes@2.4.0) transitivePeerDependencies: - msw @@ -14505,9 +14558,9 @@ snapshots: whatwg-mimetype@5.0.0: {} - whatwg-url@16.0.1: + whatwg-url@16.0.1(@noble/hashes@2.4.0): dependencies: - '@exodus/bytes': 1.16.0 + '@exodus/bytes': 1.16.0(@noble/hashes@2.4.0) tr46: 6.0.0 webidl-conversions: 8.0.1 transitivePeerDependencies: diff --git a/scripts/algorithm-experiments/README.md b/scripts/algorithm-experiments/README.md new file mode 100644 index 000000000..e9d95a1d5 --- /dev/null +++ b/scripts/algorithm-experiments/README.md @@ -0,0 +1,578 @@ +# Off-pod algorithm experiments + +`executePackageExperiment` runs an inert, hash-verified package in a reviewed +local Docker image selected by an operator policy. It is development tooling, +not a pod API, installation service, or qualification decision. Known pod DAC +socket paths are rejected. Never bundle this runner into the pod web server. + +The image must be operator-reviewed and selected by immutable local `sha256:` +ID. The package's dependency-lock artifact uses the +`biometric-python-environment-v1` JSON schema: exact image ID, Python version, +implementation, OS, architecture and the complete sorted installed distribution +inventory. A restricted probe container with no package files mounted collects +that inventory before candidate execution. Any mismatch rejects the experiment. +The result carries the exact lock-byte hash and observed environment hash. +Do not pass a candidate-selected image or policy. + +Use `inspectPythonEnvironment(imageId)` to capture a lock from a reviewed image, +then include those JSON bytes as a hashed package artifact. This is an +**environment lock**, not a pip or uv dependency-resolution file; existing uv +locks may also be retained as package artifacts for review. Preparing an image +from the source dependency lock remains an operator's reviewed build step. The +runner never installs dependencies or executes build hooks. The immutable image +ID binds the configuration and layers; inventory verification additionally +makes interpreter/platform/package-version mismatches explicit. See +[Docker image identities](https://docs.docker.com/reference/cli/docker/image/pull/) +and [Python distribution metadata](https://docs.python.org/3.12/library/importlib.metadata.html). +A matching lock does not establish that dependencies or the image are safe. + +The process runs as UID 65534 with no network, capabilities, privilege elevation, +or writable root filesystem. Only a fresh verified copy of the package is +mounted, read-only. No pod files, host credentials, Docker socket, or database +are mounted. Docker's default seccomp profile remains enabled. Memory, swap, +PID count, stdout/stderr bytes, input bytes, and a deadline are bounded. The +runner checks that the daemon reports the required limit and seccomp support. +These controls use Docker's [documented container options](https://docs.docker.com/engine/containers/run/). + +CPU is restricted to one core and the deadline is the smaller of requested CPU +and wall time, conservatively charging idle time against CPU budget. Scheduler +and kill latency can overshoot the deadline; execution metadata is not a CPU or +peak-memory qualification receipt. Experiments still require measured resource +qualification before any promotion. Use a dedicated external Docker host or VM +with no pod credentials; containers share its kernel and are not a substitute +for maintaining that host's isolation and security updates. + +The package entrypoint is a Python executable implementing JSON Lines: + +1. Read one `initialize` object with `request` and `runHash`; reset all state and + initialize deterministically from the explicit seed/configuration. +2. Read each `frame` object with `frame` and `frameHash`. +3. Write exactly one `biometric-metric-frame-v1` response line per input frame. +4. Exit successfully at stdin EOF. Logs may go to stderr within the same total + output budget. Additional stdout is an error. + +Only one frame is in flight. Host validation runs before sending a frame and +before retaining its response. The caller supplies a trusted synchronous frame iterator. Before starting the +candidate, the runner writes canonical frames to a private bounded file and +checks their domain-separated SHA-256 hash against the request's `inputHash`. +The file is not mounted into the container; only one frame at a time is sent +from that frozen copy. The hash encoding is UTF-8 +`biometric-package-input-v1\n` followed by each canonical frame plus LF. +Input hash validation does not establish a clinical reference or holdout split; +those associations still need the qualification dataset pipeline. +The host staging parent stays mode 0700, with frozen input mode 0600. Only its +`package/` subdirectory (0755, files 0444) is bind-mounted read-only. The container +UID can traverse that mount without granting other host users access through the +private parent. An opt-in real-Docker test verifies these modes after container +creation, successful execution as UID 65534, and removal after completion. +Package files and outputs are never imported as host code. Cleanup removes the +owned container and staging directory on normal completion and caught errors; +hard host termination is handled by the explicit container recovery sweep +described below, followed by ownership-checked private staging recovery. No arbitrary dependency installation occurs during execution. + +Run the opt-in Docker regression tests with a reviewed Python image: + +```sh +BIOMETRIC_EXPERIMENT_TEST_IMAGE=sha256: pnpm exec vitest run scripts/algorithm-experiments/container.test.ts +``` + +The tests cover a valid exchange, filesystem/network/environment/PID probes, +output floods, memory exhaustion, deadlines, and invalid results. They do not +qualify any biometric algorithm or prove resistance to kernel vulnerabilities. + +## Determinism evidence + +`checkPackageDeterminism` runs the same manifest, byte snapshots, request and +operator policy two to five times, with a fresh container each time. Its trusted +frame factory must reproduce the retained input: every run independently +verifies that hash. It hashes every response, including serialized state, in +order using the `biometric-package-output-v1` domain. Execution timings and +transport measurements are excluded from the output identity. + +The result records run/input/output identities and execution metadata. A +changing state fails even when numeric readings are identical. A deterministic +result is only one qualification dimension: it does not approve dependency +compatibility, resource fitness, artifact/noise behavior, held-out agreement, +shadow use, installation, or live activation. Those checks and reviewed release +promotion remain required. + +## File-based command line + +Run these commands on the external experiment host, from the repository: + +```sh +pnpm biometrics:experiment lock sha256: environment.lock.json +pnpm biometrics:experiment inspect proposal.json intake.json +pnpm biometrics:experiment input-hash frames.jsonl 33554432 input-identity.json +pnpm biometrics:experiment run proposal.json request.json frames.jsonl operator-policy.json experiment.json +pnpm biometrics:experiment determinism proposal.json request.json frames.jsonl operator-policy.json 2 determinism.json +``` + +The proposal is a single inert JSON envelope with `schema` equal to +`biometric-package-proposal-v1`, a complete `manifest`, and `files` containing +`{ "path": "engine.py", "base64": "..." }` entries for every declared artifact. +Include the captured environment lock as the manifest's `dependencyLock` file; +its exact bytes, like all package files, must match the artifact size and hash. +`inspect` verifies the proposal without launching Docker or executing its code. + +`frames.jsonl` contains one `biometric-signal-frame-v1` object per line. +`request.json` follows `packageRunRequestSchema` in `protocol.ts`: set its +`packageHash` from the intake report and `inputHash` from the input-identity +report, plus explicit hardware, sample rate, side, configuration, and seed. + +`operator-policy.json` is separate from candidate artifacts and contains the +reviewed image ID and positive integer ceilings: `maximumWallMs` (at most one +hour), `maximumMemoryBytes`, `maximumProcesses`, `maximumOutputBytes`, and +`maximumInputBytes`. Select these for the external host; they do not establish +that an algorithm fits the pod. Candidate resource requests must fit the policy. + +All file inputs must be bounded regular files, not symlinks, devices or FIFOs. +JSONL has a separate per-line cap. Reports are mode 0600, written completely +before atomic publication, and never overwrite an existing path. Errors return +nonzero and do not publish a success report. The report includes raw candidate +outputs and should be treated as private biometric data. No command sends it to +a reviewer or installs the package. A successful `run` is explicitly marked +`experiment_only`; use the remaining qualification checks and normal reviewed +release process before promotion. + +## Host resource observations + +Experiment reports now include `execution.measurements`, collected from the +Docker Engine stats API on the host, never from candidate output. The collector +uses the actual created container ID, validates each response's identity, and +records observed cumulative CPU milliseconds, memory bytes including cache, +and process counts. It attempts sampling every 250 ms without overlapping +requests, and requests a final sample while the candidate is still waiting for +stdin EOF. Reads have time and byte limits. + +The receipt records attempts, usable samples, and maximum gaps including +startup and shutdown. Missing, stale, or regressing samples leave it incomplete. Memory is an observed peak, +not an exact lifetime high-water mark; CPU is the largest observed cumulative +counter, not a guaranteed final lifetime total. A process may consume resources +between samples. The collector's overhead counts toward the experiment's wall +deadline. Qualification must inspect coverage and gaps and retain these limits. +No fit-for-pod decision is inferred from measurements on an external host. + +Currently measurement supports local Unix Docker endpoints and API 1.41 or +newer. Unsupported transports or failed observations leave nullable counters +and incomplete evidence; they do not fabricate zero usage or waive the existing +container resource limits. See the [Docker stats API](https://docs.docker.com/reference/api/engine/version/v1.51/). + +## Artifact/noise fixture suites + +```sh +pnpm biometrics:experiment fixtures proposal.json suite.json operator-policy.json fixture-report.json +``` + +A reviewed suite uses `biometric-package-fixtures-v1` and a `fixtures` array. +Each fixture has a unique `id`, `category`, full run `request`, inline signal +`frames`, and `expectations`. Its request must bind the proposal and actual +frame hashes. The suite file is bounded to 32 MiB; inline fixtures are intended +for focused signal tests, not whole-night held-out comparisons. + +Expectation windows use inclusive `startMs`, exclusive `endMs`, and a declared +`metric`. A `numeric` expectation specifies `minimum`, `maximum`, +`minimumReadings`, and `minimumQuality`. Missing/null readings cannot satisfy +numeric coverage. An `abstain` expectation permits absent or null output and +rejects numeric values. Duplicate readings, unexpected metrics/times, +out-of-range values, low quality, insufficient coverage and overlapping +expectations fail. Windows must fall inside the supplied input interval. +Clean-signal fixtures must require numeric output; empty-bed fixtures must +require abstention. + +Each fixture executes in a fresh isolated container. Reports retain the suite, +fixture, input and output identities, the complete frozen suite with frames and +expectations, execution output, independent resource observations and failures. +A crashed or invalid-output case remains a failed case in the report. + +Required category coverage is `clean_signal`, `pump`, `empty_bed`, `motion`, +`cross_side`, `harmonics`, `gaps`, `warmup`, and `boundary_reset`. Omitted categories +make an otherwise passing report `incomplete`. All covered checks passing yields +`fixture_checks_passed`, never automatic qualification or installation. Category +labels do not prove that a signal realistically represents that condition: +review the exact hashed suite and its expected behavior. The test suite includes +synthetic clean/empty-bed examples to verify this workflow; representative +shipped-algorithm fixtures, held-out agreement, compatibility and pod resource +qualification remain separate requirements. + +Each fixture suite also requires an explicit reviewed `resourcePolicy` with +schema `biometric-package-resource-policy-v1`, `maximumWallMs`, `maximumCpuMs`, +`maximumMemoryBytes`, `maximumProcesses`, `minimumSamples`, +`maximumObservationGapMs`, and `evidenceMode` equal to +`sampled_counters_and_enforced_limits`. There are no production default +thresholds. The policy is part of the frozen suite hash and every per-case +resource assessment records its own policy hash. + +An assessment can be `within_observed_limits`, `insufficient_evidence`, or +`failed`. It requires consistent host receipts, adequate sample count and gaps, +and enforced limits no wider than the reviewed budget. Known wall, observed +CPU/memory/process, or enforced-limit overruns fail even when sampling is also +incomplete. Missing measurements never mean zero cost. A passing signal check +with incomplete resource evidence makes the overall fixture report incomplete; +a resource failure makes it fail. These sampled observations plus hard limits +still do not establish exact lifetime costs or target-pod fitness. + +## Reference comparison + +```sh +pnpm biometrics:experiment compare comparison-input.json comparison-report.json +``` + +This command reuses the application's reference normalization, point/interval +alignment, and common-support numeric comparison. The input uses schema +`biometric-package-comparison-v1`, a `metricId`, whole-night `startMs`/`endMs` +(up to 24 hours), explicit `minimumQuality`, raw `referenceImport`, frozen +`alignment`, and exactly three `outputs` labelled `default`, `current`, and +`candidate`. Each output supplies its `manifest`, run `request`, and complete +executor `result`. Inputs are bounded to 32 MiB. Run IDs include configuration; +reported baseline/candidate hashes in the shared comparison are those run IDs. + +Each selected manifest metric must declare a `reference` contract with a +supported metric definition and nullable `windowMs`. HRV definitions require +an explicit window, and units must be canonical (`bpm`, `breaths/min`, or `ms`). +A free-text description or a coincidentally matching output name is insufficient. +All packages must agree on that contract, input identity, hardware, sample rate, +side and input-frame sequence. Different HRV definitions or windows cannot be +compared as the same metric. + +Reference rows are normalized using the shared adapter. Out-of-night rows and +compatibility exclusions are reported explicitly. Comparisons use identical, +unambiguous observation timestamps shared by all three runs before reference +pairing; rejected/low-quality readings remain missing support, never filled +values. MAE, bias and coverage differences are descriptive reference agreement. +The command does not trust file-provided results as execution attestations, +record holdout access, estimate whole-night uncertainty, designate an objective, +or authorize promotion. Those require the authoritative experiment/qualification +workflow, not a report assembled from arbitrary files. + +## Durable study ledger (orchestration component) + +`PackageExperimentLedger` maintains frozen study identities, exact trial +requests, idempotency keys, expiring worker leases and report hashes in a +**dedicated off-pod SQLite database**. Its generated schema and migrations are +under `src/db/package-experiment-*`; generate changes with +`pnpm exec drizzle-kit generate --config=drizzle.package-experiments.config.ts`. +Do not use either pod database for this ledger. Initialization refuses nonempty +foreign databases, uses an application identifier, and rejects future format +versions rather than migrating them backward. + +Study nights bind subject/side, start/end, input/reference hashes, split and +protocol hash. Splits must be chronological and nonoverlapping. Claims record +exposure transactionally before selected input is opened. Every attempt counts, +including failures, expired leases and retries; changed study IDs or corrected +input/reference hashes cannot erase overlap with an earlier attempt on the +same subject and side. A stale attempt cannot renew a replacement's lease or +publish over its report. Completed trials return their retained report identity +rather than silently running again. + +Claim-time `unexposedHoldoutAtClaim` is historical information. Re-read +`currentEvidence` for current eligibility: another overlapping trial can make a +previously unique holdout attempt non-unique. Keeping this ledger across studies +is required; a new/deleted ledger, relabelled subjects, or data inspected outside +its workflow cannot prove independent holdout use. Final qualification must +bind subjects and windows to retained datasets and recheck exposure while +publishing its decision. + +The exploratory `run`, `compare`, `determinism`, and `fixtures` commands do not +claim studies through this ledger and cannot issue independent-holdout promotion +evidence. `runManagedTrial` and the `managed` command integrate it as described +below. `assessManagedQualification` combines verified external evidence as +described below; installation remains a separate reviewed code/release step. + +## Managed whole-night runner + +`runManagedTrial` in `managed.ts` accepts a reserved ledger trial, its frozen +`biometric-managed-protocol-v1`, exactly three package variants, a private report +directory, and a trusted retained-night loader. The protocol freezes hardware, +sample rate, metric, quality threshold, reference alignment, whole-night +bootstrap settings and resource policy. Each trial variant binds the package +hash, runtime image, and `configHash({ config, seed })`. + +It verifies package/configuration/protocol identities before claiming a lease. +The loader runs only after all selected night exposures are committed. Loaded +references must match the study's subject, side and shared import content hash. +Frames must lie within their frozen night and match its actual input hash. All +three variants run in independent containers; results use the shared typed +comparison and paired whole-night uncertainty functions. Resource assessments +remain attached to every variant. + +Private per-night and trial reports are published under their content hashes, +then the ledger's live attempt fence selects the winning report. Heartbeats and +checks between expensive steps renew leases; lost leases abort active execution +or reject publication. Failures retain partial-night evidence and exposure. +An old worker may leave an unreferenced artifact but cannot overwrite the winning +ledger pointer. Restarted completed trials verify the complete retained report graph before +reusing it and never silently reread holdout input. Missing or corrupted +per-night artifacts fail rather than trigger a new evaluation. + +A successful run means the evaluation completed, not that its candidate passed +all promotion criteria. Check current ledger exposure, coverage, uncertainty, +resource assessments, deterministic/noise/compatibility evidence and reviewed +release admission. The runner does not install packages or claim pod fitness. +Hard process termination can still leave orphan containers/artifacts; use the +ownership-checked recovery procedures below. Combined external qualification +rechecks the retained evidence and current exposure without authorizing installation. + +## Managed descriptor command + +```sh +pnpm biometrics:experiment managed ./experiment-workspace descriptor.json receipt.json +``` + +The command creates a private workspace (0700) containing a dedicated +`experiments.sqlite` (0600) and private `reports/`. Reuse the same workspace +across studies; deleting or replacing it destroys the exposure history needed +for future decisions. Existing public directories, symlinked workspace/report +paths, symlinked database/sidecar files, and foreign databases are rejected. +No pod database is opened or migrated. + +Managed execution acquires an OS-released SQLite write lock in `.writer.sqlite` +for the entire workspace operation. Competing controllers fail before changing +the ledger or starting containers. Never unlink this lock file to resolve a busy +workspace: doing so would create a second lock inode. A killed controller releases +the database lock automatically; container and private-staging recovery are still +separate responsibilities. Low-level trial APIs are trusted building blocks; +`managed` is the bounded persistent-workspace entry point. + +The default workstation storage policy allows 2,048 report files, 1 GiB of total +report bytes, 32 MiB per report and a 64 MiB ledger, while requiring 1 GiB of free +space plus room for ledger growth/rollback and the next report. These are off-pod +storage limits, not production pod resource budgets. Supply a strict JSON policy +as the optional fourth managed argument to override them: + +```sh +pnpm biometrics:experiment managed WORKSPACE DESCRIPTOR RECEIPT STORAGE_POLICY +``` + +Policy fields are `maximumReportFiles`, `maximumReportBytes`, +`maximumSingleReportBytes`, `maximumDatabaseBytes`, and `minimumFreeBytes`. +Checks run before work, before each retained-night load, and before each report +publication. They bound managed report files, including abandoned temporary files, +and SQLite growth via `max_page_count` with DELETE-mode rollback journals. The +report directory must contain only private, owned, single-link regular files. +Symlinks, subdirectories and unsafe database/lock files are rejected. No existing +reports or exposure history are deleted to satisfy a quota. When publication is +refused, the failure-finalization behavior below still applies. + +Free-space checks are admission snapshots; unrelated applications can consume +space afterward. The ledger cap plus its rollback journal is accounted separately +from report bytes. Operator source files, the output receipt path, Docker storage, +and container input staging outside this workspace are not covered by these +workspace limits. Their existing per-input/output limits still apply; aggregate +staging quotas and crash recovery require separate controls. + +A descriptor uses `biometric-managed-descriptor-v1` with: + +- `study`: the frozen `biometric-package-study-v1`, including all chronological + night assignments and their input/reference hashes. +- `protocol`: the exact managed protocol whose canonical hash is the study's + `protocolHash`. +- `split` and an `idempotencyKey` for this exact trial. +- `variants`: exactly `default`, `current`, and `candidate`, each specifying + `proposal` (file path), `config`, `seed`, and the operator execution `policy`. +- `nights`: exactly the selected split's night IDs, each with `frames` (JSONL + path) and `referenceImport` (raw reference-import JSON path). + +Failure finalization also runs when the failure report cannot be published (for +example, a full report volume). If the ledger remains writable and the lease is +current, the attempt becomes `failed` with a null report hash. No nonexistent +artifact is claimed. Publication or ledger-finalization failures are returned +alongside the original error in an `AggregateError`; its cause remains the original +failure. Failed attempts retain their exposure history. A stale lease cannot end +or annotate a replacement attempt. A failed ledger write leaves state unconfirmed +and requires subsequent lease recovery; the CLI does not report successful cleanup. + +Relative source paths resolve from the descriptor's directory. The descriptor +is operator-owned input, not candidate-provided instructions. Source files must +pass the bounded regular-file reader. Study reference hashes use the shared +`previewReferenceImport(...).summary.contentHash`; input hashes use +`packageInputHash`. Keep retained sources and proposals alongside the study for +reproducibility; metadata alone does not prove a complete night or subject +identity. + +Night files are opened only after the ledger records exposure. The receipt +contains the winning trial/report identity, report path and current ledger +evidence. Repeating an unchanged completed descriptor verifies and reuses its +report without rereading night files. Reusing its idempotency key with changed +configuration or study fails. Review current exposure again at any later +promotion decision, since other trials can invalidate first-use evidence. +A receipt is not an installation or qualification approval. + +## Recovery after a terminated experiment controller + +```sh +pnpm biometrics:experiment recover recovery-report.json +``` + +Experiment and environment-probe containers carry immutable version, invoking +host UID, kind and expiry labels. Expiry includes the maximum execution deadline +plus two minutes for startup and cleanup. Run this command after restarting an +experiment host and periodically while accepting work. It checks labels, names +and full container IDs on the currently selected Docker daemon, then force-removes +only overdue owned containers by immutable ID. Unexpired, unlabelled and foreign +containers remain untouched. The private report records inspected count and +removed IDs; repeating a sweep is safe. A daemon outage fails the command and +must be retried. Legacy containers without these labels require manual review. + +Recovery does not retroactively qualify an interrupted trial or erase exposure. +The ledger's expired lease permits a fresh fenced attempt, with earlier holdout +access still counted. The container sweep does not delete retained reports; its subsequent staging +sweep is described below. This is not a background supervisor: the external host +must schedule it. Host clock correctness is required for absolute expiry. Aggregate +temporary-staging quotas and periodic recovery scheduling remain separate work. + +## Retained evidence verification + +`readManagedEvidence(ledger, trialId, reportDirectory)` starts from the successful +ledger-selected report hash. It verifies every linked night artifact, exact +study/request/protocol/attempt identities, the complete selected-night set, +reference content and subject/side, all three package/configuration/input/image +identities, and declared resource limits. It recomputes reference comparisons, +resource assessments and whole-night uncertainty, requiring both the per-night +and top-level summaries to match those calculations. Updating hashes around an +altered score does not make that score valid. + +Completed managed runs use this verifier before returning a cached result. It +reads reports only, without rerunning candidate code or reopening retained raw +night inputs. The returned evidence includes a fresh ledger exposure snapshot; +later overlapping access can invalidate a previously unique holdout attempt. +This does not prove that a local administrator has never replaced the ledger or +fabricated raw execution records. The experiment host and its retained history +remain trusted. Qualification must recheck current exposure when publishing its +decision and still requires objective criteria, determinism, representative +fixtures, compatibility, target-pod resources and reviewed release promotion. + +## Frozen accuracy criteria + +Managed protocols may include `accuracyPolicy` using schema +`biometric-package-accuracy-policy-v1`. Choose and review its explicit criteria +before freezing the study: `minimumMaeImprovement`, +`maximumNightMaeRegression`, `maximumNightP95Regression`, +`maximumNightAbsoluteBiasIncrease`, `minimumCommonPairsPerNight`, +`minimumReferenceCoverage`, `maximumReferenceCoverageLoss`, and +`maximumLostTimestampFraction`. Error thresholds use the selected metric's +canonical unit; coverage/loss thresholds are fractions. There are no implicit +production thresholds. Changing criteria changes the protocol hash and cannot +retrofit the original study. Legacy protocols without criteria remain readable +but cannot meet the accuracy gate. + +The managed receipt's `accuracy` assessment is recomputed from verified retained +evidence and current ledger exposure. It requires a unique current holdout +attempt, sufficient common pairs on every selected night, coverage and support +within the frozen limits, no excessive per-night MAE/tail/bias regression, and +an estimated whole-night confidence interval supporting the requested MAE +improvement against both default and current. Missing or degenerate uncertainty, +omitted nights, ambiguous support and absent criteria remain insufficient. +Known violations produce `criteria_not_met` even if other evidence is missing. + +`criteria_met` means only that these reference-agreement criteria passed. It is +not clinical validation or package qualification: determinism, resource/noise +fixtures, compatibility, pod fitness, prospective monitoring and reviewed release +promotion still apply. The assessment retains exact failures and missing evidence +and never installs code or changes live processing. + +## Combined external preflight + +```sh +pnpm biometrics:experiment preflight proposal.json plan.json suite.json operator-policy.json preflight-report.json +``` + +The plan uses `biometric-package-preflight-plan-v1`, `suiteHash` (the canonical +`dataHash` of the parsed fixture suite), `repetitions` (2–5), `configHash` +(`configHash({ config, seed })`), `hardwareId`, `sampleRateHz`, and `side`. +Every fixture must use that exact candidate configuration and seed, hardware, +rate and side. A changed suite or configuration is rejected before execution. +Use reviewed synthetic fixtures here, not undisclosed holdout data. + +The runner snapshots candidate bytes, suite and operator policy, executes the +fixture suite, then repeats every fixture in independent containers to check +full-output/state determinism. Each repeat also receives a resource assessment +under the suite's frozen policy. Execution failures and nondeterminism fail the +preflight. Missing fixture categories or incomplete resource observations keep +it incomplete. Cancellation propagates to active experiments and stops further +fixtures/repeats. Reports retain the exact plan and its hash, fixture report, +per-fixture determinism reports, resource assessments and bounded errors. + +`external_checks_passed` is external preflight only. Managed studies can bind +the preflight artifact as described below; standalone reports cannot prove +holdout independence or pod fitness. The final combined qualification decision +remains separate. The candidate still needs representative fixture review, current +held-out accuracy, target-pod compatibility/resources, and normal reviewed +release promotion before it can become live. + +## Binding preflight to a managed study + +Include `preflightPlan` in the managed protocol and `preflightSuite` (a suite-file +path relative to the descriptor) in the managed descriptor. Both are required +together. The study protocol hash freezes the plan, including suite identity, +repetitions and candidate configuration/seed. Its hardware, rate and side must +match the managed study. The runner snapshots the suite before asynchronous work +and rejects mismatches before claiming a trial. + +During the leased attempt, preflight runs under the same cancellation heartbeat +before night comparisons. Its complete report, including candidate manifest, is +published by content hash and referenced by the final trial report. Failed or +incomplete preflight remains visible alongside the night results; completing a +trial does not imply that preflight passed. Failure reports retain a completed +preflight reference if a later step fails. Legacy trials without a plan have no +preflight evidence and cannot supply that qualification dimension. + +The retained-evidence verifier reopens the bound preflight artifact and checks +its plan, candidate/image, exact fixture set, fixture inputs and output protocol, +resource limits and measurements, repeat count, run identities and determinism +hashes. It recomputes fixture outcomes, resource assessments, missing categories +and overall status. Missing, altered or inconsistent artifacts fail cached reuse. +Completed file workflows do not reread the original suite or night input files. +The managed receipt exposes verified preflight status alongside accuracy; target +pod compatibility, representative-fixture review and release promotion remain +required before an algorithm can become live. + +## Combined external qualification assessment + +`assessManagedQualification(ledger, trialId, reportDirectory)` verifies the full +retained graph once, reads current holdout exposure, assesses frozen accuracy, +and combines it with bound preflight and every night/variant's resource evidence. +The managed command includes this `qualification` object and an `assessmentHash` +in its receipt. The hash binds trial/report/protocol/request identities, current +exposure and all assessment dimensions. Re-reviewing unchanged evidence is stable; +later overlapping holdout use changes the snapshot and invalidates its independent +holdout claim even though the original execution report remains unchanged. + +The combined status is `failed`, `insufficient_evidence`, or +`external_criteria_met`. Known failures dominate missing evidence, with both +reason lists retained. Passing accuracy cannot waive absent/failed preflight, +missing or duplicated resource rows, substituted resource policy, budget excess, +or inadequate measurements for default/current/candidate on any selected night. + +This is an external evidence snapshot, not a durable installation authorization. +`installationAuthorized` remains false, and the receipt lists outstanding +representative-fixture review, target-pod compatibility and resource qualification, +prospective health/reference comparison, and reviewed shipped-registry release. +A future promotion step must recheck authoritative evidence at publication; an +old exported assessment must never substitute for that current check. + + +### Abandoned private staging + +Before freezing inputs, the executor writes a private, fsynced ownership receipt +inside its mode-0700 staging directory. It binds the directory name/device/inode, +host UID, controller PID, exact container name, and wall deadline plus two-minute +grace. These receipts require a Unix host UID (macOS or Linux). Package code cannot +read or modify them through its package-only mount. + +`recover OUTPUT` first performs the existing expired-container sweep, then scans +host temporary staging. Its output is `biometric-experiment-recovery-v1` with +separate `containers` and `staging` results. Staging is removed only when the +receipt is valid and expired, its controller PID is confirmed absent with ESRCH, +and Docker confirms the named container does not exist. A present container or +an unavailable daemon never proves absence. PID reuse/permission errors retain +staging conservatively. Directory and marker identities are rechecked before +removal; markers must be private, owned, bounded regular files with one link. +The sweep is bounded to 16,384 temporary-directory entries and 128 candidates. + +Normal cleanup also preserves staging if container removal fails. Legacy or +unmarked directories, damaged receipts, symlinks, and unproven ownership are left +untouched. A crash between creating the empty directory and writing its receipt +can leave an unmarked empty directory; recovery will not guess its ownership. +Reports and exposure ledgers are not touched. This is an explicit recovery sweep; +a periodic host service and aggregate temporary-staging quotas remain separate. diff --git a/scripts/algorithm-experiments/accuracy-policy.ts b/scripts/algorithm-experiments/accuracy-policy.ts new file mode 100644 index 000000000..d9414ffd5 --- /dev/null +++ b/scripts/algorithm-experiments/accuracy-policy.ts @@ -0,0 +1,15 @@ +import { z } from 'zod' + +/** Operator-reviewed criteria, frozen into the protocol before holdout access. + * There are deliberately no production defaults or clinical accuracy claims. */ +export const packageAccuracyPolicySchema = z.object({ + schema: z.literal('biometric-package-accuracy-policy-v1'), + minimumMaeImprovement: z.number().finite().positive(), + maximumNightMaeRegression: z.number().finite().nonnegative(), + maximumNightP95Regression: z.number().finite().nonnegative(), + maximumNightAbsoluteBiasIncrease: z.number().finite().nonnegative(), + minimumCommonPairsPerNight: z.number().int().positive().max(100000), + minimumReferenceCoverage: z.number().positive().max(1), + maximumReferenceCoverageLoss: z.number().min(0).max(1), + maximumLostTimestampFraction: z.number().min(0).max(1), +}).strict() diff --git a/scripts/algorithm-experiments/accuracy.test.ts b/scripts/algorithm-experiments/accuracy.test.ts new file mode 100644 index 000000000..b05be08e8 --- /dev/null +++ b/scripts/algorithm-experiments/accuracy.test.ts @@ -0,0 +1,82 @@ +// @vitest-environment node +import { createHash } from 'node:crypto' +import { describe, expect, it } from 'vitest' +import { compareNumericConfigurations } from '../../src/biometrics/backtest/numeric-comparison' +import { createNightUncertainty } from '../../src/biometrics/backtest/uncertainty-core' +import { normalizeReferenceImport } from '../../src/biometrics/references/normalize-core' +import { assessPackageAccuracy } from './accuracy' +import { managedProtocolSchema } from './managed-protocol' +import { fixtureResourceTestPolicy } from './resource-test-policy' +import { configHash } from '../../src/biometrics/algorithms/canonical' + +type Input = Parameters[0] +const policy = { schema: 'biometric-package-accuracy-policy-v1', minimumMaeImprovement: 0.5, maximumNightMaeRegression: 0, + maximumNightP95Regression: 0, maximumNightAbsoluteBiasIncrease: 0, minimumCommonPairsPerNight: 2, + minimumReferenceCoverage: 1, maximumReferenceCoverageLoss: 0, maximumLostTimestampFraction: 0 } +function fixture(): Input { + const protocol = managedProtocolSchema.parse({ schema: 'biometric-managed-protocol-v1', hardwareId: 'pod4', sampleRateHz: 500, metricId: 'heart_rate', minimumQuality: 0.8, + alignment: { version: 'reference-alignment-v1', pointToleranceMs: 0, clockOffsetMs: 0, offsetSource: 'fixed' }, + uncertainty: { version: 'paired-night-percentile-v1', seed: 'accuracy-test', resamples: 200, minimumNights: 3 }, + resourcePolicy: fixtureResourceTestPolicy, accuracyPolicy: policy }) + const reference = normalizeReferenceImport({ subjectId: 'fixture', side: 'left', source: 'synthetic', + rows: [0, 1].map(index => ({ recordKey: String(index), metric: 'heart_rate', value: 60, unit: 'bpm', timestampMs: 1700000000000 + index * 1000 })) }) + const nights = [0, 1, 2].map((index) => { + const outputs = (['default', 'current', 'candidate'] as const).map((label, variant) => ({ label, configHash: String(variant), + observations: [0, 1].map(point => ({ timestampMs: 1700000000000 + point * 1000, value: variant === 2 ? 60 : 64 + index + (1 - variant) })) })) + const comparison = { schema: 'biometric-package-comparison-report-v1' as const, inputHash: 'a', referenceHash: 'b', metricId: 'heart_rate', + contract: { metric: 'heart_rate' as const, windowMs: null }, alignment: protocol.alignment, startMs: 1700000000000, endMs: 1700000060000, + minimumQuality: 0.8, excludedBoundaryReferences: 0, compatibilityExclusions: {}, referenceSummary: reference.summary, + comparisons: compareNumericConfigurations('heart_rate', null, outputs, reference.measurements, protocol.alignment), + interpretation: 'descriptive_reference_agreement_not_independent_holdout_or_promotion_evidence' as const } + return { nightId: String(index), reportHash: 'c', comparison, resources: [] } + }) + const uncertainty = createNightUncertainty(text => createHash('sha256').update(text).digest(), 'equal_night_mean_common_mae_difference') + return { protocol, nights, uncertainty: (['default', 'current'] as const).map(baseline => ({ baseline, + report: uncertainty(nights.map(night => ({ cohortId: night.nightId, difference: night.comparison.comparisons.find(row => row.baseline === baseline && row.candidate === 'candidate')?.differences.commonMae ?? null })), protocol.uncertainty) })), + currentEvidence: { trialId: 'fixture', state: 'succeeded', reportHash: 'hash', otherExposures: [], uniqueHoldoutAttempt: true } } +} +describe('frozen package accuracy criteria', () => { + it('requires supported improvement against both baselines without authorizing installation', () => { + expect(assessPackageAccuracy(fixture())).toMatchObject({ status: 'criteria_met', failures: [], missing: [], interpretation: expect.stringContaining('requires_full_package') }) + const input = fixture() + const report = input.uncertainty[1].report + if (report.interval) report.interval.upper = 0 + expect(assessPackageAccuracy(input)).toMatchObject({ status: 'criteria_not_met', failures: ['current:improvement_not_supported'] }) + }) + it('cannot backfill criteria into an already frozen protocol or reuse exposed holdout', () => { + const input = fixture(), frozen = configHash(input.protocol) + delete input.protocol.accuracyPolicy + expect(configHash(input.protocol)).not.toBe(frozen) + expect(assessPackageAccuracy(input).missing).toContain('no_frozen_accuracy_policy') + input.currentEvidence.uniqueHoldoutAttempt = false + expect(assessPackageAccuracy(input).missing).toContain('no_unique_current_holdout') + }) + it('rejects coverage loss and individual-night regressions even with favorable aggregate intervals', () => { + const input = fixture() + const row = input.nights[0].comparison.comparisons.find(row => row.candidate === 'candidate' && row.baseline === 'default') + if (!row) throw new Error('Missing fixture comparison') + row.ownSupport.candidate.referenceCoverage = 0.5 + row.differences.referenceCoverage = -0.5 + row.lostObservationTimestamps = 1 + row.differences.commonMae = 1 + row.differences.commonP95AbsoluteError = 2 + if (row.commonSupport.candidate.metrics) row.commonSupport.candidate.metrics.signedBias = 99 + const result = assessPackageAccuracy(input) + expect(result.status).toBe('criteria_not_met') + expect(result.failures).toEqual(expect.arrayContaining(['0:default:reference_coverage_below_minimum', '0:default:reference_coverage_loss', + '0:default:observation_support_loss', '0:default:night_mae_regression', '0:default:night_tail_regression', '0:default:night_bias_regression'])) + }) + it('does not treat omitted nights, missing pairs or degenerate intervals as successful evidence', () => { + const input = fixture() + input.uncertainty[0].report.omittedNights = 1 + input.uncertainty[1].report = { ...input.uncertainty[1].report, status: 'degenerate', interval: null } + const row = input.nights[0].comparison.comparisons.find(row => row.candidate === 'candidate' && row.baseline === 'default') + if (!row) throw new Error('Missing fixture comparison') + row.commonSupport.candidate.pairedCount = 0 + row.ambiguousObservationTimestamps.candidate = 1 + expect(assessPackageAccuracy(input)).toMatchObject({ status: 'insufficient_evidence', missing: expect.arrayContaining([ + 'default:insufficient_whole_night_uncertainty', 'current:insufficient_whole_night_uncertainty', + '0:default:insufficient_common_pairs', '0:default:ambiguous_observation_support', + ]) }) + }) +}) diff --git a/scripts/algorithm-experiments/accuracy.ts b/scripts/algorithm-experiments/accuracy.ts new file mode 100644 index 000000000..900052d73 --- /dev/null +++ b/scripts/algorithm-experiments/accuracy.ts @@ -0,0 +1,66 @@ +import { configHash } from '../../src/biometrics/algorithms/canonical' +import { packageAccuracyPolicySchema } from './accuracy-policy' +import { readManagedEvidence } from './managed-evidence' + +type Evidence = ReturnType +type AccuracyInput = Pick + +/** Pure assessment of already verified evidence. The authoritative entry point + * below reads the retained graph and fresh ledger state first. */ +export function assessPackageAccuracy(evidence: AccuracyInput) { + const failures: string[] = [], missing: string[] = [] + const rawPolicy = evidence.protocol.accuracyPolicy + const policy = rawPolicy === undefined ? null : packageAccuracyPolicySchema.parse(rawPolicy) + if (!policy) missing.push('no_frozen_accuracy_policy') + if (!evidence.currentEvidence.uniqueHoldoutAttempt) missing.push('no_unique_current_holdout') + if (!evidence.nights.length) missing.push('no_nights') + for (const night of evidence.nights) { + for (const baseline of ['default', 'current'] as const) { + const key = `${night.nightId}:${baseline}` + const comparison = night.comparison.comparisons.find(row => row.candidate === 'candidate' && row.baseline === baseline) + if (!comparison) { + missing.push(`${key}:missing_comparison`) + continue + } + if (comparison.commonSupport.candidate.status === 'incompatible' || comparison.commonSupport.baseline.status === 'incompatible') { + failures.push(`${key}:incompatible_metric`) + continue + } + if (comparison.ambiguousObservationTimestamps.baseline || comparison.ambiguousObservationTimestamps.candidate) missing.push(`${key}:ambiguous_observation_support`) + if (!policy) continue + const candidate = comparison.commonSupport.candidate, previous = comparison.commonSupport.baseline + if (!candidate.metrics || !previous.metrics || candidate.pairedCount < policy.minimumCommonPairsPerNight + || previous.pairedCount < policy.minimumCommonPairsPerNight) missing.push(`${key}:insufficient_common_pairs`) + const own = comparison.ownSupport.candidate + if (own.referenceCoverage === null) missing.push(`${key}:missing_reference_coverage`) + else if (own.referenceCoverage < policy.minimumReferenceCoverage) failures.push(`${key}:reference_coverage_below_minimum`) + const coverageChange = comparison.differences.referenceCoverage + if (coverageChange === null) missing.push(`${key}:missing_coverage_difference`) + else if (coverageChange < -policy.maximumReferenceCoverageLoss) failures.push(`${key}:reference_coverage_loss`) + const baselineCount = comparison.ownSupport.baseline.validObservationCount + if (!baselineCount) missing.push(`${key}:missing_baseline_observations`) + else if (comparison.lostObservationTimestamps / baselineCount > policy.maximumLostTimestampFraction) failures.push(`${key}:observation_support_loss`) + if (comparison.differences.commonMae !== null && comparison.differences.commonMae > policy.maximumNightMaeRegression) failures.push(`${key}:night_mae_regression`) + if (comparison.differences.commonP95AbsoluteError !== null && comparison.differences.commonP95AbsoluteError > policy.maximumNightP95Regression) failures.push(`${key}:night_tail_regression`) + if (candidate.metrics && previous.metrics + && Math.abs(candidate.metrics.signedBias) - Math.abs(previous.metrics.signedBias) > policy.maximumNightAbsoluteBiasIncrease) failures.push(`${key}:night_bias_regression`) + } + } + for (const baseline of ['default', 'current'] as const) { + const report = evidence.uncertainty.find(row => row.baseline === baseline)?.report + if (!report || report.status !== 'estimated' || !report.interval || report.omittedNights > 0 + || report.scoredNights !== evidence.nights.length) missing.push(`${baseline}:insufficient_whole_night_uncertainty`) + else if (policy && (report.estimate === null || report.estimate > -policy.minimumMaeImprovement + || report.interval.upper > -policy.minimumMaeImprovement)) failures.push(`${baseline}:improvement_not_supported`) + } + return { schema: 'biometric-package-accuracy-assessment-v1' as const, + status: failures.length ? 'criteria_not_met' as const : missing.length ? 'insufficient_evidence' as const : 'criteria_met' as const, + policyHash: policy ? configHash(policy) : null, failures, missing, + interpretation: 'reference_agreement_criteria_only_requires_full_package_qualification_and_review' as const } +} + +export function assessManagedAccuracy(...args: Parameters) { + const evidence = readManagedEvidence(...args) + return { trialId: evidence.trialId, reportHash: evidence.reportHash, currentEvidence: evidence.currentEvidence, + assessment: assessPackageAccuracy(evidence), preflight: evidence.preflight } +} diff --git a/scripts/algorithm-experiments/cli.ts b/scripts/algorithm-experiments/cli.ts new file mode 100644 index 000000000..bbd3bbe08 --- /dev/null +++ b/scripts/algorithm-experiments/cli.ts @@ -0,0 +1,87 @@ +import { recoverExperimentStaging } from './staging-recovery' +import { runPackagePreflight } from './preflight' +import { recoverExpiredContainers } from './recovery' +import { pathToFileURL } from 'node:url' +import { z } from 'zod' +import { runManagedFiles } from './managed-files' +import { experimentPolicySchema as policySchema } from './policy' +import { comparePackageOutputs } from './comparison' +import { runPackageFixtures } from './fixtures' +import { packageInputHash } from './input' +import { inspectPythonEnvironment } from './environment' +import { executePackageExperiment } from './container' +import { checkPackageDeterminism } from './determinism' +import { publishJson, readJsonFile, readJsonLines, readPackageProposal } from './files' + +const positive = z.number().int().positive().max(Number.MAX_SAFE_INTEGER) +const usage = 'Usage: preflight PROPOSAL PLAN SUITE POLICY OUTPUT | recover OUTPUT | managed WORKSPACE DESCRIPTOR RECEIPT [STORAGE_POLICY] | compare INPUT OUTPUT | fixtures PROPOSAL SUITE POLICY OUTPUT | lock IMAGE OUTPUT | input-hash FRAMES MAX_BYTES OUTPUT | inspect PROPOSAL OUTPUT | run PROPOSAL REQUEST FRAMES POLICY OUTPUT | determinism PROPOSAL REQUEST FRAMES POLICY REPETITIONS OUTPUT' + +export async function runExperimentCli(args: string[]) { + const [command, ...paths] = args + if (command === 'preflight' && paths.length === 5) { + const proposal = readPackageProposal(paths[0]) + const plan = readJsonFile(paths[1], 16384) + const suite = readJsonFile(paths[2], 32 * 1024 * 1024) + const policy = policySchema.parse(readJsonFile(paths[3], 16384)) + await publishJson(paths[4], await runPackagePreflight({ ...proposal, policy }, plan, suite)) + return + } + if (command === 'recover' && paths.length === 1) { + const containers = await recoverExpiredContainers() + const staging = await recoverExperimentStaging() + await publishJson(paths[0], { schema: 'biometric-experiment-recovery-v1', containers, staging }) + return + } + if (command === 'managed' && (paths.length === 3 || paths.length === 4)) { + await publishJson(paths[2], await runManagedFiles(paths[0], paths[1], paths[3] ? readJsonFile(paths[3], 16384) : undefined)) + return + } + if (command === 'compare' && paths.length === 2) { + await publishJson(paths[1], comparePackageOutputs(readJsonFile(paths[0], 32 * 1024 * 1024))) + return + } + if (command === 'fixtures' && paths.length === 4) { + const proposal = readPackageProposal(paths[0]) + const suite = readJsonFile(paths[1], 32 * 1024 * 1024) + const policy = policySchema.parse(readJsonFile(paths[2], 16384)) + const report = await runPackageFixtures({ manifest: proposal.manifest, files: proposal.files, policy }, suite) + await publishJson(paths[3], report) + return + } + if (command === 'lock' && paths.length === 2) { + await publishJson(paths[1], await inspectPythonEnvironment(paths[0])) + return + } + if (command === 'input-hash' && paths.length === 3) { + const maximumBytes = positive.parse(Number(paths[1])) + const hash = packageInputHash(readJsonLines(paths[0], maximumBytes)) + await publishJson(paths[2], { schema: 'biometric-package-input-v1', hash }) + return + } + if (command === 'inspect' && paths.length === 2) { + const proposal = readPackageProposal(paths[0]) + await publishJson(paths[1], { schema: 'biometric-package-intake-v1', packageHash: proposal.packageHash, + status: proposal.status, manifest: proposal.manifest, totalBytes: proposal.totalBytes }) + return + } + if ((command === 'run' && paths.length === 5) || (command === 'determinism' && paths.length === 6)) { + const policy = policySchema.parse(readJsonFile(paths[3], 16384)) + const proposal = readPackageProposal(paths[0]) + const request = readJsonFile(paths[1], 65536) + const frames = () => readJsonLines(paths[2], policy.maximumInputBytes) + const options = { manifest: proposal.manifest, files: proposal.files, request, policy } + const report = command === 'run' + ? { schema: 'biometric-package-experiment-v1', status: 'experiment_only', ...await executePackageExperiment({ ...options, frames: frames() }) } + : await checkPackageDeterminism({ ...options, frames }, Number(paths[4])) + await publishJson(paths[paths.length - 1], report) + return + } + throw new Error(usage) +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + runExperimentCli(process.argv.slice(2)).catch((error: unknown) => { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) + process.exitCode = 1 + }) +} diff --git a/scripts/algorithm-experiments/comparison.test.ts b/scripts/algorithm-experiments/comparison.test.ts new file mode 100644 index 000000000..a645e6f13 --- /dev/null +++ b/scripts/algorithm-experiments/comparison.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from 'vitest' +import { configHash } from '../../src/biometrics/algorithms/canonical' +import { packageFixture } from '../../src/biometrics/packages/tests/fixture' +import { comparePackageOutputs } from './comparison' +const start = 1791673200000 +function fixture() { + const outputs = (['default', 'current', 'candidate'] as const).map((label, index) => { + const { manifest } = packageFixture() + const declared = { ...manifest, metrics: manifest.metrics.map(metric => ({ ...metric, reference: { metric: 'heart_rate', windowMs: null as number | null } })) } + const packageHash = configHash(declared) + const request = { schema: 'biometric-package-run-v1', packageHash, inputHash: 'a'.repeat(64), hardwareId: 'pod4', sampleRateHz: 500, side: 'left', seed: 1, config: { offset: index } } + const runHash = configHash(request) + return { label, manifest: declared, request, result: { packageHash, runHash, + frozenInput: { schema: 'biometric-package-input-v1', hash: request.inputHash, bytes: 1000, frameCount: 1 }, + results: [{ schema: 'biometric-metric-frame-v1', runHash, frameHash: 'b'.repeat(64), sequence: 0, state: {}, + readings: [0, 60000].map(offset => ({ timestampMs: start + offset, side: 'left', metric: 'heart_rate', value: 64 - index * 2 as number | null, quality: 1, flags: [] })) }] } } + }) + return { schema: 'biometric-package-comparison-v1', metricId: 'heart_rate', startMs: start, endMs: start + 120000, minimumQuality: 0.8, + referenceImport: { side: 'left', subjectId: 'fixture', source: 'synthetic', rows: [0, 60000].map((offset, index) => ({ recordKey: String(index), metric: 'heart_rate', value: 60, unit: 'bpm', timestampMs: start + offset })) }, + alignment: { version: 'reference-alignment-v1', pointToleranceMs: 0, clockOffsetMs: 0, offsetSource: 'fixed' }, outputs } +} +function rebind(output: ReturnType['outputs'][number]) { + output.request.packageHash = configHash(output.manifest) + output.result.packageHash = output.request.packageHash + output.result.runHash = configHash(output.request) + output.result.results.forEach((frame) => { + frame.runHash = output.result.runHash + }) +} +describe('package reference agreement', () => { + it('compares both baselines with shared pairing without granting promotion', () => { + const result = comparePackageOutputs(fixture()) + expect(result.comparisons.filter(row => row.candidate === 'candidate').map(row => row.differences.commonMae)).toEqual([-4, -2]) + expect(result.interpretation).toContain('not_independent_holdout_or_promotion') + }) + it('exposes dropped and low-quality values as lost support', () => { + for (const lowQuality of [false, true]) { + const input = fixture() + const row = input.outputs[2].result.results[0].readings[0] + if (lowQuality) row.quality = 0.1 + else row.value = null + const comparison = comparePackageOutputs(input).comparisons.find(row => row.candidate === 'candidate' && row.baseline === 'default') + expect(comparison).toMatchObject({ coverageReduced: true, lostObservationTimestamps: 1, commonObservationCount: 1 }) + } + }) + it('rejects substituted run, frame, input, side and duplicate baseline identities', () => { + for (const change of [ + (input: ReturnType) => { input.outputs[2].result.runHash = 'c'.repeat(64) }, + (input: ReturnType) => { input.outputs[2].result.results[0].frameHash = 'c'.repeat(64) }, + (input: ReturnType) => { input.outputs[2].request.side = 'right' }, + (input: ReturnType) => { input.outputs[2].request.inputHash = 'c'.repeat(64) }, + (input: ReturnType) => { input.outputs[2].label = 'default' }, + ]) { + const input = fixture() + change(input) + expect(() => comparePackageOutputs(input)).toThrow() + } + }) + it('rejects incompatible package definitions before scoring', () => { + const input = fixture() + input.outputs[2].manifest.metrics[0].reference = { metric: 'hrv_rmssd', windowMs: 300000 } + input.outputs[2].manifest.metrics[0].unit = 'ms' + rebind(input.outputs[2]) + expect(() => comparePackageOutputs(input)).toThrow('Incompatible package metric') + }) + it('does not score external RMSSD against the window IBI variability index', () => { + const input = fixture() + for (const output of input.outputs) { + output.manifest.metrics[0].reference = { metric: 'window_ibi_variability_v1', windowMs: 300000 } + output.manifest.metrics[0].unit = 'ms' + rebind(output) + } + input.referenceImport.rows = input.referenceImport.rows.map(row => ({ ...row, metric: 'hrv_rmssd', unit: 'ms', windowMs: 300000 })) + const report = comparePackageOutputs(input) + expect(report.compatibilityExclusions).toEqual({ incompatible_metric_definition: 2 }) + expect(report.comparisons.every(row => row.commonSupport.candidate.status === 'incompatible' && row.commonSupport.candidate.metrics === null)).toBe(true) + }) +}) diff --git a/scripts/algorithm-experiments/comparison.ts b/scripts/algorithm-experiments/comparison.ts new file mode 100644 index 000000000..f1c4a5cf4 --- /dev/null +++ b/scripts/algorithm-experiments/comparison.ts @@ -0,0 +1,76 @@ +import { z } from 'zod' +import { configHash, dataHash } from '../../src/biometrics/algorithms/canonical' +import { compareNumericConfigurations } from '../../src/biometrics/backtest/numeric-comparison' +import { algorithmPackageManifestSchema } from '../../src/biometrics/packages/manifest' +import { metricFrameSchema, packageRunRequestSchema } from '../../src/biometrics/packages/protocol' +import { referenceCompatibility } from '../../src/biometrics/references/alignment' +import { normalizeReferenceImport } from '../../src/biometrics/references/normalize-core' +import { referenceImportSchema, timestampMsSchema } from '../../src/biometrics/references/contracts' + +const hash = z.string().regex(/^[a-f0-9]{64}$/) +export const packageComparisonInputSchema = z.object({ + schema: z.literal('biometric-package-comparison-v1'), metricId: z.string().min(1).max(100), + startMs: timestampMsSchema, endMs: timestampMsSchema, minimumQuality: z.number().min(0).max(1), + referenceImport: referenceImportSchema, + alignment: z.object({ version: z.literal('reference-alignment-v1'), pointToleranceMs: z.number().int().min(0).max(300000), + clockOffsetMs: z.number().int().min(-3600000).max(3600000), offsetSource: z.enum(['fixed', 'training']) }).strict(), + outputs: z.array(z.object({ label: z.enum(['default', 'current', 'candidate']), manifest: algorithmPackageManifestSchema, + request: packageRunRequestSchema, + result: z.object({ packageHash: hash, runHash: hash, + frozenInput: z.object({ schema: z.literal('biometric-package-input-v1'), hash, bytes: z.number().int().positive(), frameCount: z.number().int().positive() }).strict(), + results: z.array(metricFrameSchema).max(100000), + }), + }).strict()).length(3), +}).strict().refine(value => value.endMs > value.startMs && value.endMs - value.startMs <= 86400000, 'Comparison requires a bounded whole-night interval') + +/** Descriptive scoring of host-produced experiment reports. This does not + * attest execution provenance or independent holdout use; promotion must use + * the authoritative experiment store and reviewed qualification pipeline. */ +export function comparePackageOutputs(raw: unknown) { + const input = packageComparisonInputSchema.parse(raw) + if (new Set(input.outputs.map(output => output.label)).size !== 3) throw new Error('Default, current and candidate outputs are required') + const reference = normalizeReferenceImport(input.referenceImport) + let commonInput: string | undefined + let frameIdentity: string | undefined + let contract: { metric: 'heart_rate' | 'breathing_rate' | 'window_ibi_variability_v1' | 'hrv_rmssd' | 'hrv_sdnn', windowMs: number | null } | undefined + const outputs = input.outputs.map((output) => { + const { manifest, request, result } = output + const packageHash = configHash(manifest) + if (packageHash !== request.packageHash || packageHash !== result.packageHash || configHash(request) !== result.runHash + || result.frozenInput.hash !== request.inputHash || result.frozenInput.frameCount !== result.results.length) throw new Error('Package comparison identity mismatch') + if (request.side !== input.referenceImport.side) throw new Error('Reference side mismatch') + const identity = configHash({ inputHash: request.inputHash, side: request.side, hardwareId: request.hardwareId, sampleRateHz: request.sampleRateHz }) + if (commonInput !== undefined && commonInput !== identity) throw new Error('Packages did not process the same input and side') + commonInput = identity + const frameHash = dataHash(result.results.map(frame => frame.frameHash)) + if (frameIdentity !== undefined && frameIdentity !== frameHash) throw new Error('Packages received different frames') + frameIdentity = frameHash + const metric = manifest.metrics.find(metric => metric.id === input.metricId) + if (!metric?.reference) throw new Error('Package lacks a comparable metric contract') + if (contract && configHash(contract) !== configHash(metric.reference)) throw new Error('Incompatible package metric definitions or windows') + contract = metric.reference + const observations = [] + for (const [index, frame] of result.results.entries()) { + if (frame.runHash !== result.runHash || frame.sequence !== index) throw new Error('Package output identity mismatch') + for (const reading of frame.readings) { + if (reading.side !== request.side) throw new Error('Package output side mismatch') + if (reading.metric !== input.metricId) continue + if (reading.value !== null && (reading.value < metric.minimum || reading.value > metric.maximum)) throw new Error('Package metric outside declared range') + if (reading.timestampMs >= input.startMs && reading.timestampMs < input.endMs) observations.push({ timestampMs: reading.timestampMs, value: reading.quality >= input.minimumQuality ? reading.value : null }) + } + } + return { label: output.label, configHash: result.runHash, observations } + }) + if (!contract) throw new Error('No comparable metric contract') + const selected = reference.measurements.filter(row => row.startMs >= input.startMs && row.startMs < input.endMs && row.endMs <= input.endMs) + const compatibilityExclusions: Record = {} + for (const row of selected) { + const reason = referenceCompatibility(contract.metric, contract.windowMs, row) + if (reason) compatibilityExclusions[reason] = (compatibilityExclusions[reason] ?? 0) + 1 + } + const comparisons = compareNumericConfigurations(contract.metric, contract.windowMs, outputs, selected, input.alignment) + return { schema: 'biometric-package-comparison-report-v1' as const, inputHash: dataHash(input), referenceHash: dataHash(reference.measurements), + metricId: input.metricId, contract, alignment: input.alignment, startMs: input.startMs, endMs: input.endMs, + minimumQuality: input.minimumQuality, excludedBoundaryReferences: reference.measurements.length - selected.length, compatibilityExclusions, + referenceSummary: reference.summary, comparisons, interpretation: 'descriptive_reference_agreement_not_independent_holdout_or_promotion_evidence' as const } +} diff --git a/scripts/algorithm-experiments/container.test.ts b/scripts/algorithm-experiments/container.test.ts new file mode 100644 index 000000000..d39b5cdba --- /dev/null +++ b/scripts/algorithm-experiments/container.test.ts @@ -0,0 +1,213 @@ +import { fixtureResourceTestPolicy } from './resource-test-policy' +// @vitest-environment node +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { dirname, join } from 'node:path' +import { statSync } from 'node:fs' +import * as frozenInput from './input' +import * as resourceReader from './resources' +import { tmpdir } from 'node:os' +import { runPackageFixtures } from './fixtures' +import { runExperimentCli } from './cli' +import { createHash } from 'node:crypto' +import { beforeAll, describe, expect, it, vi } from 'vitest' +import { configHash } from '../../src/biometrics/algorithms/canonical' +import { packageFixture } from '../../src/biometrics/packages/tests/fixture' +import { inspectPythonEnvironment } from './environment' +import { checkPackageDeterminism } from './determinism' +import { packageInputHash } from './input' +import { executePackageExperiment } from './container' + +const image = process.env.BIOMETRIC_EXPERIMENT_TEST_IMAGE +let lockBytes: Buffer +beforeAll(async () => { + if (image) lockBytes = Buffer.from(JSON.stringify(await inspectPythonEnvironment(image))) +}) +const program = `import json, sys +init = json.loads(sys.stdin.readline()) +for line in sys.stdin: + message = json.loads(line) + frame = message['frame'] + print(json.dumps({'schema': 'biometric-metric-frame-v1', 'runHash': init['runHash'], 'frameHash': message['frameHash'], 'sequence': frame['sequence'], 'readings': [], 'state': {}}), flush=True) +` +function options(code: string) { + const { manifest, files } = packageFixture() + files.set('engine.py', Buffer.from(code)) + files.set(manifest.dependencyLock, lockBytes) + manifest.artifacts = [...files].map(([path, bytes]) => ({ path, bytes: bytes.length, sha256: createHash('sha256').update(bytes).digest('hex') })) + manifest.resources = { cpuMs: 5000, wallMs: 5000, memoryBytes: 64 * 1024 * 1024, processes: 1, outputBytes: 16384 } + const frames = [{ schema: 'biometric-signal-frame-v1', sequence: 0, timestampMs: 1000, samples: { left: [1, 2, 3], right: null }, context: { occupied: null, pumpActive: null } }] + frames.push({ ...frames[0], sequence: 1, timestampMs: 1006 }) + return { manifest, files, + request: { schema: 'biometric-package-run-v1', packageHash: configHash(manifest), inputHash: packageInputHash(frames), hardwareId: 'pod4', sampleRateHz: 500, side: 'left', seed: 123, config: {} }, + frames, + policy: { image: image ?? '', maximumWallMs: 10000, maximumMemoryBytes: 128 * 1024 * 1024, maximumProcesses: 4, maximumOutputBytes: 32768, maximumInputBytes: 32768 } } +} + +describe.skipIf(!image)('real external Docker experiments', () => { + it('runs the frame protocol with read-only input, no network, no extra processes or host environment', async () => { + const probes = `import os, socket +assert os.getuid() == 65534 +assert 'BIOMETRIC_EXPERIMENT_TEST_IMAGE' not in os.environ +assert not os.path.exists('/var/run/docker.sock') +assert not os.path.exists('/root/.ssh') +for path in ['/package/engine.py', '/tmp/write-attempt']: + try: + open(path, 'w') + except OSError: + pass + else: + raise AssertionError('Writable filesystem') +try: + os.fork() +except OSError: + pass +else: + raise AssertionError('Fork succeeded') +s = socket.socket() +s.settimeout(0.2) +try: + s.connect(('1.1.1.1', 443)) +except OSError: + pass +else: + raise AssertionError('Network available') +s.close() +` + const input = options(probes + program) + const execution = executePackageExperiment(input) + input.policy.image = 'sha256:' + 'b'.repeat(64) + const result = await execution + expect(result.execution.environment.image).toBe(image) + expect(result.execution.measurements.samples).toBeGreaterThan(0) + expect(result.execution.measurements.maximumObservedCpuMs).toBeGreaterThan(0) + expect(result.execution.measurements.peakObservedMemoryBytes).toBeGreaterThan(0) + expect(result.execution.measurements).toMatchObject({ source: 'host_docker_engine_stats', exactPeakMemory: false, exactLifetimeCpu: false }) + expect(result.results).toHaveLength(2) + expect(result.execution.isolation).toBe('docker-restricted-v1') + }, 20000) + it('keeps the host staging parent private while the unprivileged container reads its package mount', async () => { + let parent: string | undefined + let modes: number[] | undefined + const freeze = frozenInput.freezePackageInput + const readResources = resourceReader.dockerResourceReader + const capture = vi.spyOn(frozenInput, 'freezePackageInput').mockImplementation(async (path, frames, maximumBytes) => { + parent = dirname(path) + return freeze(path, frames, maximumBytes) + }) + const observe = vi.spyOn(resourceReader, 'dockerResourceReader').mockImplementation(async (id) => { + if (!parent) throw new Error('Missing staged input') + // This runs after Docker create, when all host permissions are final. + modes = [parent, join(parent, 'input.jsonl'), join(parent, 'package')].map(path => statSync(path).mode & 0o777) + return readResources(id) + }) + try { + expect((await executePackageExperiment(options(program))).results).toHaveLength(2) + expect(observe).toHaveBeenCalledOnce() + expect(modes).toEqual([0o700, 0o600, 0o755]) + const stagedParent = parent + if (!stagedParent) throw new Error('Missing staging') + expect(() => statSync(stagedParent)).toThrow() + } + finally { + capture.mockRestore() + observe.mockRestore() + } + }, 20000) + it('runs a complete file-based CLI experiment and publishes a reviewable report', async () => { + const root = await mkdtemp(join(tmpdir(), 'package-cli-test-')) + try { + const input = options(program) + const proposal = { schema: 'biometric-package-proposal-v1', manifest: input.manifest, + files: [...input.files].map(([path, bytes]) => ({ path, base64: bytes.toString('base64') })) } + for (const [name, value] of [['proposal', proposal], ['request', input.request], ['policy', input.policy]] as const) { + await writeFile(join(root, name), JSON.stringify(value)) + } + await writeFile(join(root, 'frames'), input.frames.map(frame => JSON.stringify(frame)).join('\n') + '\n') + await runExperimentCli(['run', ...['proposal', 'request', 'frames', 'policy', 'report'].map(name => join(root, name))]) + const report = JSON.parse(await readFile(join(root, 'report'), 'utf8')) + expect(report).toMatchObject({ schema: 'biometric-package-experiment-v1', status: 'experiment_only', packageHash: input.request.packageHash, + frozenInput: { hash: input.request.inputHash, frameCount: 2 } }) + expect(report.results).toHaveLength(2) + } + finally { await rm(root, { recursive: true, force: true }) } + }, 20000) + it('checks numeric coverage and catches false vitals during an empty-bed fixture', async () => { + const reading = '[{\'timestampMs\': frame[\'timestampMs\'], \'side\': \'left\', \'metric\': \'heart_rate\', \'value\': 60, \'quality\': 1, \'flags\': []}]' + const makeSuite = (input: ReturnType) => ({ schema: 'biometric-package-fixtures-v1', resourcePolicy: fixtureResourceTestPolicy, fixtures: [true, false].map((occupied) => { + const frames = input.frames.map(frame => ({ ...frame, context: { ...frame.context, occupied } })) + return { id: occupied ? 'clean' : 'empty', category: occupied ? 'clean_signal' : 'empty_bed', + request: { ...input.request, inputHash: packageInputHash(frames) }, frames, + expectations: [{ metric: 'heart_rate', startMs: 1000, endMs: 1012, ...(occupied + ? { kind: 'numeric', minimum: 59, maximum: 61, minimumReadings: 2, minimumQuality: 0.8 } + : { kind: 'abstain' }) }] } + }) }) + const candidate = options(program.replace('\'readings\': []', `'readings': ${reading} if frame['context']['occupied'] else []`)) + const suite = makeSuite(candidate) + const running = runPackageFixtures(candidate, suite) + suite.fixtures[0].frames[0].samples.left[0] = 999 + const report = await running + expect(report.suite.fixtures[0].frames[0].samples.left?.[0]).toBe(1) + expect(report.status).toBe('incomplete') + expect(report.results.every(result => result.assessment.passed)).toBe(true) + expect(report.missingCategories).toContain('pump') + const strictBudget = makeSuite(candidate) + strictBudget.resourcePolicy = { ...fixtureResourceTestPolicy, maximumWallMs: 1 } + const overBudget = await runPackageFixtures(candidate, strictBudget) + expect(overBudget.results.every(result => result.assessment.passed)).toBe(true) + expect(overBudget.results.every(result => result.resourceAssessment?.status === 'failed')).toBe(true) + expect(overBudget.status).toBe('failed') + const bad = options(program.replace('\'readings\': []', `'readings': ${reading}`)) + const regression = await runPackageFixtures(bad, makeSuite(bad)) + expect(regression.status).toBe('failed') + expect(regression.results[1].assessment.failures).toContain('failed_abstention:heart_rate:1000') + }, 30000) + it('repeats in fresh containers and detects changing state even with identical readings', async () => { + const input = options(program) + const stable = await checkPackageDeterminism({ ...input, frames: () => input.frames }) + expect(stable.deterministic).toBe(true) + expect(stable.runs).toHaveLength(2) + const unstable = options(program.replace('import json, sys', 'import json, sys, secrets').replace('\'state\': {}', '\'state\': {\'nonce\': secrets.token_hex(16)}')) + expect((await checkPackageDeterminism({ ...unstable, frames: () => unstable.frames })).deterministic).toBe(false) + }, 20000) + it('rejects a locked Python version that differs from the actual image before running code', async () => { + const input = options('raise AssertionError("candidate must not run")') + const wrong = JSON.parse(lockBytes.toString()) + wrong.pythonVersion = '0.0.0' + const bytes = Buffer.from(JSON.stringify(wrong)) + input.files.set(input.manifest.dependencyLock, bytes) + input.manifest.artifacts = input.manifest.artifacts.map(artifact => artifact.path === input.manifest.dependencyLock + ? { ...artifact, bytes: bytes.length, sha256: createHash('sha256').update(bytes).digest('hex') } + : artifact) + input.request.packageHash = configHash(input.manifest) + await expect(executePackageExperiment(input)).rejects.toThrow('differs from dependency lock') + }, 20000) + it('terminates an output flood without buffering it', async () => { + await expect(executePackageExperiment(options('import sys\nsys.stdout.write(\'x\' * 1000000)\nsys.stdout.flush()\n'))).rejects.toThrow('Output transport limit') + }, 20000) + it('terminates a package that never responds', async () => { + const input = options('while True: pass\n') + input.manifest.resources.cpuMs = 1000 + input.request.packageHash = configHash(input.manifest) + await expect(executePackageExperiment(input)).rejects.toThrow('deadline') + }, 20000) + it('stops a cancelled experiment and removes its container', async () => { + const controller = new AbortController() + const timer = setTimeout(() => controller.abort(), 1000) + try { + await expect(executePackageExperiment({ ...options('import time\ntime.sleep(60)\n'), signal: controller.signal })).rejects.toThrow(/abort/i) + } + finally { clearTimeout(timer) } + }, 20000) + it('terminates memory exhaustion', async () => { + await expect(executePackageExperiment(options('data = bytearray(256 * 1024 * 1024)\n'))).rejects.toThrow() + }, 20000) + it('rejects invalid output identities and operator limit excess', async () => { + await expect(executePackageExperiment(options(program.replace('init[\'runHash\']', '\'b\' * 64')))).rejects.toThrow('identity') + const substituted = options(program) + substituted.frames[0].samples.left[0]++ + await expect(executePackageExperiment(substituted)).rejects.toThrow('input identity mismatch') + const input = options(program) + input.policy.maximumMemoryBytes = 1 + await expect(executePackageExperiment(input)).rejects.toThrow('operator resource policy') + }, 20000) +}) diff --git a/scripts/algorithm-experiments/container.ts b/scripts/algorithm-experiments/container.ts new file mode 100644 index 000000000..abc8206fd --- /dev/null +++ b/scripts/algorithm-experiments/container.ts @@ -0,0 +1,198 @@ +import { recordExperimentStaging } from './staging-recovery' +import { recoveryLabelArguments } from './recovery' +import { spawn, execFile } from 'node:child_process' +import { existsSync } from 'node:fs' +import { randomUUID } from 'node:crypto' +import { mkdtemp, mkdir, writeFile, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' +import { promisify } from 'node:util' +import { performance } from 'node:perf_hooks' +import { verifyAlgorithmPackage } from '../../src/biometrics/packages/manifest' +import { PackageFrameValidator } from '../../src/biometrics/packages/protocol' +import { dockerResourceReader, startResourceMonitor } from './resources' +import { inspectPythonEnvironment, matchPythonEnvironment, parsePythonEnvironmentLock } from './environment' +import { freezePackageInput, readFrozenPackageInput } from './input' +import { dataHash } from '../../src/biometrics/algorithms/canonical' + +const exec = promisify(execFile) +async function removeContainer(name: string) { + try { + await exec('docker', ['rm', '-f', name], { timeout: 10000, maxBuffer: 65536 }) + } + catch (error) { + if (!(error instanceof Error) || !error.message.includes('No such container:')) throw error + } +} +/** Operator-owned policy, never accepted from a candidate or pod settings. */ +export interface ExperimentPolicy { + image: string + maximumWallMs: number + maximumMemoryBytes: number + maximumProcesses: number + maximumOutputBytes: number + maximumInputBytes: number +} + +/** Off-pod CLI only. Do not import from the web application or run on a pod. + * The image must be a reviewed Python runtime with locked dependencies already + * installed; dependency installation and network access are not permitted here. */ +export async function executePackageExperiment(options: { + manifest: unknown + files: ReadonlyMap + request: unknown + frames: Iterable + policy: ExperimentPolicy + signal?: AbortSignal +}) { + options.signal?.throwIfAborted() + if (existsSync('/persistent/deviceinfo/dac.sock') || existsSync('/deviceinfo/dac.sock')) throw new Error('Algorithm experiments must run off-pod') + const policy = structuredClone(options.policy) + if (!/^sha256:[a-f0-9]{64}$/.test(policy.image)) throw new Error('A local immutable image ID is required') + for (const value of [policy.maximumWallMs, policy.maximumMemoryBytes, policy.maximumProcesses, policy.maximumOutputBytes, policy.maximumInputBytes]) { + if (!Number.isSafeInteger(value) || value <= 0) throw new Error('Invalid experiment policy') + } + verifyAlgorithmPackage(options.manifest, options.files) + const files = new Map([...options.files].map(([path, bytes]) => [path, Buffer.from(bytes)])) + const verified = verifyAlgorithmPackage(options.manifest, files) + const validator = new PackageFrameValidator(verified.manifest, options.request) + const resources = verified.manifest.resources + if (resources.wallMs > policy.maximumWallMs || resources.memoryBytes > policy.maximumMemoryBytes + || resources.processes > policy.maximumProcesses || resources.outputBytes > policy.maximumOutputBytes) throw new Error('Package exceeds operator resource policy') + if (resources.wallMs > 3600000) throw new Error('Experiment deadline exceeds one hour') + const info = JSON.parse((await exec('docker', ['info', '--format', '{{json .}}'], { timeout: 10000, maxBuffer: 1024 * 1024 })).stdout) + if (info.OSType !== 'linux' || !info.MemoryLimit || !info.PidsLimit || !info.CpuCfsQuota || !info.SecurityOptions?.some((item: string) => item.startsWith('name=seccomp'))) throw new Error('Docker lacks required isolation controls') + const lockBytes = files.get(verified.manifest.dependencyLock) + if (!lockBytes) throw new Error('Missing dependency lock') + parsePythonEnvironmentLock(lockBytes, policy.image) + const environment = matchPythonEnvironment(lockBytes, await inspectPythonEnvironment(policy.image), policy.image) + const root = await mkdtemp(join(tmpdir(), 'sleepypod-package-')) + const name = `sleepypod-experiment-${randomUUID()}` + let created = false + try { + await recordExperimentStaging(root, name, Math.min(resources.wallMs, resources.cpuMs)) + const inputPath = join(root, 'input.jsonl') + const frozenInput = await freezePackageInput(inputPath, options.frames, policy.maximumInputBytes) + if (frozenInput.hash !== validator.request.inputHash) throw new Error('Experiment input identity mismatch') + const packagePath = join(root, 'package') + await mkdir(packagePath, { mode: 0o755 }) + // Docker mounts only packagePath; its container UID needs access inside that + // mount, not traversal of the host parent. Keep the mkdtemp root at 0700 so + // host users cannot list package source or the private frozen sensor input. + for (const artifact of verified.manifest.artifacts) { + const path = join(packagePath, artifact.path) + await mkdir(dirname(path), { recursive: true, mode: 0o755 }) + // Copy to privately owned staging so caller mutation cannot alter the mount. + await writeFile(path, files.get(artifact.path) as Uint8Array, { flag: 'wx', mode: 0o444 }) + } + // Bound aggregate CPU conservatively: one CPU and a wall deadline no longer + // than the CPU budget. Idle time consumes that budget too. This is not a + // measured CPU receipt and cannot qualify a package's runtime performance. + const deadlineMs = Math.min(resources.wallMs, resources.cpuMs) + options.signal?.throwIfAborted() + created = true + const creation = await exec('docker', ['create', '--name', name, ...recoveryLabelArguments('experiment', deadlineMs), '--network', 'none', '--ipc', 'none', '--read-only', + '--cap-drop', 'ALL', '--security-opt', 'no-new-privileges=true', '--user', '65534:65534', + '--pids-limit', String(resources.processes), '--memory', String(resources.memoryBytes), + '--memory-swap', String(resources.memoryBytes), '--cpus', '1', '--ulimit', 'core=0', + '--log-driver', 'none', '--restart', 'no', '--no-healthcheck', '-i', + '--mount', `type=bind,source=${packagePath},target=/package,readonly`, '--workdir', '/package', + '--entrypoint', 'python', policy.image, '-I', '-B', `/package/${verified.manifest.entrypoint}`], + { timeout: 10000, maxBuffer: 65536 }) + const resourceReader = await dockerResourceReader(creation.stdout.trim()).catch(() => null) + const started = performance.now() + const resourceMonitor = startResourceMonitor(resourceReader, started) + const child = spawn('docker', ['start', '-ai', name], { stdio: ['pipe', 'pipe', 'pipe'] }) + let failure: Error | null = null + let pending = Buffer.alloc(0), outputBytes = 0, inputBytes = 0 + let waiting: { resolve: (line: string) => void, reject: (error: Error) => void } | null = null + let stopPromise: Promise | null = null + const fail = (error: Error) => { + if (failure) return + failure = error + waiting?.reject(error) + waiting = null + child.stdin.destroy() + stopPromise = exec('docker', ['kill', name], { timeout: 10000, maxBuffer: 65536 }).catch(() => undefined) + child.kill('SIGKILL') + } + const abort = () => fail(new Error('Experiment aborted')) + options.signal?.addEventListener('abort', abort, { once: true }) + if (options.signal?.aborted) abort() + child.stdin.on('error', error => fail(error)) + child.on('error', error => fail(error)) + child.stdout.on('data', (chunk: Buffer) => { + if (failure) return + outputBytes += chunk.length + if (outputBytes > resources.outputBytes) return fail(new Error('Output transport limit exceeded')) + pending = Buffer.concat([pending, chunk]) + let newline: number + while ((newline = pending.indexOf(10)) >= 0) { + if (!waiting) return fail(new Error('Unexpected package output')) + const line = pending.subarray(0, newline).toString('utf8') + pending = pending.subarray(newline + 1) + const receiver = waiting + waiting = null + receiver.resolve(line) + } + }) + child.stderr.on('data', (chunk: Buffer) => { + outputBytes += chunk.length + if (outputBytes > resources.outputBytes) fail(new Error('Output transport limit exceeded')) + }) + const completion = new Promise((resolve, reject) => { + child.once('close', (code) => { + if (waiting) fail(new Error('Package exited before responding')) + if (failure) reject(failure) + else if (code !== 0 || pending.length) reject(new Error(`Package exited with invalid output (exit ${code})`)) + else resolve() + }) + }) + // Observe immediately, even if a frame iterator is still pending. + void completion.catch(() => undefined) + const timer = setTimeout(() => fail(new Error('Package deadline exceeded')), deadlineMs) + const send = async (value: unknown) => { + if (failure) throw failure + const bytes = JSON.stringify(value) + '\n' + inputBytes += Buffer.byteLength(bytes) + if (inputBytes > policy.maximumInputBytes) throw new Error('Input transport limit exceeded') + await new Promise((resolve, reject) => child.stdin.write(bytes, error => error ? reject(error) : resolve())) + } + const results = [] + try { + await send({ type: 'initialize', request: validator.request, runHash: validator.runHash }) + for await (const input of readFrozenPackageInput(inputPath)) { + const { frame } = validator.validateFrame(input) + const response = new Promise((resolve, reject) => { + waiting = { resolve, reject } + }) + void response.catch(() => undefined) + await send({ type: 'frame', frame, frameHash: dataHash(frame) }) + const result = validator.validateStep(frame, JSON.parse(await response)).result + results.push(result) + } + await resourceMonitor.sample() + child.stdin.end() + await completion + const measurements = await resourceMonitor.stop() + return { packageHash: verified.packageHash, runHash: validator.runHash, frozenInput, results, + execution: { limits: resources, measurements, environment, image: policy.image, wallMs: performance.now() - started, outputBytes, inputBytes, isolation: 'docker-restricted-v1' as const } } + } + catch (error) { + fail(error instanceof Error ? error : new Error(String(error))) + await completion.catch(() => undefined) + throw error + } + finally { + options.signal?.removeEventListener('abort', abort) + clearTimeout(timer) + await resourceMonitor.stop() + if (stopPromise) await stopPromise + } + } + finally { + // Keep ownership metadata and files if Docker cleanup is unconfirmed. + if (created) await removeContainer(name) + await rm(root, { recursive: true, force: true }) + } +} diff --git a/scripts/algorithm-experiments/determinism.ts b/scripts/algorithm-experiments/determinism.ts new file mode 100644 index 000000000..85b6ff0c2 --- /dev/null +++ b/scripts/algorithm-experiments/determinism.ts @@ -0,0 +1,28 @@ +import { createHash } from 'node:crypto' +import { canonicalJson } from '../../src/biometrics/algorithms/canonical' +import { verifyAlgorithmPackage } from '../../src/biometrics/packages/manifest' +import { executePackageExperiment } from './container' + +type ExperimentOptions = Parameters[0] +/** Independent containers provide a fresh process and filesystem for every run. + * The trusted caller supplies a fresh iterator over the same retained inputs. */ +export async function checkPackageDeterminism(options: Omit & { + frames: () => Iterable +}, repetitions = 2) { + if (!Number.isInteger(repetitions) || repetitions < 2 || repetitions > 5) throw new Error('Determinism requires 2–5 independent runs') + verifyAlgorithmPackage(options.manifest, options.files) + const stable = { manifest: structuredClone(options.manifest), request: structuredClone(options.request), + policy: structuredClone(options.policy), signal: options.signal, files: new Map([...options.files].map(([path, bytes]) => [path, Buffer.from(bytes)])) } + type Result = Awaited> + const runs: (Pick & { input: Result['frozenInput'], outputHash: string, responseCount: number })[] = [] + for (let index = 0; index < repetitions; index++) { + const result = await executePackageExperiment({ ...stable, frames: options.frames() }) + const hash = createHash('sha256').update('biometric-package-output-v1\n') + for (const frame of result.results) hash.update(canonicalJson(frame) + '\n') + runs.push({ packageHash: result.packageHash, runHash: result.runHash, input: result.frozenInput, + outputHash: hash.digest('hex'), responseCount: result.results.length, execution: result.execution }) + } + const deterministic = runs.every(run => run.runHash === runs[0].runHash && run.outputHash === runs[0].outputHash) + return { schema: 'biometric-package-determinism-v1' as const, deterministic, runs, + qualification: 'requires_resource_noise_holdout_compatibility_checks_and_review' as const } +} diff --git a/scripts/algorithm-experiments/environment.test.ts b/scripts/algorithm-experiments/environment.test.ts new file mode 100644 index 000000000..c258e3464 --- /dev/null +++ b/scripts/algorithm-experiments/environment.test.ts @@ -0,0 +1,26 @@ +// @vitest-environment node +import { describe, expect, it } from 'vitest' +import { matchPythonEnvironment, parsePythonEnvironmentLock, pythonEnvironmentLockSchema } from './environment' +const image = 'sha256:' + 'a'.repeat(64) +const lock = { schema: 'biometric-python-environment-v1', image, implementation: 'cpython', pythonVersion: '3.12.13', system: 'Linux', machine: 'aarch64', distributions: [{ name: 'numpy', version: '2.0.0' }, { name: 'scipy', version: '1.0.0' }] } +const bytes = Buffer.from(JSON.stringify(lock)) +describe('locked Python environment', () => { + it('binds the image, interpreter, architecture and complete distribution inventory', () => { + expect(matchPythonEnvironment(bytes, lock, image)).toMatchObject({ image, schema: 'biometric-python-environment-verification-v1' }) + for (const actual of [{ ...lock, image: 'sha256:' + 'b'.repeat(64) }, { ...lock, machine: 'x86_64' }, { ...lock, pythonVersion: '3.13.0' }, + { ...lock, distributions: lock.distributions.slice(0, 1) }, { ...lock, distributions: [{ name: 'numpy', version: '2.1.0' }, lock.distributions[1]] }]) { + expect(() => matchPythonEnvironment(bytes, actual, image)).toThrow('differs') + } + }) + it('rejects ambiguous names, ordering, duplicates and extra lock fields', () => { + for (const invalid of [{ ...lock, distributions: [...lock.distributions].reverse() }, { ...lock, distributions: [lock.distributions[0], lock.distributions[0]] }, + { ...lock, distributions: [{ name: 'Num_Py', version: '2.0.0' }] }, { ...lock, command: 'pip install latest' }]) { + expect(pythonEnvironmentLockSchema.safeParse(invalid).success).toBe(false) + } + }) + it('rejects unpinned images, foreign image locks, invalid encoding and oversized locks', () => { + expect(() => parsePythonEnvironmentLock(bytes, 'python:latest')).toThrow('operator image') + expect(() => parsePythonEnvironmentLock(Buffer.alloc(256 * 1024 + 1), image)).toThrow('byte limit') + expect(() => parsePythonEnvironmentLock(Buffer.from([255]), image)).toThrow() + }) +}) diff --git a/scripts/algorithm-experiments/environment.ts b/scripts/algorithm-experiments/environment.ts new file mode 100644 index 000000000..b15de5795 --- /dev/null +++ b/scripts/algorithm-experiments/environment.ts @@ -0,0 +1,65 @@ +import { recoveryLabelArguments } from './recovery' +import { execFile } from 'node:child_process' +import { createHash, randomUUID } from 'node:crypto' +import { promisify } from 'node:util' +import { z } from 'zod' +import { configHash } from '../../src/biometrics/algorithms/canonical' + +const exec = promisify(execFile) +const imageId = z.string().regex(/^sha256:[a-f0-9]{64}$/) +export const pythonEnvironmentLockSchema = z.object({ + schema: z.literal('biometric-python-environment-v1'), image: imageId, + implementation: z.literal('cpython'), pythonVersion: z.string().regex(/^\d+\.\d+\.\d+$/), + system: z.literal('Linux'), machine: z.string().regex(/^[a-zA-Z0-9_+-]{1,40}$/), + distributions: z.array(z.object({ name: z.string().regex(/^[a-z0-9]+(?:-[a-z0-9]+)*$/), version: z.string().min(1).max(100) }).strict()).max(512), +}).strict().superRefine((value, context) => { + const names = value.distributions.map(item => item.name) + if (names.some((name, index) => index > 0 && name <= names[index - 1])) context.addIssue({ code: 'custom', message: 'Distributions must be unique and sorted by normalized name' }) +}) + +const probe = `import importlib.metadata, json, platform, re, sys +packages = sorted([{'name': re.sub(r'[-_.]+', '-', dist.metadata['Name']).lower(), 'version': dist.version} for dist in importlib.metadata.distributions()], key=lambda item: item['name']) +print(json.dumps({'implementation': sys.implementation.name, 'pythonVersion': platform.python_version(), 'system': platform.system(), 'machine': platform.machine(), 'distributions': packages})) +` + +/** Probe an operator-reviewed immutable image, with no candidate code or mounts. + * Captures installed versions and pins all runtime bytes through the image ID. + * This is not a dependency resolver or an approval of the image's contents. */ +export async function inspectPythonEnvironment(image: string) { + imageId.parse(image) + const name = `sleepypod-environment-${randomUUID()}` + try { + await exec('docker', ['create', '--name', name, ...recoveryLabelArguments('environment', 10000), '--network', 'none', '--ipc', 'none', '--read-only', + '--cap-drop', 'ALL', '--security-opt', 'no-new-privileges=true', '--user', '65534:65534', + '--pids-limit', '16', '--memory', '134217728', '--memory-swap', '134217728', '--cpus', '1', + '--ulimit', 'core=0', '--ulimit', 'cpu=5', '--log-driver', 'none', '--restart', 'no', '--no-healthcheck', + '--entrypoint', 'python', image, '-I', '-B', '-c', probe], { timeout: 10000, maxBuffer: 65536 }) + const { stdout } = await exec('docker', ['start', '-a', name], { timeout: 10000, maxBuffer: 256 * 1024 }) + return pythonEnvironmentLockSchema.parse({ ...JSON.parse(stdout), schema: 'biometric-python-environment-v1', image }) + } + finally { + await removeProbe(name) + } +} +async function removeProbe(name: string) { + try { + await exec('docker', ['rm', '-f', name], { timeout: 10000, maxBuffer: 65536 }) + } + catch (error) { + if (!(error instanceof Error) || !error.message.includes('No such container:')) throw error + } +} + +export function parsePythonEnvironmentLock(bytes: Uint8Array, image: string) { + if (bytes.byteLength > 256 * 1024) throw new Error('Environment lock byte limit exceeded') + const lock = pythonEnvironmentLockSchema.parse(JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes))) + if (lock.image !== image) throw new Error('Environment lock does not match operator image') + return lock +} +export function matchPythonEnvironment(bytes: Uint8Array, actual: unknown, image: string) { + const locked = parsePythonEnvironmentLock(bytes, image) + const observed = pythonEnvironmentLockSchema.parse(actual) + if (configHash(locked) !== configHash(observed)) throw new Error('Installed Python environment differs from dependency lock') + return { schema: 'biometric-python-environment-verification-v1' as const, image, + lockHash: createHash('sha256').update(bytes).digest('hex'), environmentHash: configHash(observed) } +} diff --git a/scripts/algorithm-experiments/files.test.ts b/scripts/algorithm-experiments/files.test.ts new file mode 100644 index 000000000..2ede06f8a --- /dev/null +++ b/scripts/algorithm-experiments/files.test.ts @@ -0,0 +1,62 @@ +// @vitest-environment node +import { mkdtemp, readFile, readdir, rm, stat, symlink, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { packageFixture } from '../../src/biometrics/packages/tests/fixture' +import { publishJson, readJsonFile, readJsonLines, readPackageProposal } from './files' +import { runExperimentCli } from './cli' +let root: string +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'package-files-test-')) +}) +afterEach(async () => { + await rm(root, { recursive: true, force: true }) +}) +function proposal() { + const { files, manifest } = packageFixture() + return { schema: 'biometric-package-proposal-v1', manifest, files: [...files].map(([path, bytes]) => ({ path, base64: bytes.toString('base64') })) } +} +describe('experiment CLI file boundary', () => { + it('inspects an inert proposal and creates a complete private report without running its code', async () => { + const path = join(root, 'proposal.json'), report = join(root, 'report.json') + await writeFile(path, JSON.stringify(proposal())) + await runExperimentCli(['inspect', path, report]) + expect(JSON.parse(await readFile(report, 'utf8'))).toMatchObject({ status: 'inert_proposal', manifest: { engine: { id: 'test.hr' } } }) + expect((await stat(report)).mode & 0o777).toBe(0o600) + await expect(publishJson(report, { replaced: true })).rejects.toThrow() + expect(await readdir(root)).toEqual(['proposal.json', 'report.json']) + }) + it('rejects changed bytes, duplicate files and noncanonical base64', async () => { + const original = proposal() + for (const changed of [{ ...original, files: [original.files[0], original.files[0]] }, + { ...original, files: [{ ...original.files[0], base64: 'cGFzcw==' }, original.files[1]] }, + { ...original, files: [{ ...original.files[0], base64: original.files[0].base64 + '\n' }, original.files[1]] }]) { + const path = join(root, 'proposal.json') + await writeFile(path, JSON.stringify(changed)) + expect(() => readPackageProposal(path)).toThrow() + } + }) + it('rejects symlinks, nonfiles and byte limits for JSON and JSONL', async () => { + const path = join(root, 'input'), link = join(root, 'symlink') + await writeFile(path, '{"value":1}\n{"value":2}\n') + await symlink(path, link) + expect([...readJsonLines(path, 100)]).toEqual([{ value: 1 }, { value: 2 }]) + for (const invalid of [root, link]) { + expect(() => readJsonFile(invalid, 100)).toThrow() + expect(() => [...readJsonLines(invalid, 100)]).toThrow() + } + expect(() => [...readJsonLines(path, 5)]).toThrow() + expect(() => [...readJsonLines(path, 100, 2)]).toThrow('line byte limit') + expect(() => readJsonFile(path, 5)).toThrow() + }) + it('rejects malformed UTF-8, blank lines and incorrect CLI arguments', async () => { + const path = join(root, 'input') + await writeFile(path, Buffer.from([255])) + expect(() => readJsonFile(path, 100)).toThrow() + expect(() => [...readJsonLines(path, 100)]).toThrow() + await writeFile(path, '\n') + expect(() => [...readJsonLines(path, 100)]).toThrow('Empty JSON') + await expect(runExperimentCli(['run'])).rejects.toThrow('Usage:') + }) +}) diff --git a/scripts/algorithm-experiments/files.ts b/scripts/algorithm-experiments/files.ts new file mode 100644 index 000000000..c743add13 --- /dev/null +++ b/scripts/algorithm-experiments/files.ts @@ -0,0 +1,99 @@ +import { constants, closeSync, fstatSync, openSync, readSync } from 'node:fs' +import { link, open, unlink } from 'node:fs/promises' +import { randomUUID } from 'node:crypto' +import { z } from 'zod' +import { algorithmPackageManifestSchema, maximumPackageBytes, verifyAlgorithmPackage } from '../../src/biometrics/packages/manifest' + +function openRegular(path: string, maximumBytes: number) { + if (!Number.isSafeInteger(maximumBytes) || maximumBytes <= 0) throw new Error('Invalid file byte limit') + const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK) + try { + const stat = fstatSync(fd) + if (!stat.isFile() || stat.size > maximumBytes) throw new Error('Expected a bounded regular file') + return fd + } + catch (error) { + closeSync(fd) + throw error + } +} + +/** Symlinks, devices and FIFOs are never followed. Size is checked both before + * and during reads so a growing input cannot bypass its transport limit. */ +export function* readJsonLines(path: string, maximumBytes: number, maximumLineBytes = 8 * 1024 * 1024) { + const fd = openRegular(path, maximumBytes) + const chunk = Buffer.alloc(65536) + let total = 0, pending = Buffer.alloc(0) + try { + let count: number + while ((count = readSync(fd, chunk)) !== 0) { + total += count + if (total > maximumBytes) throw new Error('File byte limit exceeded') + pending = Buffer.concat([pending, chunk.subarray(0, count)]) + let newline: number + while ((newline = pending.indexOf(10)) >= 0) { + if (newline > maximumLineBytes) throw new Error('JSON line byte limit exceeded') + const line = pending.subarray(0, newline) + pending = pending.subarray(newline + 1) + if (!line.length) throw new Error('Empty JSON line') + yield JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(line)) as unknown + } + if (pending.length > maximumLineBytes) throw new Error('JSON line byte limit exceeded') + } + if (pending.length) yield JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(pending)) as unknown + } + finally { closeSync(fd) } +} + +export function readJsonFile(path: string, maximumBytes: number): unknown { + const fd = openRegular(path, maximumBytes) + try { + const bytes = Buffer.alloc(maximumBytes + 1) + let offset = 0, count: number + while ((count = readSync(fd, bytes, offset, bytes.length - offset, null)) !== 0) { + offset += count + if (offset > maximumBytes) throw new Error('File byte limit exceeded') + } + return JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes.subarray(0, offset))) + } + finally { closeSync(fd) } +} + +const proposalSchema = z.object({ schema: z.literal('biometric-package-proposal-v1'), manifest: algorithmPackageManifestSchema, + files: z.array(z.object({ path: z.string().min(1).max(240), base64: z.string().max(Math.ceil(maximumPackageBytes / 3) * 4) }).strict()).max(128), +}).strict() + +/** A single inert JSON envelope avoids archive extraction and filesystem paths + * derived from candidate data. Every byte is verified against the manifest. */ +export function readPackageProposal(path: string) { + const proposal = proposalSchema.parse(readJsonFile(path, 48 * 1024 * 1024)) + if (new Set(proposal.files.map(file => file.path)).size !== proposal.files.length) throw new Error('Duplicate proposal file') + const files = new Map() + let total = 0 + for (const file of proposal.files) { + const bytes = Buffer.from(file.base64, 'base64') + if (bytes.toString('base64') !== file.base64) throw new Error('Expected canonical base64') + total += bytes.length + if (total > maximumPackageBytes) throw new Error('Proposal byte limit exceeded') + files.set(file.path, bytes) + } + const verified = verifyAlgorithmPackage(proposal.manifest, files) + return { ...verified, files } +} + +/** Publish only complete JSON, mode 0600, and never replace an existing report. + * Hard-link publication is atomic and refuses existing files or symlinks. */ +export async function publishJson(path: string, value: unknown) { + const staging = `${path}.${randomUUID()}.tmp` + const file = await open(staging, 'wx', 0o600) + try { + await file.writeFile(JSON.stringify(value, null, 2) + '\n') + await file.sync() + await file.close() + await link(staging, path) + } + finally { + await file.close() + await unlink(staging) + } +} diff --git a/scripts/algorithm-experiments/fixtures.test.ts b/scripts/algorithm-experiments/fixtures.test.ts new file mode 100644 index 000000000..b7f4e791c --- /dev/null +++ b/scripts/algorithm-experiments/fixtures.test.ts @@ -0,0 +1,39 @@ +import { fixtureResourceTestPolicy } from './resource-test-policy' +import { describe, expect, it } from 'vitest' +import { assessFixtureReadings, packageFixtureSuiteSchema } from './fixtures' +const numeric = { kind: 'numeric' as const, metric: 'heart_rate', startMs: 1000, endMs: 2000, minimum: 59, maximum: 61, minimumReadings: 2, minimumQuality: 0.8 } +const reading = { timestampMs: 1000, side: 'left' as const, metric: 'heart_rate', value: 60, quality: 1, flags: [] } +describe('artifact/noise fixture scoring', () => { + it('requires coverage and rejects dropping difficult readings', () => { + expect(assessFixtureReadings([numeric], [reading, { ...reading, timestampMs: 1500 }]).passed).toBe(true) + expect(assessFixtureReadings([numeric], [reading]).failures).toContain('insufficient_coverage:heart_rate:0') + expect(assessFixtureReadings([numeric], [{ ...reading, value: null }]).passed).toBe(false) + expect(assessFixtureReadings([numeric], [reading, reading]).passed).toBe(false) + }) + it('rejects out-of-range, low quality, unknown metric and out-of-window readings', () => { + for (const changed of [{ ...reading, value: 120 }, { ...reading, quality: 0.2 }, { ...reading, timestampMs: 2000 }, { ...reading, metric: 'breathing_rate' }]) { + expect(assessFixtureReadings([{ ...numeric, minimumReadings: 1 }], [changed]).passed).toBe(false) + } + }) + it('accepts explicit null or absent output for abstention, but rejects invented vitals', () => { + const expected = { kind: 'abstain' as const, metric: 'heart_rate', startMs: 1000, endMs: 2000 } + expect(assessFixtureReadings([expected], []).passed).toBe(true) + expect(assessFixtureReadings([expected], [{ ...reading, value: null }]).passed).toBe(true) + expect(assessFixtureReadings([expected], [reading]).failures).toContain('failed_abstention:heart_rate:1000') + }) + it('rejects overlapping, vacuous, out-of-input and duplicate fixture definitions', () => { + const fixture = { id: 'clean', category: 'clean_signal', + request: { schema: 'biometric-package-run-v1', packageHash: 'a'.repeat(64), inputHash: 'b'.repeat(64), hardwareId: 'pod4', sampleRateHz: 1, side: 'left', seed: 1, config: {} }, + frames: [{ schema: 'biometric-signal-frame-v1', sequence: 0, timestampMs: 1000, samples: { left: [1], right: null }, context: { occupied: true, pumpActive: false } }], expectations: [numeric] } + const suite = (fixtures: unknown[]) => ({ schema: 'biometric-package-fixtures-v1', resourcePolicy: fixtureResourceTestPolicy, fixtures }) + expect(packageFixtureSuiteSchema.safeParse(suite([fixture])).success).toBe(true) + for (const fixtures of [[fixture, fixture], [{ ...fixture, expectations: [numeric, numeric] }], + [{ ...fixture, expectations: [{ ...numeric, startMs: 0 }] }], [{ ...fixture, category: 'empty_bed' }], + [{ ...fixture, expectations: [{ kind: 'abstain', metric: 'heart_rate', startMs: 1000, endMs: 2000 }] }]]) { + expect(packageFixtureSuiteSchema.safeParse(suite(fixtures)).success).toBe(false) + } + }) + it('rejects empty suites', () => { + expect(packageFixtureSuiteSchema.safeParse({ schema: 'biometric-package-fixtures-v1', resourcePolicy: fixtureResourceTestPolicy, fixtures: [] }).success).toBe(false) + }) +}) diff --git a/scripts/algorithm-experiments/fixtures.ts b/scripts/algorithm-experiments/fixtures.ts new file mode 100644 index 000000000..ca8927765 --- /dev/null +++ b/scripts/algorithm-experiments/fixtures.ts @@ -0,0 +1,110 @@ +import { z } from 'zod' +import { dataHash } from '../../src/biometrics/algorithms/canonical' +import { packageRunRequestSchema, signalFrameSchema } from '../../src/biometrics/packages/protocol' +import { verifyAlgorithmPackage } from '../../src/biometrics/packages/manifest' +import { assessPackageResources, packageResourcePolicySchema } from './resource-assessment' +import { packageInputHash } from './input' +import { executePackageExperiment } from './container' + +const time = z.number().finite().nonnegative().max(Number.MAX_SAFE_INTEGER) +const common = { metric: z.string().min(1).max(100), startMs: time, endMs: time } +const expectationSchema = z.discriminatedUnion('kind', [ + z.object({ ...common, kind: z.literal('numeric'), minimum: z.number().finite(), maximum: z.number().finite(), + minimumReadings: z.number().int().positive().max(20000), minimumQuality: z.number().min(0).max(1) }).strict(), + z.object({ ...common, kind: z.literal('abstain') }).strict(), +]) +export const fixtureCategories = ['clean_signal', 'pump', 'empty_bed', 'motion', 'cross_side', 'harmonics', 'gaps', 'warmup', 'boundary_reset'] as const +const fixtureSchema = z.object({ + id: z.string().regex(/^[a-z0-9][a-z0-9_-]{0,99}$/), category: z.enum(fixtureCategories), + request: packageRunRequestSchema, frames: z.array(signalFrameSchema).min(1).max(256), + expectations: z.array(expectationSchema).min(1).max(256), +}).strict().superRefine((fixture, context) => { + if (!fixture.frames.length) return + const firstMs = fixture.frames[0].timestampMs + const last = fixture.frames[fixture.frames.length - 1] + const endMs = last.timestampMs + (last.samples[fixture.request.side]?.length ?? 0) * 1000 / fixture.request.sampleRateHz + if (fixture.category === 'clean_signal' && !fixture.expectations.some(expected => expected.kind === 'numeric')) context.addIssue({ code: 'custom', message: 'Clean-signal fixture requires numeric coverage' }) + if (fixture.category === 'empty_bed' && fixture.expectations.some(expected => expected.kind !== 'abstain')) context.addIssue({ code: 'custom', message: 'Empty-bed fixture must require abstention' }) + for (let index = 0; index < fixture.expectations.length; index++) { + const expected = fixture.expectations[index] + if (expected.startMs < firstMs || expected.endMs > endMs) context.addIssue({ code: 'custom', message: 'Expectation outside fixture input interval' }) + if (expected.startMs >= expected.endMs || (expected.kind === 'numeric' && expected.minimum > expected.maximum)) { + context.addIssue({ code: 'custom', message: 'Invalid fixture expectation range' }) + } + if (fixture.expectations.slice(0, index).some(other => other.metric === expected.metric && other.startMs < expected.endMs && expected.startMs < other.endMs)) { + context.addIssue({ code: 'custom', message: 'Overlapping fixture expectations' }) + } + } +}) +export const packageFixtureSuiteSchema = z.object({ schema: z.literal('biometric-package-fixtures-v1'), resourcePolicy: packageResourcePolicySchema, + fixtures: z.array(fixtureSchema).min(1).max(64), +}).strict().refine(suite => new Set(suite.fixtures.map(fixture => fixture.id)).size === suite.fixtures.length, 'Duplicate fixture identity') + +type Expectation = z.infer +type Reading = Awaited>['results'][number]['readings'][number] +/** Expectations come from reviewed fixtures, never from package responses. + * Every returned reading must belong to a declared expectation window. */ +export function assessFixtureReadings(expectations: Expectation[], readings: Reading[]) { + const failures: string[] = [] + const counts = expectations.map(() => 0) + const identities = new Set() + for (const reading of readings) { + const identity = `${reading.metric}:${reading.timestampMs}` + if (identities.has(identity)) failures.push(`duplicate_reading:${identity}`) + identities.add(identity) + const index = expectations.findIndex(expected => expected.metric === reading.metric && expected.startMs <= reading.timestampMs && reading.timestampMs < expected.endMs) + if (index < 0) { + failures.push(`unexpected_reading:${identity}`) + continue + } + const expected = expectations[index] + if (expected.kind === 'abstain') { + if (reading.value !== null) failures.push(`failed_abstention:${identity}`) + } + else if (reading.value !== null) { + if (reading.value < expected.minimum || reading.value > expected.maximum || reading.quality < expected.minimumQuality) failures.push(`out_of_range_or_low_quality:${identity}`) + else counts[index]++ + } + } + expectations.forEach((expected, index) => { + if (expected.kind === 'numeric' && counts[index] < expected.minimumReadings) failures.push(`insufficient_coverage:${expected.metric}:${index}`) + }) + return { passed: failures.length === 0, failures, usableReadings: counts } +} + +/** This is one qualification dimension, not a held-out improvement claim. + * Fresh containers also prevent state leaking between independent fixtures. */ +export async function runPackageFixtures(options: Pick[0], 'manifest' | 'files' | 'policy' | 'signal'>, suiteInput: unknown) { + const suite = packageFixtureSuiteSchema.parse(structuredClone(suiteInput)) + const suiteHash = dataHash(suite) + const verified = verifyAlgorithmPackage(options.manifest, options.files) + const stable = { manifest: verified.manifest, files: new Map([...options.files].map(([path, bytes]) => [path, Buffer.from(bytes)])), policy: structuredClone(options.policy), signal: options.signal } + for (const fixture of suite.fixtures) { + for (const expected of fixture.expectations) { + if (!verified.manifest.metrics.some(metric => metric.id === expected.metric)) throw new Error('Fixture expects an undeclared metric') + } + if (fixture.request.packageHash !== verified.packageHash || fixture.request.inputHash !== packageInputHash(fixture.frames)) throw new Error('Fixture input/package identity mismatch') + } + const results = [] + for (const fixture of suite.fixtures) { + options.signal?.throwIfAborted() + const identity = { fixtureId: fixture.id, category: fixture.category, fixtureHash: dataHash(fixture) } + try { + const execution = await executePackageExperiment({ ...stable, request: fixture.request, frames: fixture.frames }) + const assessment = assessFixtureReadings(fixture.expectations, execution.results.flatMap(frame => frame.readings)) + results.push({ ...identity, assessment, resourceAssessment: assessPackageResources(execution.execution, suite.resourcePolicy), execution, error: null }) + } + catch (error) { + options.signal?.throwIfAborted() + results.push({ ...identity, assessment: { passed: false, failures: ['execution_failed'], usableReadings: [] }, resourceAssessment: null, execution: null, + error: error instanceof Error ? error.message.slice(0, 1000) : 'Experiment failed' }) + } + } + const covered = new Set(suite.fixtures.map(fixture => fixture.category)) + const missingCategories = fixtureCategories.filter(category => !covered.has(category)) + return { schema: 'biometric-package-fixture-report-v1' as const, packageHash: verified.packageHash, suiteHash, suite, + status: results.some(result => !result.assessment.passed || result.resourceAssessment?.status === 'failed') + ? 'failed' + : missingCategories.length || results.some(result => result.resourceAssessment?.status !== 'within_observed_limits') ? 'incomplete' : 'fixture_checks_passed', + missingCategories, results, promotion: 'requires_determinism_resource_holdout_compatibility_and_release_review' as const } +} diff --git a/scripts/algorithm-experiments/input.test.ts b/scripts/algorithm-experiments/input.test.ts new file mode 100644 index 000000000..a8cfe2414 --- /dev/null +++ b/scripts/algorithm-experiments/input.test.ts @@ -0,0 +1,36 @@ +// @vitest-environment node +import { mkdtemp, rm } from 'node:fs/promises' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { describe, expect, it } from 'vitest' +import { freezePackageInput, packageInputHash, readFrozenPackageInput } from './input' + +const frame = { schema: 'biometric-signal-frame-v1', sequence: 0, timestampMs: 1000, + samples: { left: [1, 2, 3], right: null }, context: { occupied: null, pumpActive: null } } + +describe('frozen experiment input', () => { + it('binds sample bytes, context, ordering and timestamps to a streamed identity', () => { + const original = packageInputHash([frame]) + for (const changed of [{ ...frame, timestampMs: 1001 }, { ...frame, context: { occupied: true, pumpActive: null } }, + { ...frame, samples: { left: [1, 2, 4], right: null } }]) expect(packageInputHash([changed])).not.toBe(original) + expect(packageInputHash([frame, { ...frame, sequence: 1 }])).not.toBe(packageInputHash([{ ...frame, sequence: 1 }, frame])) + expect(packageInputHash([{ context: frame.context, samples: frame.samples, timestampMs: 1000, sequence: 0, schema: frame.schema }])).toBe(original) + }) + it('reads the frozen copy after source mutation and enforces storage bounds', async () => { + const root = await mkdtemp(join(tmpdir(), 'package-input-test-')) + try { + const source = structuredClone(frame) + const next = { ...frame, sequence: 1, timestampMs: 1006 } + const frozen = await freezePackageInput(join(root, 'input'), [source, next], 10000) + source.samples.left[0] = 999 + const collected = [] + for await (const value of readFrozenPackageInput(join(root, 'input'))) collected.push(value) + expect(collected).toEqual([frame, next]) + expect(frozen).toMatchObject({ hash: packageInputHash([frame, next]), frameCount: 2 }) + await expect(freezePackageInput(join(root, 'input'), [frame], 10000)).rejects.toThrow() + await expect(freezePackageInput(join(root, 'small'), [frame], 10)).rejects.toThrow('byte limit') + await expect(freezePackageInput(join(root, 'empty'), [], 10000)).rejects.toThrow('at least one') + } + finally { await rm(root, { recursive: true, force: true }) } + }) +}) diff --git a/scripts/algorithm-experiments/input.ts b/scripts/algorithm-experiments/input.ts new file mode 100644 index 000000000..16acdb80c --- /dev/null +++ b/scripts/algorithm-experiments/input.ts @@ -0,0 +1,51 @@ +import { createHash } from 'node:crypto' +import { open } from 'node:fs/promises' +import { createReadStream } from 'node:fs' +import { createInterface } from 'node:readline' +import { canonicalJson } from '../../src/biometrics/algorithms/canonical' +import { signalFrameSchema } from '../../src/biometrics/packages/protocol' + +const domain = 'biometric-package-input-v1\n' +/** Streaming identity; canonical frame JSON followed by LF, prefixed by domain. + * These are actual experiment frames, not a user-supplied dataset name/hash. */ +export function packageInputHash(frames: Iterable) { + const hash = createHash('sha256').update(domain) + for (const frame of frames) hash.update(canonicalJson(signalFrameSchema.parse(frame)) + '\n') + return hash.digest('hex') +} + +/** Freeze bounded input before any candidate starts. The owned file is never + * mounted into the container: the executor sends one frame at a time. */ +export async function freezePackageInput(path: string, frames: Iterable, maximumBytes: number) { + if (!Number.isSafeInteger(maximumBytes) || maximumBytes <= 0) throw new Error('Invalid input byte limit') + const file = await open(path, 'wx', 0o600) + let bytes = 0, frameCount = 0 + const hash = createHash('sha256').update(domain) + try { + for (const frame of frames) { + const encoded = canonicalJson(signalFrameSchema.parse(frame)) + '\n' + bytes += Buffer.byteLength(encoded) + if (bytes > maximumBytes) throw new Error('Frozen input byte limit exceeded') + hash.update(encoded) + await file.writeFile(encoded) + frameCount++ + } + if (!frameCount) throw new Error('Experiment requires at least one frame') + } + finally { + await file.close() + } + return { schema: 'biometric-package-input-v1' as const, hash: hash.digest('hex'), bytes, frameCount } +} + +export async function* readFrozenPackageInput(path: string) { + const stream = createReadStream(path) + const lines = createInterface({ input: stream, crlfDelay: Infinity }) + try { + for await (const line of lines) yield JSON.parse(line) as unknown + } + finally { + lines.close() + stream.destroy() + } +} diff --git a/scripts/algorithm-experiments/ledger.test.ts b/scripts/algorithm-experiments/ledger.test.ts new file mode 100644 index 000000000..176cece30 --- /dev/null +++ b/scripts/algorithm-experiments/ledger.test.ts @@ -0,0 +1,142 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { mkdtempSync, rmSync } from 'node:fs' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { PackageExperimentLedger, packageStudySchema } from './ledger' +const start = 1791673200000 +function study() { + return { schema: 'biometric-package-study-v1', subjectId: 'fixture', side: 'left', protocolHash: 'a'.repeat(64), nights: [ + { id: 'night', startMs: start, endMs: start + 28800000, inputHash: 'b'.repeat(64), referenceHash: 'c'.repeat(64), split: 'holdout' }, + ] } +} +function trial(studyId: string, split = 'holdout') { + return { studyId, split, variants: ['default', 'current', 'candidate'].map(label => ({ label, packageHash: 'd'.repeat(64), configHash: 'e'.repeat(64), image: 'sha256:' + 'f'.repeat(64) })) } +} +let db: Database.Database, ledger: PackageExperimentLedger, now: number +beforeEach(() => { + db = new Database(':memory:') + now = 1000 + ledger = new PackageExperimentLedger(db, () => now) +}) +afterEach(() => db.close()) +describe('dedicated package experiment ledger', () => { + it('freezes studies and reserves exactly one trial per unchanged request key', () => { + const first = ledger.createStudy(study()) + expect(ledger.createStudy(study()).id).toBe(first.id) + const id = ledger.reserveTrial(trial(first.id), 'request') + expect(ledger.reserveTrial(trial(first.id), 'request')).toBe(id) + const changed = trial(first.id) + changed.variants[2].configHash = '0'.repeat(64) + expect(() => ledger.reserveTrial(changed, 'request')).toThrow('idempotency') + expect(ledger.currentEvidence(id).uniqueHoldoutAttempt).toBe(false) + }) + it('records exposure before execution and fences expired workers and their renewals', () => { + const id = ledger.reserveTrial(trial(ledger.createStudy(study()).id), 'request') + const first = ledger.claim(id, 1000) + expect(first.unexposedHoldoutAtClaim).toBe(true) + expect(() => ledger.claim(id, 1000)).toThrow('leased') + now = 2000 + const retry = ledger.claim(id, 1000) + expect(retry).toMatchObject({ unexposedHoldoutAtClaim: false, priorExposures: [{ nightId: 'night', count: 1 }] }) + expect(() => ledger.finish(first, 'succeeded', '1'.repeat(64))).toThrow('Stale') + expect(() => ledger.renew(first, 1000)).toThrow('Stale') + ledger.renew(retry, 2000) + now = 3500 + ledger.finish(retry, 'succeeded', '2'.repeat(64)) + expect(ledger.currentEvidence(id)).toMatchObject({ reportHash: '2'.repeat(64), uniqueHoldoutAttempt: false }) + expect(() => ledger.claim(id, 1000)).toThrow('completed') + }) + it('detects overlap across changed studies, hashes and splits, including failed attempts', () => { + const tuning = study() + tuning.nights[0].split = 'tuning' + const tuningId = ledger.reserveTrial(trial(ledger.createStudy(tuning).id, 'tuning'), 'tuning') + ledger.finish(ledger.claim(tuningId, 1000), 'failed', '1'.repeat(64)) + const changed = study() + changed.nights[0] = { ...changed.nights[0], id: 'renamed', startMs: start + 1000, inputHash: '0'.repeat(64), referenceHash: '1'.repeat(64) } + const next = ledger.reserveTrial(trial(ledger.createStudy(changed).id), 'new') + expect(ledger.claim(next, 1000)).toMatchObject({ unexposedHoldoutAtClaim: false, priorExposures: [{ nightId: 'renamed', count: 1 }] }) + }) + it('invalidates a previous first-use claim when later overlapping trials run', () => { + const studyId = ledger.createStudy(study()).id + const first = ledger.reserveTrial(trial(studyId), 'first') + ledger.finish(ledger.claim(first, 1000), 'succeeded', '1'.repeat(64)) + expect(ledger.currentEvidence(first).uniqueHoldoutAttempt).toBe(true) + const next = ledger.reserveTrial(trial(studyId), 'next') + ledger.claim(next, 1000) + expect(ledger.currentEvidence(first).uniqueHoldoutAttempt).toBe(false) + expect(new PackageExperimentLedger(db, () => now).currentEvidence(first).otherExposures[0].count).toBe(1) + }) + it('preserves independent sides and disjoint nights', () => { + const original = ledger.reserveTrial(trial(ledger.createStudy(study()).id), 'original') + ledger.claim(original, 1000) + for (const [key, changed] of [['right', { ...study(), side: 'right' }], ['later', { ...study(), nights: study().nights.map(night => ({ ...night, startMs: start + 86400000, endMs: start + 86400000 + 28800000 })) }]] as const) { + const id = ledger.reserveTrial(trial(ledger.createStudy(changed).id), key) + expect(ledger.claim(id, 1000).unexposedHoldoutAtClaim).toBe(true) + } + }) + it('rejects nonchronological and overlapping split plans', () => { + const value = study() + const later = { ...value.nights[0], id: 'later', startMs: start + 86400000, endMs: start + 86400000 + 28800000, split: 'tuning' } + expect(packageStudySchema.safeParse({ ...value, nights: [...value.nights, later] }).success).toBe(false) + expect(packageStudySchema.safeParse({ ...value, nights: [...value.nights, { ...value.nights[0], id: 'overlap' }] }).success).toBe(false) + }) + it('persists exposure and fences competing database connections across reopening', () => { + const directory = mkdtempSync(join(tmpdir(), 'package-ledger-test-')) + const path = join(directory, 'experiments.db') + const firstDb = new Database(path), secondDb = new Database(path) + try { + const first = new PackageExperimentLedger(firstDb, () => now), second = new PackageExperimentLedger(secondDb, () => now) + const studyId = first.createStudy(study()).id + const trialId = first.reserveTrial(trial(studyId), 'shared') + expect(second.reserveTrial(trial(studyId), 'shared')).toBe(trialId) + const lease = first.claim(trialId, 1000) + expect(() => second.claim(trialId, 1000)).toThrow('leased') + now += 1000 + const replacement = second.claim(trialId, 1000) + expect(() => first.finish(lease, 'succeeded', '1'.repeat(64))).toThrow('Stale') + second.finish(replacement, 'succeeded', '2'.repeat(64)) + firstDb.close() + secondDb.close() + const reopened = new Database(path) + try { + expect(new PackageExperimentLedger(reopened).currentEvidence(trialId)).toMatchObject({ uniqueHoldoutAttempt: false, reportHash: '2'.repeat(64) }) + } + finally { reopened.close() } + } + finally { + if (firstDb.open) firstDb.close() + if (secondDb.open) secondDb.close() + rmSync(directory, { recursive: true, force: true }) + } + }) + it('resumes interrupted initial migration branding and rejects unknown future versions', () => { + db.pragma('user_version = 0') + expect(() => new PackageExperimentLedger(db)).not.toThrow() + db.pragma('user_version = 2') + expect(() => new PackageExperimentLedger(db)).toThrow('Unsupported') + expect(db.pragma('user_version', { simple: true })).toBe(2) + }) + it('refuses another application database without changing its schema or data', () => { + const other = new Database(':memory:') + try { + other.exec('CREATE TABLE vitals(value INTEGER); INSERT INTO vitals VALUES (60)') + expect(() => new PackageExperimentLedger(other)).toThrow('non-experiment') + expect(other.prepare('SELECT value FROM vitals').get()).toEqual({ value: 60 }) + expect(other.pragma('application_id', { simple: true })).toBe(0) + } + finally { other.close() } + }) +}) + +it('allows a failed attempt without a report while refusing success without evidence', () => { + const id = ledger.reserveTrial(trial(ledger.createStudy(study()).id), 'missing-report') + const lease = ledger.claim(id, 1000) + expect(() => ledger.finish(lease, 'succeeded', null)).toThrow() + expect(ledger.trial(id).state).toBe('running') + ledger.finish(lease, 'failed', null) + expect(ledger.trial(id)).toMatchObject({ state: 'failed', report_hash: null, lease_until: null }) + expect(ledger.claim(id, 1000).unexposedHoldoutAtClaim).toBe(false) + expect(() => ledger.finish(lease, 'failed', null)).toThrow('Stale') +}) diff --git a/scripts/algorithm-experiments/ledger.ts b/scripts/algorithm-experiments/ledger.ts new file mode 100644 index 000000000..39e9dc290 --- /dev/null +++ b/scripts/algorithm-experiments/ledger.ts @@ -0,0 +1,140 @@ +import type Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { fileURLToPath } from 'node:url' +import { randomUUID } from 'node:crypto' +import { z } from 'zod' +import { configHash, dataHash } from '../../src/biometrics/algorithms/canonical' +import { timestampMsSchema } from '../../src/biometrics/references/contracts' + +const hash = z.string().regex(/^[a-f0-9]{64}$/) +const key = z.string().min(1).max(200) +const split = z.enum(['tuning', 'validation', 'holdout']) +export const packageStudySchema = z.object({ schema: z.literal('biometric-package-study-v1'), subjectId: key, side: z.enum(['left', 'right']), + protocolHash: hash, nights: z.array(z.object({ id: key, startMs: timestampMsSchema, endMs: timestampMsSchema, inputHash: hash, referenceHash: hash, split }).strict()).min(1).max(60), +}).strict().superRefine((study, context) => { + const order = { tuning: 0, validation: 1, holdout: 2 } + if (new Set(study.nights.map(night => night.id)).size !== study.nights.length) context.addIssue({ code: 'custom', message: 'Duplicate night identity' }) + const sorted = [...study.nights].sort((a, b) => a.startMs - b.startMs) + for (const [index, night] of sorted.entries()) { + if (night.endMs <= night.startMs || night.endMs - night.startMs > 86400000) context.addIssue({ code: 'custom', message: 'Invalid whole-night interval' }) + if (index > 0 && (sorted[index - 1].endMs > night.startMs || order[sorted[index - 1].split] > order[night.split])) context.addIssue({ code: 'custom', message: 'Nights must be disjoint chronological splits' }) + } +}) +const trialRequestSchema = z.object({ studyId: z.string().uuid(), split, + variants: z.array(z.object({ label: z.enum(['default', 'current', 'candidate']), packageHash: hash, configHash: hash, image: z.string().regex(/^sha256:[a-f0-9]{64}$/) }).strict()).length(3), +}).strict().refine(request => new Set(request.variants.map(variant => variant.label)).size === 3, 'All three variants are required') +interface StudyRow { id: string, hash: string, payload: string } +interface TrialRow { id: string, request_hash: string, request: string, state: 'pending' | 'running' | 'succeeded' | 'failed', attempt_id: string | null, lease_until: number | null, report_hash: string | null } +export interface PackageTrialLease { trialId: string, attemptId: string } +const applicationId = 0x53504558 + +/** Dedicated off-pod metadata database, never the pod's configuration or vitals + * database. Retain it across experiments: a fresh ledger cannot prove that a + * night was never observed elsewhere. Publication is fenced by attempt ID. */ +export class PackageExperimentLedger { + constructor(private readonly db: Database.Database, private readonly now = Date.now) { + const id = db.pragma('application_id', { simple: true }) + const version = db.pragma('user_version', { simple: true }) + if (id !== applicationId) { + const tables = db.prepare('SELECT name FROM sqlite_master WHERE name NOT LIKE \'sqlite_%\'').all() + if (id !== 0 || version !== 0 || tables.length) throw new Error('Refusing a non-experiment database') + db.pragma(`application_id = ${applicationId}`) + } + else if (version !== 0 && version !== 1) throw new Error('Unsupported experiment ledger version') + migrate(drizzle(db), { migrationsFolder: fileURLToPath(new URL('../../src/db/package-experiment-migrations', import.meta.url)) }) + db.pragma('user_version = 1') + } + + createStudy(input: unknown) { + const study = packageStudySchema.parse(input) + const hash = dataHash(study) + return this.db.transaction(() => { + const existing = this.db.prepare('SELECT * FROM package_studies WHERE hash = ?').get(hash) as StudyRow | undefined + if (existing) return { id: existing.id, hash, study } + const id = randomUUID() + this.db.prepare('INSERT INTO package_studies VALUES (?, ?, ?)').run(id, hash, JSON.stringify(study)) + return { id, hash, study } + }).immediate() + } + + reserveTrial(input: unknown, idempotencyKey: string) { + const request = trialRequestSchema.parse(input) + key.parse(idempotencyKey) + const requestHash = configHash(request) + const study = this.study(request.studyId) + if (!study.nights.some(night => night.split === request.split)) throw new Error('Selected split has no nights') + return this.db.transaction(() => { + const previous = this.db.prepare('SELECT * FROM package_trials WHERE idempotency_key = ?').get(idempotencyKey) as TrialRow | undefined + if (previous) { + if (previous.request_hash !== requestHash) throw new Error('Trial idempotency conflict') + return previous.id + } + const id = randomUUID() + this.db.prepare('INSERT INTO package_trials(id, idempotency_key, request_hash, request, state) VALUES (?, ?, ?, ?, \'pending\')').run(id, idempotencyKey, requestHash, JSON.stringify(request)) + return id + }).immediate() + } + + study(id: string) { + const row = this.db.prepare('SELECT * FROM package_studies WHERE id = ?').get(id) as StudyRow | undefined + if (!row) throw new Error('Unknown experiment study') + const study = packageStudySchema.parse(JSON.parse(row.payload)) + if (dataHash(study) !== row.hash) throw new Error('Study integrity mismatch') + return study + } + + trial(id: string) { + const row = this.db.prepare('SELECT * FROM package_trials WHERE id = ?').get(id) as TrialRow | undefined + if (!row) throw new Error('Unknown package trial') + const request = trialRequestSchema.parse(JSON.parse(row.request)) + if (configHash(request) !== row.request_hash) throw new Error('Trial integrity mismatch') + return { ...row, request } + } + + /** Exposure is committed before opening any selected input. Failed or expired + * attempts remain exposures; changing study/input/reference hashes cannot + * erase prior use of an overlapping night for the same subject and side. */ + claim(trialId: string, leaseMs: number) { + z.number().int().min(1000).max(3600000).parse(leaseMs) + return this.db.transaction(() => { + const trial = this.trial(trialId), now = this.now() + if (trial.state === 'succeeded') throw new Error('Trial already completed; reuse its report') + if (trial.state === 'running' && (trial.lease_until ?? 0) > now) throw new Error('Trial already leased') + const study = this.study(trial.request.studyId) + const nights = study.nights.filter(night => night.split === trial.request.split) + const priorExposures = nights.map(night => ({ nightId: night.id, count: (this.db.prepare(`SELECT COUNT(DISTINCT attempt_id) AS count FROM package_exposures + WHERE subject_id = ? AND side = ? AND start_ms < ? AND end_ms > ?`).get(study.subjectId, study.side, night.endMs, night.startMs) as { count: number }).count })) + const attemptId = randomUUID() + for (const night of nights) this.db.prepare('INSERT INTO package_exposures VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)').run(attemptId, trialId, night.id, study.subjectId, study.side, night.startMs, night.endMs, trial.request.split, now) + this.db.prepare('UPDATE package_trials SET state = \'running\', attempt_id = ?, lease_until = ?, report_hash = NULL WHERE id = ?').run(attemptId, now + leaseMs, trialId) + return { trialId, attemptId, priorExposures, unexposedHoldoutAtClaim: trial.request.split === 'holdout' && priorExposures.every(night => night.count === 0) } + }).immediate() + } + + /** Recomputed when evidence is reviewed; never rely on the claim-time flag + * after other experiments have accessed overlapping nights. */ + currentEvidence(trialId: string) { + return this.db.transaction(() => { + const trial = this.trial(trialId) + const study = this.study(trial.request.studyId) + const otherExposures = study.nights.filter(night => night.split === trial.request.split).map(night => ({ nightId: night.id, + count: (this.db.prepare(`SELECT COUNT(DISTINCT attempt_id) AS count FROM package_exposures + WHERE subject_id = ? AND side = ? AND start_ms < ? AND end_ms > ? AND attempt_id != ?`).get(study.subjectId, study.side, night.endMs, night.startMs, trial.attempt_id ?? '') as { count: number }).count })) + return { trialId, state: trial.state, reportHash: trial.report_hash, otherExposures, + uniqueHoldoutAttempt: trial.state === 'succeeded' && trial.request.split === 'holdout' && otherExposures.every(night => night.count === 0) } + }).deferred() + } + + renew(lease: PackageTrialLease, leaseMs: number) { + z.number().int().min(1000).max(3600000).parse(leaseMs) + const now = this.now() + if (this.db.prepare('UPDATE package_trials SET lease_until = ? WHERE id = ? AND attempt_id = ? AND state = \'running\' AND lease_until > ?').run(now + leaseMs, lease.trialId, lease.attemptId, now).changes !== 1) throw new Error('Stale trial lease') + } + + finish(lease: PackageTrialLease, status: 'succeeded' | 'failed', reportHash: string | null) { + z.enum(['succeeded', 'failed']).parse(status) + if (status === 'succeeded' || reportHash !== null) hash.parse(reportHash) + if (this.db.prepare('UPDATE package_trials SET state = ?, report_hash = ?, lease_until = NULL WHERE id = ? AND attempt_id = ? AND state = \'running\' AND lease_until > ?').run(status, reportHash, lease.trialId, lease.attemptId, this.now()).changes !== 1) throw new Error('Stale trial lease') + } +} diff --git a/scripts/algorithm-experiments/managed-evidence.ts b/scripts/algorithm-experiments/managed-evidence.ts new file mode 100644 index 000000000..7116d854e --- /dev/null +++ b/scripts/algorithm-experiments/managed-evidence.ts @@ -0,0 +1,100 @@ +import { verifyPreflightEvidence } from './preflight-evidence' +import { createHash } from 'node:crypto' +import { join } from 'node:path' +import { z } from 'zod' +import { configHash, dataHash } from '../../src/biometrics/algorithms/canonical' +import { createNightUncertainty } from '../../src/biometrics/backtest/uncertainty-core' +import { previewReferenceImport } from '../../src/biometrics/references/normalize' +import { comparePackageOutputs, packageComparisonInputSchema } from './comparison' +import { readJsonFile } from './files' +import { packageStudySchema, type PackageExperimentLedger } from './ledger' +import { managedProtocolSchema } from './managed-protocol' +import { assessPackageResources } from './resource-assessment' + +const hash = z.string().regex(/^[a-f0-9]{64}$/) +const summarySchema = z.object({ nightId: z.string(), reportHash: hash, comparison: z.unknown(), resources: z.unknown() }).strict() +const trialSchema = z.object({ schema: z.literal('biometric-managed-trial-v1'), trialId: z.string().uuid(), attemptId: z.string().uuid(), + study: packageStudySchema, protocol: managedProtocolSchema, request: z.unknown(), nights: z.array(summarySchema).min(1).max(60), + preflight: z.object({ reportHash: hash, status: z.enum(['failed', 'incomplete', 'external_checks_passed']) }).strict().optional(), + uncertainty: z.unknown(), exposureAtClaim: z.unknown(), + interpretation: z.literal('requires_current_ledger_exposure_check_and_full_qualification_review'), +}).strict() +const outputSchema = packageComparisonInputSchema.shape.outputs.element +const nightSchema = z.object({ schema: z.literal('biometric-managed-night-v1'), trialId: z.string().uuid(), attemptId: z.string().uuid(), + night: packageStudySchema.shape.nights.element, referenceImport: packageComparisonInputSchema.shape.referenceImport, + outputs: z.array(outputSchema.extend({ result: outputSchema.shape.result.extend({ execution: z.object({ + image: z.string(), limits: z.unknown(), measurements: z.unknown(), wallMs: z.unknown(), + }).passthrough() }).passthrough() })).length(3), comparison: z.unknown(), resources: z.unknown(), +}).strict() +const uncertainty = createNightUncertainty(text => createHash('sha256').update(text).digest(), 'equal_night_mean_common_mae_difference') +function retained(directory: string, identity: string) { + hash.parse(identity) + const value = readJsonFile(join(directory, `${identity}.json`), 32 * 1024 * 1024) + if (dataHash(value) !== identity) throw new Error('Retained trial report integrity mismatch') + return value +} +function same(actual: unknown, expected: unknown, description: string) { + if (dataHash(actual) !== dataHash(expected)) throw new Error(`Retained ${description} integrity mismatch`) +} + +/** Verifies the ledger-selected report graph without reopening night inputs or + * executing candidate code. Derived scores are recomputed, never accepted from + * the summary. This is an evidence snapshot, not an installation approval. */ +export function readManagedEvidence(ledger: PackageExperimentLedger, trialId: string, directory: string) { + const trial = ledger.trial(trialId) + if (trial.state !== 'succeeded' || !trial.report_hash) throw new Error('Trial has no successful retained evidence') + const report = trialSchema.parse(retained(directory, trial.report_hash)) + if (report.trialId !== trial.id || report.attemptId !== trial.attempt_id) throw new Error('Retained trial report identity mismatch') + const study = ledger.study(trial.request.studyId) + same(report.study, study, 'study') + same(report.request, trial.request, 'request') + if (configHash(report.protocol) !== study.protocolHash) throw new Error('Retained protocol integrity mismatch') + let preflight: ReturnType | null = null + if (report.protocol.preflightPlan) { + const plan = report.protocol.preflightPlan + const candidate = trial.request.variants.find(variant => variant.label === 'candidate') + if (!candidate || !report.preflight || plan.configHash !== candidate.configHash || plan.side !== study.side + || plan.hardwareId !== report.protocol.hardwareId || plan.sampleRateHz !== report.protocol.sampleRateHz) throw new Error('Retained preflight binding mismatch') + preflight = verifyPreflightEvidence(retained(directory, report.preflight.reportHash), { plan, packageHash: candidate.packageHash, image: candidate.image }) + same(preflight.status, report.preflight.status, 'preflight status') + } + else if (report.preflight) throw new Error('Unexpected retained preflight evidence') + const selected = study.nights.filter(night => night.split === trial.request.split) + if (report.nights.length !== selected.length || new Set(report.nights.map(night => night.nightId)).size !== selected.length + || selected.some(night => !report.nights.some(row => row.nightId === night.id))) throw new Error('Retained night set integrity mismatch') + const nights = report.nights.map((summary) => { + const night = nightSchema.parse(retained(directory, summary.reportHash)) + const expected = selected.find(row => row.id === summary.nightId) + same(night.night, expected, 'night') + if (night.trialId !== trial.id || night.attemptId !== trial.attempt_id) throw new Error('Retained night attempt integrity mismatch') + if (night.referenceImport.side !== study.side || night.referenceImport.subjectId !== study.subjectId + || previewReferenceImport(night.referenceImport).summary.contentHash !== night.night.referenceHash) throw new Error('Retained reference integrity mismatch') + for (const output of night.outputs) { + const variant = trial.request.variants.find(row => row.label === output.label) + if (!variant || output.request.packageHash !== variant.packageHash + || configHash({ config: output.request.config, seed: output.request.seed }) !== variant.configHash + || output.request.inputHash !== night.night.inputHash || output.request.side !== study.side + || output.request.hardwareId !== report.protocol.hardwareId || output.request.sampleRateHz !== report.protocol.sampleRateHz + || output.result.execution.image !== variant.image) throw new Error('Retained variant integrity mismatch') + same(output.result.execution.limits, output.manifest.resources, 'resource limits') + } + const comparison = comparePackageOutputs({ schema: 'biometric-package-comparison-v1', metricId: report.protocol.metricId, + startMs: night.night.startMs, endMs: night.night.endMs, minimumQuality: report.protocol.minimumQuality, + referenceImport: night.referenceImport, alignment: report.protocol.alignment, outputs: night.outputs }) + const resources = night.outputs.map(output => ({ label: output.label, assessment: assessPackageResources(output.result.execution, report.protocol.resourcePolicy) })) + same(night.comparison, comparison, 'night comparison') + same(summary.comparison, comparison, 'summary comparison') + same(night.resources, resources, 'night resources') + same(summary.resources, resources, 'summary resources') + return { nightId: summary.nightId, reportHash: summary.reportHash, comparison, resources } + }) + const uncertaintyReports = (['default', 'current'] as const).map(baseline => ({ baseline, + report: uncertainty(nights.map(night => ({ cohortId: night.nightId, + difference: night.comparison.comparisons.find(row => row.candidate === 'candidate' && row.baseline === baseline)?.differences.commonMae ?? null })), report.protocol.uncertainty) })) + same(report.uncertainty, uncertaintyReports, 'uncertainty') + const currentEvidence = ledger.currentEvidence(trialId) + if (currentEvidence.state !== 'succeeded' || currentEvidence.reportHash !== trial.report_hash) throw new Error('Trial changed during evidence verification') + return { schema: 'biometric-managed-evidence-v1' as const, trialId, reportHash: trial.report_hash, + study, protocol: report.protocol, request: trial.request, preflight, nights, uncertainty: uncertaintyReports, currentEvidence, + interpretation: 'verified_retained_evidence_requires_full_qualification_and_review' as const } +} diff --git a/scripts/algorithm-experiments/managed-files.test.ts b/scripts/algorithm-experiments/managed-files.test.ts new file mode 100644 index 000000000..ba57d87ec --- /dev/null +++ b/scripts/algorithm-experiments/managed-files.test.ts @@ -0,0 +1,77 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { chmod, mkdir, mkdtemp, readFile, rm, symlink, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { configHash } from '../../src/biometrics/algorithms/canonical' +import { packageFixture } from '../../src/biometrics/packages/tests/fixture' +import { fixtureResourceTestPolicy } from './resource-test-policy' +import { managedDescriptorSchema, runManagedFiles } from './managed-files' +let root: string +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'managed-storage-test-')) +}) +afterEach(async () => { + await rm(root, { recursive: true, force: true }) +}) +async function descriptor() { + const { manifest, files } = packageFixture() + await writeFile(join(root, 'proposal.json'), JSON.stringify({ schema: 'biometric-package-proposal-v1', manifest, + files: [...files].map(([path, bytes]) => ({ path, base64: bytes.toString('base64') })) })) + const protocol = { schema: 'biometric-managed-protocol-v1', hardwareId: 'pod4', sampleRateHz: 500, metricId: 'heart_rate', minimumQuality: 0.8, + alignment: { version: 'reference-alignment-v1', pointToleranceMs: 0, clockOffsetMs: 0, offsetSource: 'fixed' }, + uncertainty: { version: 'paired-night-percentile-v1', seed: 'fixture', resamples: 200, minimumNights: 3 }, resourcePolicy: fixtureResourceTestPolicy } + const value = { schema: 'biometric-managed-descriptor-v1', study: { schema: 'biometric-package-study-v1', subjectId: 'fixture', side: 'left', protocolHash: configHash(protocol), + nights: [{ id: 'night', startMs: 1791673200000, endMs: 1791673260000, inputHash: 'a'.repeat(64), referenceHash: 'b'.repeat(64), split: 'holdout' }] }, + protocol, split: 'holdout', idempotencyKey: 'test', variants: ['default', 'current', 'candidate'].map(label => ({ label, proposal: 'proposal.json', config: {}, seed: 1, + policy: { image: 'sha256:' + 'c'.repeat(64), maximumWallMs: 5000, maximumMemoryBytes: 100000000, maximumProcesses: 1, maximumOutputBytes: 10000, maximumInputBytes: 10000 } })), + nights: [{ id: 'night', frames: 'absent-frames', referenceImport: 'absent-reference' }] } + await writeFile(join(root, 'descriptor.json'), JSON.stringify(value)) + return value +} +describe('managed workspace boundary', () => { + it('rejects missing/extra night sources and protocol substitutions before opening inputs', async () => { + const value = await descriptor() + expect(managedDescriptorSchema.safeParse(value).success).toBe(true) + for (const changed of [{ ...value, nights: [] }, { ...value, nights: [...value.nights, value.nights[0]] }, + { ...value, protocol: { ...value.protocol, minimumQuality: 0.1 } }]) { + expect(managedDescriptorSchema.safeParse(changed).success).toBe(false) + } + }) + it('refuses public or symlinked workspaces', async () => { + await descriptor() + const publicPath = join(root, 'public') + await mkdir(publicPath) + await chmod(publicPath, 0o755) + await expect(runManagedFiles(publicPath, join(root, 'descriptor.json'))).rejects.toThrow('private owned directory') + const link = join(root, 'link') + await symlink(root, link) + await expect(runManagedFiles(link, join(root, 'descriptor.json'))).rejects.toThrow('private owned directory') + }) + it('refuses linked databases and sidecars without changing their targets', async () => { + await descriptor() + const target = join(root, 'target') + await writeFile(target, 'preserve me') + for (const name of ['experiments.sqlite', 'experiments.sqlite-wal', 'experiments.sqlite-shm', 'experiments.sqlite-journal']) { + const workspace = join(root, name + '-workspace') + await mkdir(workspace, { mode: 0o700 }) + await symlink(target, join(workspace, name)) + await expect(runManagedFiles(workspace, join(root, 'descriptor.json'))).rejects.toThrow('Unsafe experiment database') + expect(await readFile(target, 'utf8')).toBe('preserve me') + } + }) + it('refuses a foreign database and preserves its contents', async () => { + await descriptor() + const workspace = join(root, 'workspace') + await mkdir(workspace, { mode: 0o700 }) + const path = join(workspace, 'experiments.sqlite') + const database = new Database(path) + database.exec('CREATE TABLE vitals(value INTEGER); INSERT INTO vitals VALUES (60)') + database.close() + await chmod(path, 0o600) + const before = await readFile(path) + await expect(runManagedFiles(workspace, join(root, 'descriptor.json'))).rejects.toThrow('non-experiment database') + expect(await readFile(path)).toEqual(before) + }) +}) diff --git a/scripts/algorithm-experiments/managed-files.ts b/scripts/algorithm-experiments/managed-files.ts new file mode 100644 index 000000000..088a32676 --- /dev/null +++ b/scripts/algorithm-experiments/managed-files.ts @@ -0,0 +1,95 @@ +import { withExperimentWorkspace, defaultWorkspacePolicy } from './workspace' +import { assessManagedQualification } from './qualification' +import Database from 'better-sqlite3' +import { constants } from 'node:fs' +import { lstat, mkdir, open } from 'node:fs/promises' +import { dirname, join, resolve } from 'node:path' +import { z } from 'zod' +import { configHash } from '../../src/biometrics/algorithms/canonical' +import { packageStudySchema, PackageExperimentLedger } from './ledger' +import { managedProtocolSchema, runManagedTrial } from './managed' +import { experimentPolicySchema } from './policy' +import { readJsonFile, readJsonLines, readPackageProposal } from './files' + +const sourcePath = z.string().min(1).max(4096) +export const managedDescriptorSchema = z.object({ schema: z.literal('biometric-managed-descriptor-v1'), + study: packageStudySchema, protocol: managedProtocolSchema, split: z.enum(['tuning', 'validation', 'holdout']), + preflightSuite: sourcePath.optional(), + idempotencyKey: z.string().min(1).max(200), + variants: z.array(z.object({ label: z.enum(['default', 'current', 'candidate']), proposal: sourcePath, + config: z.record(z.string(), z.unknown()), seed: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER), policy: experimentPolicySchema }).strict()).length(3), + nights: z.array(z.object({ id: z.string().min(1).max(200), frames: sourcePath, referenceImport: sourcePath }).strict()).min(1).max(60), +}).strict().superRefine((value, context) => { + if (Boolean(value.protocol.preflightPlan) !== Boolean(value.preflightSuite)) context.addIssue({ code: 'custom', message: 'Preflight plan and suite must be supplied together' }) + if (new Set(value.variants.map(variant => variant.label)).size !== 3) context.addIssue({ code: 'custom', message: 'All three variants are required' }) + const selected = value.study.nights.filter(night => night.split === value.split) + if (new Set(value.nights.map(night => night.id)).size !== value.nights.length || selected.length !== value.nights.length + || selected.some(night => !value.nights.some(source => source.id === night.id))) context.addIssue({ code: 'custom', message: 'Night sources must exactly match the selected split' }) + if (configHash(value.protocol) !== value.study.protocolHash) context.addIssue({ code: 'custom', message: 'Study protocol hash mismatch' }) +}) + +async function privateDirectory(path: string) { + await mkdir(path, { recursive: true, mode: 0o700 }) + const stat = await lstat(path) + if (!stat.isDirectory() || stat.isSymbolicLink() || (stat.mode & 0o077) !== 0 + || (process.getuid && stat.uid !== process.getuid())) throw new Error('Experiment workspace must be a private owned directory, not a symlink') +} + +/** CLI storage boundary. Candidate bytes never choose this workspace or its + * database path; the operator passes the dedicated private workspace argument. */ +export async function runManagedFiles(workspace: string, descriptorPath: string, storagePolicy: unknown = defaultWorkspacePolicy) { + const descriptor = managedDescriptorSchema.parse(readJsonFile(descriptorPath, 2 * 1024 * 1024)) + const base = dirname(resolve(descriptorPath)) + const variants = descriptor.variants.map((variant) => { + const proposal = readPackageProposal(resolve(base, variant.proposal)) + return { ...variant, manifest: proposal.manifest, files: proposal.files, packageHash: proposal.packageHash } + }) + await privateDirectory(workspace) + return withExperimentWorkspace(workspace, storagePolicy, async (owned) => { + const reports = join(workspace, 'reports') + await privateDirectory(reports) + const databasePath = join(workspace, 'experiments.sqlite') + // Refuse symlinked SQLite files and sidecars before opening SQLite. The + // private owned directory excludes mutation by candidate/other-user code. + for (const path of [databasePath, databasePath + '-wal', databasePath + '-shm', databasePath + '-journal']) { + try { + const stat = await lstat(path) + if (!stat.isFile() || stat.isSymbolicLink() || stat.nlink !== 1 || (stat.mode & 0o077) !== 0 || (process.getuid && stat.uid !== process.getuid())) throw new Error('Unsafe experiment database file') + } + catch (error) { + if (!(error instanceof Error) || !('code' in error) || error.code !== 'ENOENT') throw error + } + } + try { + const created = await open(databasePath, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY | constants.O_NOFOLLOW, 0o600) + await created.close() + } + catch (error) { + if (!(error instanceof Error) || !('code' in error) || error.code !== 'EEXIST') throw error + } + const db = new Database(databasePath) + try { + owned.configureDatabase(db) + const ledger = new PackageExperimentLedger(db) + const study = ledger.createStudy(descriptor.study) + const trialId = ledger.reserveTrial({ studyId: study.id, split: descriptor.split, + variants: variants.map(variant => ({ label: variant.label, packageHash: variant.packageHash, + configHash: configHash({ config: variant.config, seed: variant.seed }), image: variant.policy.image })) }, descriptor.idempotencyKey) + const maximumInputBytes = Math.min(...variants.map(variant => variant.policy.maximumInputBytes)) + const result = await runManagedTrial({ ledger, trialId, protocol: descriptor.protocol, variants, reportDirectory: reports, publishReport: owned.publishReport, + preflightSuite: descriptor.preflightSuite && ledger.trial(trialId).state !== 'succeeded' ? readJsonFile(resolve(base, descriptor.preflightSuite), 32 * 1024 * 1024) : undefined, + loadNight: async (id) => { + await owned.checkCapacity() + const source = descriptor.nights.find(night => night.id === id) + if (!source) throw new Error('Missing retained night source') + return { referenceImport: readJsonFile(resolve(base, source.referenceImport), 2 * 1024 * 1024), + frames: () => readJsonLines(resolve(base, source.frames), maximumInputBytes) } + } }) + const reviewed = assessManagedQualification(ledger, trialId, reports) + return { schema: 'biometric-managed-receipt-v1' as const, studyId: study.id, trialId, ...result, + currentEvidence: reviewed.currentEvidence, accuracy: reviewed.qualification.accuracy, preflight: reviewed.qualification.preflight, + qualification: reviewed.qualification, assessmentHash: reviewed.assessmentHash, reportFile: join(reports, `${result.reportHash}.json`) } + } + finally { db.close() } + }) +} diff --git a/scripts/algorithm-experiments/managed-protocol.ts b/scripts/algorithm-experiments/managed-protocol.ts new file mode 100644 index 000000000..74bfc07d8 --- /dev/null +++ b/scripts/algorithm-experiments/managed-protocol.ts @@ -0,0 +1,14 @@ +import { packagePreflightPlanSchema } from './preflight' +import { packageAccuracyPolicySchema } from './accuracy-policy' +import { z } from 'zod' +import { nightBootstrapSchema } from '../../src/biometrics/backtest/uncertainty-policy' +import { packageResourcePolicySchema } from './resource-assessment' + +export const managedProtocolSchema = z.object({ schema: z.literal('biometric-managed-protocol-v1'), + hardwareId: z.string().min(1).max(100), sampleRateHz: z.number().int().positive(), metricId: z.string().min(1).max(100), + minimumQuality: z.number().min(0).max(1), + alignment: z.object({ version: z.literal('reference-alignment-v1'), pointToleranceMs: z.number().int().min(0).max(300000), + clockOffsetMs: z.number().int().min(-3600000).max(3600000), offsetSource: z.enum(['fixed', 'training']) }).strict(), + accuracyPolicy: packageAccuracyPolicySchema.optional(), preflightPlan: packagePreflightPlanSchema.optional(), + uncertainty: nightBootstrapSchema, resourcePolicy: packageResourcePolicySchema, +}).strict() diff --git a/scripts/algorithm-experiments/managed.test.ts b/scripts/algorithm-experiments/managed.test.ts new file mode 100644 index 000000000..a0256a3dc --- /dev/null +++ b/scripts/algorithm-experiments/managed.test.ts @@ -0,0 +1,299 @@ +import { defaultWorkspacePolicy } from './workspace' +// @vitest-environment node +import Database from 'better-sqlite3' +import { createHash } from 'node:crypto' +import { mkdtemp, readFile, rm, writeFile, stat } from 'node:fs/promises' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import * as reportFiles from './files' +import { beforeAll, describe, expect, it, vi } from 'vitest' +import { configHash, dataHash } from '../../src/biometrics/algorithms/canonical' +import { previewReferenceImport } from '../../src/biometrics/references/normalize' +import { packageFixture } from '../../src/biometrics/packages/tests/fixture' +import { inspectPythonEnvironment } from './environment' +import { packageInputHash } from './input' +import { PackageExperimentLedger } from './ledger' +import { runManagedFiles } from './managed-files' +import { runManagedTrial } from './managed' +import { readManagedEvidence } from './managed-evidence' +import { runPackagePreflight } from './preflight' +import { verifyPreflightEvidence } from './preflight-evidence' +import { assessManagedQualification } from './qualification' +import { fixtureResourceTestPolicy } from './resource-test-policy' +const image = process.env.BIOMETRIC_EXPERIMENT_TEST_IMAGE +let environmentBytes: Buffer +beforeAll(async () => { + if (image) environmentBytes = Buffer.from(JSON.stringify(await inspectPythonEnvironment(image))) +}) +const start = 1791673200000 +function fixture(ledger: PackageExperimentLedger, withPreflight = false) { + const { manifest, files } = packageFixture() + const code = `import json, sys +init = json.loads(sys.stdin.readline()) +for line in sys.stdin: + msg = json.loads(line) + frame = msg['frame'] + print(json.dumps({'schema': 'biometric-metric-frame-v1', 'runHash': init['runHash'], 'frameHash': msg['frameHash'], 'sequence': frame['sequence'], 'state': {}, 'readings': [{'timestampMs': frame['timestampMs'], 'side': 'left', 'metric': 'heart_rate', 'value': 60 + init['request']['config']['offset'], 'quality': 1, 'flags': []}]}), flush=True) +` + files.set('engine.py', Buffer.from(code)) + if (environmentBytes) files.set(manifest.dependencyLock, environmentBytes) + const declared = { ...manifest, metrics: manifest.metrics.map(metric => ({ ...metric, reference: { metric: 'heart_rate', windowMs: null } })), + resources: { cpuMs: 5000, wallMs: 5000, memoryBytes: 64 * 1024 * 1024, processes: 1, outputBytes: 16384 }, + artifacts: [...files].map(([path, bytes]) => ({ path, bytes: bytes.length, sha256: createHash('sha256').update(bytes).digest('hex') })) } + const frames = [0, 1].map(sequence => ({ schema: 'biometric-signal-frame-v1', sequence, timestampMs: start + sequence * 1000, + samples: { left: [1, 2, 3], right: null }, context: { occupied: true, pumpActive: false } })) + const referenceImport = { side: 'left' as const, subjectId: 'fixture', source: 'synthetic', rows: [0, 1].map(index => ({ recordKey: String(index), metric: 'heart_rate', value: 60, unit: 'bpm', timestampMs: start + index * 1000 })) } + const preflightSuite = { schema: 'biometric-package-fixtures-v1', resourcePolicy: fixtureResourceTestPolicy, + fixtures: [{ id: 'clean', category: 'clean_signal', frames, + request: { schema: 'biometric-package-run-v1', packageHash: configHash(declared), inputHash: packageInputHash(frames), + hardwareId: 'pod4', sampleRateHz: 500, side: 'left', config: { offset: 0 }, seed: 1 }, + expectations: [{ kind: 'numeric', metric: 'heart_rate', startMs: start, endMs: start + 1006, + minimum: 59, maximum: 61, minimumReadings: 2, minimumQuality: 0.8 }] }] } + const preflightPlan = { schema: 'biometric-package-preflight-plan-v1', suiteHash: dataHash(preflightSuite), repetitions: 2, + configHash: configHash({ config: { offset: 0 }, seed: 1 }), hardwareId: 'pod4', sampleRateHz: 500, side: 'left' } + const protocol = { schema: 'biometric-managed-protocol-v1', hardwareId: 'pod4', sampleRateHz: 500, metricId: 'heart_rate', minimumQuality: 0.8, ...(withPreflight ? { preflightPlan } : {}), + alignment: { version: 'reference-alignment-v1', pointToleranceMs: 0, clockOffsetMs: 0, offsetSource: 'fixed' }, + uncertainty: { version: 'paired-night-percentile-v1', seed: 'fixture', resamples: 200, minimumNights: 3 }, resourcePolicy: fixtureResourceTestPolicy } + const study = ledger.createStudy({ schema: 'biometric-package-study-v1', subjectId: 'fixture', side: 'left', protocolHash: configHash(protocol), + nights: [{ id: 'night', startMs: start, endMs: start + 120000, inputHash: packageInputHash(frames), referenceHash: previewReferenceImport(referenceImport).summary.contentHash, split: 'holdout' }] }) + const variants = (['default', 'current', 'candidate'] as const).map((label, index) => ({ label, manifest: declared, files, + config: { offset: 4 - index * 2 }, seed: 1, + policy: { image: image ?? 'sha256:' + 'f'.repeat(64), maximumWallMs: 10000, maximumMemoryBytes: 128 * 1024 * 1024, maximumProcesses: 4, maximumOutputBytes: 32768, maximumInputBytes: 32768 } })) + const request = { studyId: study.id, split: 'holdout', variants: variants.map(variant => ({ label: variant.label, packageHash: configHash(declared), + configHash: configHash({ config: variant.config, seed: variant.seed }), image: variant.policy.image })) } + return { trialId: ledger.reserveTrial(request, 'first'), request, protocol, variants, referenceImport, frames, ...(withPreflight ? { preflightSuite } : {}) } +} +describe.skipIf(!image)('managed package trials through real Docker', () => { + it('executes combined fixture and determinism preflight for the exact candidate configuration', async () => { + const db = new Database(':memory:') + try { + const input = fixture(new PackageExperimentLedger(db)), candidate = input.variants[2] + const suite = { schema: 'biometric-package-fixtures-v1', resourcePolicy: fixtureResourceTestPolicy, + fixtures: [{ id: 'clean', category: 'clean_signal', frames: input.frames, + request: { schema: 'biometric-package-run-v1', packageHash: configHash(candidate.manifest), inputHash: packageInputHash(input.frames), + hardwareId: 'pod4', sampleRateHz: 500, side: 'left', config: candidate.config, seed: candidate.seed }, + expectations: [{ kind: 'numeric', metric: 'heart_rate', startMs: start, endMs: start + 1006, + minimum: 59, maximum: 61, minimumReadings: 2, minimumQuality: 0.8 }] }] } + const plan = { schema: 'biometric-package-preflight-plan-v1', suiteHash: dataHash(suite), repetitions: 2, + configHash: configHash({ config: candidate.config, seed: candidate.seed }), hardwareId: 'pod4', sampleRateHz: 500, side: 'left' } + const result = await runPackagePreflight(candidate, plan, suite) + expect(result).toMatchObject({ status: 'incomplete', packageHash: configHash(candidate.manifest), planHash: configHash(plan), + fixtureReport: { missingCategories: expect.arrayContaining(['empty_bed', 'motion']) }, + determinism: [{ fixtureId: 'clean', error: null, report: { deterministic: true, runs: expect.any(Array) } }] }) + expect(verifyPreflightEvidence(result, { plan, packageHash: configHash(candidate.manifest), image: candidate.policy.image }).status).toBe('incomplete') + const changed = structuredClone(result) + changed.status = 'external_checks_passed' + expect(() => verifyPreflightEvidence(changed, { plan, packageHash: result.packageHash, image: result.image })).toThrow('integrity mismatch') + const changedRuns = structuredClone(result) + const repeated = changedRuns.determinism[0].report + if (!repeated) throw new Error('Missing determinism fixture') + repeated.runs[1].outputHash = 'f'.repeat(64) + expect(() => verifyPreflightEvidence(changedRuns, { plan, packageHash: result.packageHash, image: result.image })).toThrow('integrity mismatch') + expect(result.determinism[0].report?.runs).toHaveLength(2) + expect(result.determinism[0].resources).toHaveLength(2) + const failingSuite = structuredClone(suite) + failingSuite.fixtures[0].expectations[0].minimum = 58 + failingSuite.fixtures[0].expectations[0].maximum = 59 + expect(await runPackagePreflight(candidate, { ...plan, suiteHash: dataHash(failingSuite) }, failingSuite)).toMatchObject({ + status: 'failed', fixtureReport: { status: 'failed' }, + }) + for (const change of [{ configHash: 'f'.repeat(64) }, { side: 'right' }, { hardwareId: 'other' }, { sampleRateHz: 100 }, { suiteHash: 'f'.repeat(64) }]) { + await expect(runPackagePreflight(candidate, { ...plan, ...change }, suite)).rejects.toThrow(/mismatch/) + } + const controller = new AbortController() + controller.abort() + await expect(runPackagePreflight({ ...candidate, signal: controller.signal }, plan, suite)).rejects.toThrow() + } + finally { db.close() } + }, 30000) + + it('records exposure before loading, compares all variants, publishes verified artifacts and reuses completion', async () => { + const db = new Database(':memory:'), root = await mkdtemp(join(tmpdir(), 'managed-trial-test-')) + try { + const ledger = new PackageExperimentLedger(db) + const input = fixture(ledger, true) + let loads = 0 + const options = { ...input, ledger, reportDirectory: root, loadNight: async () => { + loads++ + expect(ledger.trial(input.trialId).state).toBe('running') + expect((db.prepare('SELECT COUNT(*) AS count FROM package_exposures').get() as { count: number }).count).toBe(1) + return { referenceImport: input.referenceImport, frames: () => input.frames } + } } + const result = await runManagedTrial(options) + expect(result).toMatchObject({ state: 'succeeded', evidence: { uniqueHoldoutAttempt: true } }) + const report = JSON.parse(await readFile(join(root, `${result.reportHash}.json`), 'utf8')) + expect(dataHash(report)).toBe(result.reportHash) + expect(report.preflight.status).toBe('incomplete') + const preflightPath = join(root, `${report.preflight.reportHash}.json`) + const preflightBytes = await readFile(preflightPath) + await rm(preflightPath) + await expect(runManagedTrial(options)).rejects.toThrow() + expect(loads).toBe(1) + await writeFile(preflightPath, preflightBytes) + expect(readManagedEvidence(ledger, input.trialId, root).preflight?.status).toBe('incomplete') + expect(report.uncertainty.map((row: { report: { estimate: number } }) => row.report.estimate)).toEqual([-4, -2]) + const night = JSON.parse(await readFile(join(root, `${report.nights[0].reportHash}.json`), 'utf8')) + expect(dataHash(night)).toBe(report.nights[0].reportHash) + expect(night.outputs).toHaveLength(3) + expect(await runManagedTrial(options)).toEqual({ state: 'already_completed', reportHash: result.reportHash }) + expect(loads).toBe(1) + const qualification = assessManagedQualification(ledger, input.trialId, root) + expect(assessManagedQualification(ledger, input.trialId, root).assessmentHash).toBe(qualification.assessmentHash) + expect(qualification.qualification.installationAuthorized).toBe(false) + expect(readManagedEvidence(ledger, input.trialId, root)).toMatchObject({ + reportHash: result.reportHash, currentEvidence: { uniqueHoldoutAttempt: true }, + }) + // A valid top-level hash does not excuse a missing or corrupted child. + const childPath = join(root, `${report.nights[0].reportHash}.json`) + await rm(childPath) + await expect(runManagedTrial(options)).rejects.toThrow() + await writeFile(childPath, JSON.stringify({ ...night, outputs: [] })) + await expect(runManagedTrial(options)).rejects.toThrow('integrity mismatch') + await writeFile(childPath, JSON.stringify(night)) + // Even a rehashed report graph cannot substitute a derived score. The + // verifier recomputes from the retained outputs and frozen protocol. + const forgedChild = structuredClone(night) + forgedChild.comparison.comparisons[0].differences.commonMae = -999 + const forgedChildHash = dataHash(forgedChild) + await writeFile(join(root, `${forgedChildHash}.json`), JSON.stringify(forgedChild)) + const forged = structuredClone(report) + forged.nights[0].reportHash = forgedChildHash + forged.nights[0].comparison = forgedChild.comparison + const forgedHash = dataHash(forged) + await writeFile(join(root, `${forgedHash}.json`), JSON.stringify(forged)) + db.prepare('UPDATE package_trials SET report_hash = ? WHERE id = ?').run(forgedHash, input.trialId) + expect(() => readManagedEvidence(ledger, input.trialId, root)).toThrow('night comparison integrity') + // Retained request/uncertainty substitutions are rejected independently. + for (const replacement of [{ ...report, uncertainty: [] }, { ...report, request: { ...report.request, split: 'tuning' } }]) { + const replacementHash = dataHash(replacement) + await writeFile(join(root, `${replacementHash}.json`), JSON.stringify(replacement)) + db.prepare('UPDATE package_trials SET report_hash = ? WHERE id = ?').run(replacementHash, input.trialId) + expect(() => readManagedEvidence(ledger, input.trialId, root)).toThrow('integrity mismatch') + } + db.prepare('UPDATE package_trials SET report_hash = ? WHERE id = ?').run(result.reportHash, input.trialId) + await writeFile(join(root, `${result.reportHash}.json`), JSON.stringify({ ...report, trialId: 'changed' })) + await expect(runManagedTrial(options)).rejects.toThrow('integrity mismatch') + expect(loads).toBe(1) + await writeFile(join(root, `${result.reportHash}.json`), JSON.stringify(report)) + const second = ledger.reserveTrial(input.request, 'second') + ledger.claim(second, 30000) + expect(ledger.currentEvidence(input.trialId).uniqueHoldoutAttempt).toBe(false) + expect(readManagedEvidence(ledger, input.trialId, root).currentEvidence.uniqueHoldoutAttempt).toBe(false) + const afterExposure = assessManagedQualification(ledger, input.trialId, root) + expect(afterExposure.assessmentHash).not.toBe(qualification.assessmentHash) + expect(afterExposure.qualification.accuracy.missing).toContain('no_unique_current_holdout') + expect(loads).toBe(1) + } + finally { + db.close() + await rm(root, { recursive: true, force: true }) + } + }, 30000) + it('runs a managed descriptor through persistent storage and reuses it without reopening night files', async () => { + const db = new Database(':memory:'), root = await mkdtemp(join(tmpdir(), 'managed-files-test-')) + try { + const ledger = new PackageExperimentLedger(db), input = fixture(ledger, true) + const variant = input.variants[0] + const proposal = { schema: 'biometric-package-proposal-v1', manifest: variant.manifest, + files: [...variant.files].map(([path, bytes]) => ({ path, base64: bytes.toString('base64') })) } + const descriptor = { schema: 'biometric-managed-descriptor-v1', study: ledger.study(input.request.studyId), protocol: input.protocol, + split: 'holdout', idempotencyKey: 'managed-files', preflightSuite: 'suite.json', + variants: input.variants.map(variant => ({ label: variant.label, proposal: 'proposal.json', config: variant.config, seed: variant.seed, policy: variant.policy })), + nights: [{ id: 'night', frames: 'frames.jsonl', referenceImport: 'references.json' }] } + await writeFile(join(root, 'proposal.json'), JSON.stringify(proposal)) + await writeFile(join(root, 'descriptor.json'), JSON.stringify(descriptor)) + await writeFile(join(root, 'suite.json'), JSON.stringify(input.preflightSuite)) + await writeFile(join(root, 'references.json'), JSON.stringify(input.referenceImport)) + await writeFile(join(root, 'frames.jsonl'), input.frames.map(frame => JSON.stringify(frame)).join('\n')) + const workspace = join(root, 'workspace') + const first = await runManagedFiles(workspace, join(root, 'descriptor.json')) + expect(first).toMatchObject({ state: 'succeeded', currentEvidence: { uniqueHoldoutAttempt: true }, + accuracy: { status: 'insufficient_evidence', missing: expect.arrayContaining(['no_frozen_accuracy_policy']) } }) + expect((await stat(join(workspace, 'experiments.sqlite'))).mode & 0o777).toBe(0o600) + expect(dataHash(JSON.parse(await readFile(first.reportFile, 'utf8')))).toBe(first.reportHash) + expect(first.preflight?.status).toBe('incomplete') + expect(first.qualification).toMatchObject({ status: 'insufficient_evidence', installationAuthorized: false }) + expect(first.assessmentHash).toMatch(/^[a-f0-9]{64}$/) + await rm(join(root, 'suite.json')) + await rm(join(root, 'frames.jsonl')) + await rm(join(root, 'references.json')) + const repeated = await runManagedFiles(workspace, join(root, 'descriptor.json')) + expect(repeated).toMatchObject({ state: 'already_completed', trialId: first.trialId, reportHash: first.reportHash, assessmentHash: first.assessmentHash }) + await expect(runManagedFiles(workspace, join(root, 'descriptor.json'), { ...defaultWorkspacePolicy, maximumReportFiles: 1 })).rejects.toThrow('file quota') + descriptor.variants[2].config.offset++ + await writeFile(join(root, 'descriptor.json'), JSON.stringify(descriptor)) + await expect(runManagedFiles(workspace, join(root, 'descriptor.json'))).rejects.toThrow('idempotency conflict') + } + finally { + db.close() + await rm(root, { recursive: true, force: true }) + } + }, 30000) + it('refuses publication after its lease expires while loading retained input', async () => { + const db = new Database(':memory:'), root = await mkdtemp(join(tmpdir(), 'managed-trial-stale-')) + let now = 1000 + try { + const ledger = new PackageExperimentLedger(db, () => now), input = fixture(ledger) + await expect(runManagedTrial({ ...input, ledger, reportDirectory: root, loadNight: async () => { + now += 30000 + return { referenceImport: input.referenceImport, frames: () => input.frames } + } })).rejects.toThrow('Stale trial lease') + expect(ledger.trial(input.trialId).report_hash).toBeNull() + expect(ledger.claim(input.trialId, 30000).unexposedHoldoutAtClaim).toBe(false) + } + finally { + db.close() + await rm(root, { recursive: true, force: true }) + } + }) + it('fails substituted references after recording access and preserves failure evidence', async () => { + const db = new Database(':memory:'), root = await mkdtemp(join(tmpdir(), 'managed-trial-failure-')) + try { + const ledger = new PackageExperimentLedger(db), input = fixture(ledger) + await expect(runManagedTrial({ ...input, ledger, reportDirectory: root, loadNight: async () => ({ + referenceImport: { ...input.referenceImport, source: 'changed' }, frames: () => input.frames, + }) })).rejects.toThrow('reference identity') + const trial = ledger.trial(input.trialId) + expect(trial.state).toBe('failed') + const report = JSON.parse(await readFile(join(root, `${trial.report_hash}.json`), 'utf8')) + expect(dataHash(report)).toBe(trial.report_hash) + expect(ledger.claim(input.trialId, 30000).unexposedHoldoutAtClaim).toBe(false) + } + finally { + db.close() + await rm(root, { recursive: true, force: true }) + } + }) +}) + +// These fail before any container or environment probe and run without Docker. +it.each([false, true])('finalizes failed trials when report publication fails (stale lease: %s)', async (stale) => { + const db = new Database(':memory:'), root = await mkdtemp(join(tmpdir(), 'managed-publication-failure-')) + let now = 1000 + const original = new Error('retained input unavailable') + const publication = new Error('report disk full') + const publish = vi.spyOn(reportFiles, 'publishJson').mockRejectedValue(publication) + try { + const ledger = new PackageExperimentLedger(db, () => now), input = fixture(ledger) + let replacement: ReturnType | undefined + const task = runManagedTrial({ ...input, ledger, reportDirectory: root, loadNight: async () => { + if (stale) { + now += 30000 + replacement = ledger.claim(input.trialId, 30000) + } + throw original + } }) + await expect(task).rejects.toMatchObject({ cause: original, errors: expect.arrayContaining([original, publication]), message: expect.stringContaining('failure evidence incomplete') }) + const trial = ledger.trial(input.trialId) + expect(trial).toMatchObject(stale + ? { state: 'running', attempt_id: replacement?.attemptId, report_hash: null } + : { state: 'failed', report_hash: null, lease_until: null }) + if (!stale) expect(ledger.claim(input.trialId, 30000).unexposedHoldoutAtClaim).toBe(false) + expect(publish).toHaveBeenCalledOnce() + } + finally { + publish.mockRestore() + db.close() + await rm(root, { recursive: true, force: true }) + } +}) diff --git a/scripts/algorithm-experiments/managed.ts b/scripts/algorithm-experiments/managed.ts new file mode 100644 index 000000000..a6fb9cdd4 --- /dev/null +++ b/scripts/algorithm-experiments/managed.ts @@ -0,0 +1,162 @@ +import { packageFixtureSuiteSchema } from './fixtures' +import { runPackagePreflight } from './preflight' +import { readManagedEvidence } from './managed-evidence' +import { managedProtocolSchema } from './managed-protocol' +export { managedProtocolSchema } from './managed-protocol' +import { createHash } from 'node:crypto' +import { mkdir } from 'node:fs/promises' +import { join } from 'node:path' +import { z } from 'zod' +import { configHash, dataHash } from '../../src/biometrics/algorithms/canonical' +import { createNightUncertainty } from '../../src/biometrics/backtest/uncertainty-core' +import { verifyAlgorithmPackage } from '../../src/biometrics/packages/manifest' +import { signalFrameSchema } from '../../src/biometrics/packages/protocol' +import { referenceImportSchema } from '../../src/biometrics/references/contracts' +import { previewReferenceImport } from '../../src/biometrics/references/normalize' +import { executePackageExperiment, type ExperimentPolicy } from './container' +import type { PackageExperimentLedger } from './ledger' +import { comparePackageOutputs } from './comparison' +import { assessPackageResources } from './resource-assessment' +import { publishJson } from './files' + +interface Variant { + label: 'default' | 'current' | 'candidate' + manifest: unknown + files: ReadonlyMap + config: Record + seed: number + policy: ExperimentPolicy +} +const uncertainty = createNightUncertainty(text => createHash('sha256').update(text).digest(), 'equal_night_mean_common_mae_difference') + +/** The loader is trusted host code, called only after exposure is committed. + * It must read retained inputs; no candidate module is loaded on the host. */ +export async function runManagedTrial(options: { + ledger: PackageExperimentLedger + trialId: string + protocol: unknown + preflightSuite?: unknown + variants: Variant[] + reportDirectory: string + publishReport?: typeof publishJson + loadNight: (id: string) => Promise<{ referenceImport: unknown, frames: () => Iterable }> +}) { + const publishReport = options.publishReport ?? publishJson + const trial = options.ledger.trial(options.trialId) + if (trial.state === 'succeeded') { + const evidence = readManagedEvidence(options.ledger, trial.id, options.reportDirectory) + return { state: 'already_completed' as const, reportHash: evidence.reportHash } + } + const study = options.ledger.study(trial.request.studyId) + const protocol = managedProtocolSchema.parse(structuredClone(options.protocol)) + if (configHash(protocol) !== study.protocolHash) throw new Error('Managed protocol identity mismatch') + if (options.variants.length !== 3 || new Set(options.variants.map(variant => variant.label)).size !== 3) throw new Error('Three distinct variants are required') + const variants = options.variants.map((variant) => { + const verified = verifyAlgorithmPackage(variant.manifest, variant.files) + const expected = trial.request.variants.find(item => item.label === variant.label) + const config = structuredClone(variant.config) + const seed = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER).parse(variant.seed) + if (!expected || verified.packageHash !== expected.packageHash || configHash({ config, seed }) !== expected.configHash || variant.policy.image !== expected.image) throw new Error('Managed variant identity mismatch') + return { ...variant, manifest: verified.manifest, packageHash: verified.packageHash, config, seed, + files: new Map([...variant.files].map(([path, bytes]) => [path, Buffer.from(bytes)])), policy: structuredClone(variant.policy) } + }) + const candidate = variants.find(variant => variant.label === 'candidate') + if (protocol.preflightPlan && (!candidate || protocol.preflightPlan.configHash !== configHash({ config: candidate.config, seed: candidate.seed }) + || protocol.preflightPlan.hardwareId !== protocol.hardwareId || protocol.preflightPlan.sampleRateHz !== protocol.sampleRateHz + || protocol.preflightPlan.side !== study.side)) throw new Error('Managed preflight candidate mismatch') + if (Boolean(protocol.preflightPlan) !== (options.preflightSuite !== undefined)) throw new Error('Managed preflight plan and suite must be supplied together') + const preflightSuite = options.preflightSuite === undefined ? undefined : packageFixtureSuiteSchema.parse(structuredClone(options.preflightSuite)) + if (protocol.preflightPlan && dataHash(preflightSuite) !== protocol.preflightPlan.suiteHash) throw new Error('Managed preflight suite mismatch') + await mkdir(options.reportDirectory, { recursive: true, mode: 0o700 }) + const lease = options.ledger.claim(options.trialId, 30000) + const controller = new AbortController() + const heartbeat = setInterval(() => { + try { + options.ledger.renew(lease, 30000) + } + catch { controller.abort() } + }, 5000) + const nights: { + nightId: string + reportHash: string + comparison: ReturnType + resources: { label: Variant['label'], assessment: ReturnType }[] + }[] = [] + let preflight: { reportHash: string, status: 'failed' | 'incomplete' | 'external_checks_passed' } | undefined + try { + if (protocol.preflightPlan && candidate) { + const report = await runPackagePreflight({ ...candidate, signal: controller.signal }, protocol.preflightPlan, preflightSuite) + const reportHash = dataHash(report) + options.ledger.renew(lease, 30000) + await publishReport(join(options.reportDirectory, `${reportHash}.json`), report) + preflight = { reportHash, status: report.status } + } + for (const night of study.nights.filter(night => night.split === trial.request.split)) { + controller.signal.throwIfAborted() + options.ledger.renew(lease, 30000) + const source = await options.loadNight(night.id) + const referenceImport = referenceImportSchema.parse(structuredClone(source.referenceImport)) + if (referenceImport.side !== study.side || referenceImport.subjectId !== study.subjectId + || previewReferenceImport(referenceImport).summary.contentHash !== night.referenceHash) throw new Error('Managed reference identity mismatch') + const outputs = [] + for (const variant of variants) { + controller.signal.throwIfAborted() + options.ledger.renew(lease, 30000) + const request = { schema: 'biometric-package-run-v1', packageHash: variant.packageHash, inputHash: night.inputHash, + hardwareId: protocol.hardwareId, sampleRateHz: protocol.sampleRateHz, side: study.side, seed: variant.seed, config: variant.config } + const frames = function* () { + for (const input of source.frames()) { + const frame = signalFrameSchema.parse(input) + const length = frame.samples[study.side]?.length ?? 0 + if (frame.timestampMs < night.startMs || frame.timestampMs + length * 1000 / protocol.sampleRateHz > night.endMs) throw new Error('Frame outside frozen night') + yield frame + } + } + const result = await executePackageExperiment({ ...variant, request, frames: frames(), signal: controller.signal }) + outputs.push({ label: variant.label, manifest: variant.manifest, request, result }) + } + const comparison = comparePackageOutputs({ schema: 'biometric-package-comparison-v1', metricId: protocol.metricId, + startMs: night.startMs, endMs: night.endMs, minimumQuality: protocol.minimumQuality, referenceImport, alignment: protocol.alignment, outputs }) + const report = { schema: 'biometric-managed-night-v1', trialId: lease.trialId, attemptId: lease.attemptId, night, referenceImport, outputs, comparison, + resources: outputs.map(output => ({ label: output.label, assessment: assessPackageResources(output.result.execution, protocol.resourcePolicy) })) } + const reportHash = dataHash(report) + options.ledger.renew(lease, 30000) + await publishReport(join(options.reportDirectory, `${reportHash}.json`), report) + nights.push({ nightId: night.id, reportHash, comparison, resources: report.resources }) + } + const uncertaintyReports = (['default', 'current'] as const).map(baseline => ({ baseline, + report: uncertainty(nights.map(night => ({ cohortId: night.nightId, + difference: night.comparison.comparisons.find(comparison => comparison.candidate === 'candidate' && comparison.baseline === baseline)?.differences.commonMae ?? null })), protocol.uncertainty) })) + const report = { schema: 'biometric-managed-trial-v1', trialId: lease.trialId, attemptId: lease.attemptId, + study, protocol, request: trial.request, ...(preflight ? { preflight } : {}), nights, uncertainty: uncertaintyReports, exposureAtClaim: lease, + interpretation: 'requires_current_ledger_exposure_check_and_full_qualification_review' } + const reportHash = dataHash(report) + options.ledger.renew(lease, 30000) + await publishReport(join(options.reportDirectory, `${reportHash}.json`), report) + options.ledger.finish(lease, 'succeeded', reportHash) + return { state: 'succeeded' as const, reportHash, evidence: options.ledger.currentEvidence(lease.trialId) } + } + catch (error) { + controller.abort() + const report = { schema: 'biometric-managed-trial-failure-v1', trialId: lease.trialId, attemptId: lease.attemptId, ...(preflight ? { preflight } : {}), nights, + error: error instanceof Error ? error.message.slice(0, 1000) : 'Managed trial failed' } + const reportHash = dataHash(report) + let published = false + const secondaryErrors: unknown[] = [] + try { + await publishReport(join(options.reportDirectory, `${reportHash}.json`), report) + published = true + } + catch (publicationError) { secondaryErrors.push(publicationError) } + try { + // Publication failure must not leave a writable ledger's attempt running. + // A missing artifact has no report hash. The lease still fences replacement. + options.ledger.finish(lease, 'failed', published ? reportHash : null) + } + catch (ledgerError) { secondaryErrors.push(ledgerError) } + if (secondaryErrors.length) throw new AggregateError([error, ...secondaryErrors], + `${error instanceof Error ? error.message : 'Managed trial failed'}; failure evidence incomplete`, { cause: error }) + throw error + } + finally { clearInterval(heartbeat) } +} diff --git a/scripts/algorithm-experiments/policy.ts b/scripts/algorithm-experiments/policy.ts new file mode 100644 index 000000000..fcb286271 --- /dev/null +++ b/scripts/algorithm-experiments/policy.ts @@ -0,0 +1,5 @@ +import { z } from 'zod' +const positive = z.number().int().positive().max(Number.MAX_SAFE_INTEGER) +export const experimentPolicySchema = z.object({ image: z.string().regex(/^sha256:[a-f0-9]{64}$/), maximumWallMs: positive.max(3600000), + maximumMemoryBytes: positive, maximumProcesses: positive, maximumOutputBytes: positive, maximumInputBytes: positive, +}).strict() diff --git a/scripts/algorithm-experiments/preflight-evidence.ts b/scripts/algorithm-experiments/preflight-evidence.ts new file mode 100644 index 000000000..e6a0e12cf --- /dev/null +++ b/scripts/algorithm-experiments/preflight-evidence.ts @@ -0,0 +1,103 @@ +import { z } from 'zod' +import { configHash, dataHash } from '../../src/biometrics/algorithms/canonical' +import { algorithmPackageManifestSchema } from '../../src/biometrics/packages/manifest' +import { PackageFrameValidator } from '../../src/biometrics/packages/protocol' +import { packageComparisonInputSchema } from './comparison' +import { assessFixtureReadings, fixtureCategories, packageFixtureSuiteSchema } from './fixtures' +import { packageInputHash } from './input' +import { packagePreflightPlanSchema } from './preflight' +import { assessPackageResources } from './resource-assessment' + +const hash = z.string().regex(/^[a-f0-9]{64}$/) +const execution = z.object({ image: z.string(), limits: z.unknown(), measurements: z.unknown(), wallMs: z.unknown() }).passthrough() +const result = packageComparisonInputSchema.shape.outputs.element.shape.result.extend({ execution }).passthrough() +const reportSchema = z.object({ schema: z.literal('biometric-package-preflight-report-v1'), packageHash: hash, manifest: algorithmPackageManifestSchema, + image: z.string(), plan: packagePreflightPlanSchema, planHash: hash, + fixtureReport: z.object({ schema: z.literal('biometric-package-fixture-report-v1'), packageHash: hash, suiteHash: hash, + suite: packageFixtureSuiteSchema, status: z.enum(['failed', 'incomplete', 'fixture_checks_passed']), missingCategories: z.array(z.string()), + results: z.array(z.object({ fixtureId: z.string(), category: z.string(), fixtureHash: hash, assessment: z.unknown(), + resourceAssessment: z.unknown(), execution: result.nullable(), error: z.string().nullable() }).strict()).min(1).max(64), + promotion: z.literal('requires_determinism_resource_holdout_compatibility_and_release_review'), + }).strict(), + determinism: z.array(z.object({ fixtureId: z.string(), resources: z.array(z.unknown()), error: z.string().nullable(), + report: z.object({ schema: z.literal('biometric-package-determinism-v1'), deterministic: z.boolean(), + runs: z.array(z.object({ packageHash: hash, runHash: hash, input: result.shape.frozenInput, outputHash: hash, + responseCount: z.number().int().positive(), execution }).strict()).min(2).max(5), + qualification: z.literal('requires_resource_noise_holdout_compatibility_checks_and_review'), + }).strict().nullable(), + }).strict()).min(1).max(64), + status: z.enum(['failed', 'incomplete', 'external_checks_passed']), + interpretation: z.literal('requires_heldout_accuracy_target_pod_compatibility_and_reviewed_release'), +}).strict() +function same(actual: unknown, expected: unknown) { + if (dataHash(actual) !== dataHash(expected)) throw new Error('Retained preflight integrity mismatch') +} +/** Recompute derived preflight status and verify fixture/run identities. Host + * receipts remain trusted execution records, not candidate attestations. */ +export function verifyPreflightEvidence(raw: unknown, expected: { plan: unknown, packageHash: string, image: string }) { + const report = reportSchema.parse(raw), suite = report.fixtureReport.suite + same(report.plan, expected.plan) + if (report.packageHash !== expected.packageHash || configHash(report.manifest) !== expected.packageHash || report.image !== expected.image + || report.planHash !== configHash(report.plan) || dataHash(suite) !== report.plan.suiteHash + || report.fixtureReport.suiteHash !== report.plan.suiteHash || report.fixtureReport.packageHash !== expected.packageHash) throw new Error('Retained preflight identity mismatch') + if (report.fixtureReport.results.length !== suite.fixtures.length || report.determinism.length !== suite.fixtures.length + || new Set(report.fixtureReport.results.map(row => row.fixtureId)).size !== suite.fixtures.length + || new Set(report.determinism.map(row => row.fixtureId)).size !== suite.fixtures.length) throw new Error('Retained preflight fixture set mismatch') + const checkExecution = (value: z.infer) => { + if (value.image !== expected.image) throw new Error('Retained preflight image mismatch') + same(value.limits, report.manifest.resources) + return assessPackageResources(value, suite.resourcePolicy) + } + const fixtures = suite.fixtures.map((fixture) => { + const row = report.fixtureReport.results.find(row => row.fixtureId === fixture.id) + const repeated = report.determinism.find(row => row.fixtureId === fixture.id) + if (!row || !repeated || row.fixtureHash !== dataHash(fixture) || row.category !== fixture.category + || fixture.request.packageHash !== expected.packageHash || fixture.request.inputHash !== packageInputHash(fixture.frames) + || configHash({ config: fixture.request.config, seed: fixture.request.seed }) !== report.plan.configHash + || fixture.request.hardwareId !== report.plan.hardwareId || fixture.request.sampleRateHz !== report.plan.sampleRateHz + || fixture.request.side !== report.plan.side) throw new Error('Retained preflight fixture identity mismatch') + const runHash = configHash(fixture.request) + if (row.execution) { + if (row.error !== null || row.execution.runHash !== runHash || row.execution.packageHash !== expected.packageHash + || row.execution.frozenInput.hash !== fixture.request.inputHash || row.execution.frozenInput.frameCount !== fixture.frames.length + || row.execution.results.length !== fixture.frames.length) throw new Error('Retained preflight run identity mismatch') + const validator = new PackageFrameValidator(report.manifest, fixture.request) + fixture.frames.forEach((frame, index) => validator.validateStep(frame, row.execution?.results[index])) + same(row.assessment, assessFixtureReadings(fixture.expectations, row.execution.results.flatMap(frame => frame.readings))) + same(row.resourceAssessment, checkExecution(row.execution.execution)) + } + else { + if (row.error === null) throw new Error('Missing preflight execution failure') + same(row.assessment, { passed: false, failures: ['execution_failed'], usableReadings: [] }) + same(row.resourceAssessment, null) + } + if (repeated.report) { + if (repeated.error !== null || repeated.report.runs.length !== report.plan.repetitions) throw new Error('Retained repetition count mismatch') + for (const run of repeated.report.runs) { + if (run.runHash !== runHash || run.packageHash !== expected.packageHash || run.input.hash !== fixture.request.inputHash + || run.input.frameCount !== fixture.frames.length || run.responseCount !== fixture.frames.length) throw new Error('Retained determinism identity mismatch') + } + same(repeated.resources, repeated.report.runs.map(run => checkExecution(run.execution))) + same(repeated.report.deterministic, repeated.report.runs.every(run => run.outputHash === repeated.report?.runs[0].outputHash)) + } + else { + if (repeated.error === null) throw new Error('Missing determinism execution failure') + same(repeated.resources, []) + } + return { fixturePassed: row.execution !== null && assessFixtureReadings(fixture.expectations, row.execution.results.flatMap(frame => frame.readings)).passed, + resource: row.execution ? checkExecution(row.execution.execution).status : null, + deterministic: repeated.report?.deterministic ?? false, + repeatResources: repeated.report?.runs.map(run => checkExecution(run.execution).status) ?? [] } + }) + const missingCategories = fixtureCategories.filter(category => !suite.fixtures.some(fixture => fixture.category === category)) + same(report.fixtureReport.missingCategories, missingCategories) + const fixtureStatus = fixtures.some(row => !row.fixturePassed || row.resource === 'failed') + ? 'failed' + : missingCategories.length || fixtures.some(row => row.resource !== 'within_observed_limits') ? 'incomplete' : 'fixture_checks_passed' + same(report.fixtureReport.status, fixtureStatus) + const status = fixtureStatus === 'failed' || fixtures.some(row => !row.deterministic || row.repeatResources.includes('failed')) + ? 'failed' + : fixtureStatus === 'incomplete' || fixtures.some(row => row.repeatResources.some(status => status !== 'within_observed_limits')) ? 'incomplete' : 'external_checks_passed' + same(report.status, status) + return { status, planHash: report.planHash, packageHash: report.packageHash, image: report.image } +} diff --git a/scripts/algorithm-experiments/preflight.ts b/scripts/algorithm-experiments/preflight.ts new file mode 100644 index 000000000..6509b5bfb --- /dev/null +++ b/scripts/algorithm-experiments/preflight.ts @@ -0,0 +1,52 @@ +import { z } from 'zod' +import { configHash, dataHash } from '../../src/biometrics/algorithms/canonical' +import { verifyAlgorithmPackage } from '../../src/biometrics/packages/manifest' +import { checkPackageDeterminism } from './determinism' +import { packageFixtureSuiteSchema, runPackageFixtures } from './fixtures' +import { assessPackageResources } from './resource-assessment' +import type { executePackageExperiment } from './container' + +export const packagePreflightPlanSchema = z.object({ schema: z.literal('biometric-package-preflight-plan-v1'), + suiteHash: z.string().regex(/^[a-f0-9]{64}$/), repetitions: z.number().int().min(2).max(5), + configHash: z.string().regex(/^[a-f0-9]{64}$/), hardwareId: z.string().min(1).max(100), + sampleRateHz: z.number().int().positive(), side: z.enum(['left', 'right']), +}).strict() + +/** Execute all evidence in one host-controlled flow. Every fixture is replayed + * independently for determinism, with the same candidate configuration/seed. + * This does not establish pod compatibility, holdout accuracy or installation. */ +export async function runPackagePreflight(options: Pick[0], 'manifest' | 'files' | 'policy' | 'signal'>, + rawPlan: unknown, rawSuite: unknown) { + const plan = packagePreflightPlanSchema.parse(structuredClone(rawPlan)) + const suite = packageFixtureSuiteSchema.parse(structuredClone(rawSuite)) + if (dataHash(suite) !== plan.suiteHash) throw new Error('Preflight suite identity mismatch') + const files = new Map([...options.files].map(([path, bytes]) => [path, Buffer.from(bytes)])) + const verified = verifyAlgorithmPackage(options.manifest, files) + const stable = { manifest: verified.manifest, files, policy: structuredClone(options.policy), signal: options.signal } + for (const fixture of suite.fixtures) { + if (fixture.request.packageHash !== verified.packageHash + || configHash({ config: fixture.request.config, seed: fixture.request.seed }) !== plan.configHash + || fixture.request.hardwareId !== plan.hardwareId || fixture.request.sampleRateHz !== plan.sampleRateHz + || fixture.request.side !== plan.side) throw new Error('Preflight fixture configuration mismatch') + } + options.signal?.throwIfAborted() + const fixtureReport = await runPackageFixtures(stable, suite) + const determinism = [] + for (const fixture of suite.fixtures) { + options.signal?.throwIfAborted() + try { + const report = await checkPackageDeterminism({ ...stable, request: fixture.request, frames: () => fixture.frames }, plan.repetitions) + determinism.push({ fixtureId: fixture.id, report, resources: report.runs.map(run => assessPackageResources(run.execution, suite.resourcePolicy)), error: null }) + } + catch (error) { + options.signal?.throwIfAborted() + determinism.push({ fixtureId: fixture.id, report: null, resources: [], error: error instanceof Error ? error.message.slice(0, 1000) : 'Determinism execution failed' }) + } + } + const failed = fixtureReport.status === 'failed' || determinism.some(row => !row.report?.deterministic || row.resources.some(resource => resource.status === 'failed')) + const incomplete = fixtureReport.status === 'incomplete' || determinism.some(row => row.resources.some(resource => resource.status !== 'within_observed_limits')) + return { schema: 'biometric-package-preflight-report-v1' as const, packageHash: verified.packageHash, manifest: verified.manifest, image: stable.policy.image, + plan, planHash: configHash(plan), fixtureReport, determinism, + status: failed ? 'failed' as const : incomplete ? 'incomplete' as const : 'external_checks_passed' as const, + interpretation: 'requires_heldout_accuracy_target_pod_compatibility_and_reviewed_release' as const } +} diff --git a/scripts/algorithm-experiments/qualification.test.ts b/scripts/algorithm-experiments/qualification.test.ts new file mode 100644 index 000000000..4d758c8d3 --- /dev/null +++ b/scripts/algorithm-experiments/qualification.test.ts @@ -0,0 +1,71 @@ +// @vitest-environment node +import { describe, expect, it } from 'vitest' +import { configHash } from '../../src/biometrics/algorithms/canonical' +import { combinePackageQualification } from './qualification' +import { fixtureResourceTestPolicy } from './resource-test-policy' +import { managedProtocolSchema } from './managed-protocol' + +type Evidence = Parameters[0] +type Accuracy = Parameters[1] +function fixture() { + const protocol = managedProtocolSchema.parse({ schema: 'biometric-managed-protocol-v1', hardwareId: 'pod4', sampleRateHz: 500, + metricId: 'heart_rate', minimumQuality: 0.8, resourcePolicy: fixtureResourceTestPolicy, + alignment: { version: 'reference-alignment-v1', pointToleranceMs: 0, clockOffsetMs: 0, offsetSource: 'fixed' }, + uncertainty: { version: 'paired-night-percentile-v1', seed: 'test', resamples: 200, minimumNights: 3 } }) + const accuracy: Accuracy = { schema: 'biometric-package-accuracy-assessment-v1', status: 'criteria_met', policyHash: 'a'.repeat(64), failures: [], missing: [], + interpretation: 'reference_agreement_criteria_only_requires_full_package_qualification_and_review' } + // The combiner consumes only the already-verified resource rows for each + // night; comparisons themselves are evaluated by the accuracy assessor. + const resources = (['default', 'current', 'candidate'] as const).map(label => ({ label, + assessment: { status: 'within_observed_limits' as const, policyHash: configHash(protocol.resourcePolicy), failures: [], missing: [] } })) + return { protocol, accuracy, resources } +} +function input(): { evidence: Evidence, accuracy: Accuracy } { + const { protocol, accuracy, resources } = fixture() + // Keep this input's type limited to exactly what combination reads. + const evidence: Evidence = { protocol, preflight: { status: 'external_checks_passed', planHash: 'a'.repeat(64), packageHash: 'b'.repeat(64), image: 'sha256:' + 'c'.repeat(64) }, + nights: [{ nightId: 'night', resources }] } + return { evidence, accuracy } +} +describe('combined external package qualification', () => { + it('combines passing dimensions but never grants installation', () => { + const { evidence, accuracy } = input() + expect(combinePackageQualification(evidence, accuracy)).toMatchObject({ status: 'external_criteria_met', failures: [], missing: [], + installationAuthorized: false, remainingReleaseRequirements: expect.arrayContaining(['target_pod_compatibility_and_resource_qualification', 'reviewed_shipped_registry_release']) }) + }) + it('blocks missing or incomplete preflight even with passing accuracy', () => { + const { evidence, accuracy } = input() + evidence.preflight = null + expect(combinePackageQualification(evidence, accuracy).missing).toContain('no_bound_preflight') + evidence.preflight = { status: 'incomplete', planHash: 'a', packageHash: 'b', image: 'c' } + expect(combinePackageQualification(evidence, accuracy).status).toBe('insufficient_evidence') + }) + it('keeps failures dominant while retaining missing-evidence reasons', () => { + const { evidence, accuracy } = input() + accuracy.status = 'insufficient_evidence' + if (evidence.preflight) evidence.preflight.status = 'failed' + const result = combinePackageQualification(evidence, accuracy) + expect(result).toMatchObject({ status: 'failed', failures: ['preflight_failed'], missing: ['accuracy_evidence_incomplete'] }) + accuracy.status = 'criteria_not_met' + expect(combinePackageQualification(evidence, accuracy).failures).toContain('accuracy_criteria_not_met') + }) + it('requires exact complete resource evidence for every baseline and candidate', () => { + const { evidence, accuracy } = input() + evidence.nights[0].resources.pop() + expect(combinePackageQualification(evidence, accuracy).missing).toContain('night:candidate:resource_evidence_missing_or_ambiguous') + evidence.nights[0].resources.push(evidence.nights[0].resources[0]) + expect(combinePackageQualification(evidence, accuracy).missing).toContain('night:default:resource_evidence_missing_or_ambiguous') + evidence.nights = [] + expect(combinePackageQualification(evidence, accuracy).missing).toContain('no_night_resource_evidence') + }) + it('rejects budget violations, inadequate sampling and substituted policies', () => { + const { evidence, accuracy } = input() + const candidate = evidence.nights[0].resources[2].assessment + candidate.status = 'failed' + candidate.policyHash = 'f'.repeat(64) + const result = combinePackageQualification(evidence, accuracy) + expect(result.failures).toEqual(expect.arrayContaining(['night:candidate:resource_policy_mismatch', 'night:candidate:resource_budget_failed'])) + candidate.status = 'insufficient_evidence' + expect(combinePackageQualification(evidence, accuracy).missing).toContain('night:candidate:resource_evidence_incomplete') + }) +}) diff --git a/scripts/algorithm-experiments/qualification.ts b/scripts/algorithm-experiments/qualification.ts new file mode 100644 index 000000000..4fa69d8f6 --- /dev/null +++ b/scripts/algorithm-experiments/qualification.ts @@ -0,0 +1,52 @@ +import { configHash, dataHash } from '../../src/biometrics/algorithms/canonical' +import { assessPackageAccuracy } from './accuracy' +import { readManagedEvidence } from './managed-evidence' + +type Evidence = ReturnType +type Accuracy = ReturnType + +/** Combine verified external checks. This is deliberately not a registry entry + * or activation authorization: target-pod and release review are still needed. */ +export function combinePackageQualification(evidence: Pick & { nights: Pick[] }, accuracy: Accuracy) { + const failures: string[] = [], missing: string[] = [] + if (accuracy.status === 'criteria_not_met') failures.push('accuracy_criteria_not_met') + if (accuracy.status === 'insufficient_evidence' || accuracy.missing.length) missing.push('accuracy_evidence_incomplete') + if (!evidence.preflight) missing.push('no_bound_preflight') + else if (evidence.preflight.status === 'failed') failures.push('preflight_failed') + else if (evidence.preflight.status !== 'external_checks_passed') missing.push('preflight_incomplete') + const resourcePolicyHash = configHash(evidence.protocol.resourcePolicy) + if (!evidence.nights.length) missing.push('no_night_resource_evidence') + const resources = evidence.nights.map((night) => { + for (const label of ['default', 'current', 'candidate'] as const) { + const selected = night.resources.filter(row => row.label === label) + if (selected.length !== 1) { + missing.push(`${night.nightId}:${label}:resource_evidence_missing_or_ambiguous`) + continue + } + const assessment = selected[0].assessment + if (assessment.policyHash !== resourcePolicyHash) failures.push(`${night.nightId}:${label}:resource_policy_mismatch`) + if (assessment.status === 'failed') failures.push(`${night.nightId}:${label}:resource_budget_failed`) + if (assessment.status === 'insufficient_evidence' || ('missing' in assessment && assessment.missing?.length)) missing.push(`${night.nightId}:${label}:resource_evidence_incomplete`) + } + return { nightId: night.nightId, assessments: night.resources } + }) + return { schema: 'biometric-package-qualification-assessment-v1' as const, + status: failures.length ? 'failed' as const : missing.length ? 'insufficient_evidence' as const : 'external_criteria_met' as const, + failures, missing, accuracy, preflight: evidence.preflight, resourcePolicyHash, resources, + installationAuthorized: false as const, + remainingReleaseRequirements: ['representative_fixture_review', 'target_pod_compatibility_and_resource_qualification', + 'prospective_health_and_reference_comparison', 'reviewed_shipped_registry_release'] as const, + interpretation: 'external_evidence_snapshot_not_installation_or_activation_authorization' as const } +} + +/** The public orchestration entry point accepts ledger identities, never a + * client-authored success report. Re-read for every review: later overlapping + * holdout access changes the evidence and therefore the assessment hash. */ +export function assessManagedQualification(...args: Parameters) { + const evidence = readManagedEvidence(...args) + const accuracy = assessPackageAccuracy(evidence) + const qualification = combinePackageQualification(evidence, accuracy) + const snapshot = { trialId: evidence.trialId, reportHash: evidence.reportHash, protocolHash: configHash(evidence.protocol), + request: evidence.request, currentEvidence: evidence.currentEvidence, qualification } + return { ...snapshot, assessmentHash: dataHash(snapshot) } +} diff --git a/scripts/algorithm-experiments/recovery.test.ts b/scripts/algorithm-experiments/recovery.test.ts new file mode 100644 index 000000000..fecb9f178 --- /dev/null +++ b/scripts/algorithm-experiments/recovery.test.ts @@ -0,0 +1,70 @@ +// @vitest-environment node +import { describe, expect, it } from 'vitest' +import { expiredOwnedContainer, recoveryLabels } from './recovery' + +const now = 1000000 +const id = 'a'.repeat(64) +const valid = (kind: 'experiment' | 'environment' = 'experiment') => ({ Id: id, + Name: `/sleepypod-${kind}-12345678-1234-1234-1234-123456789abc`, + Config: { Labels: recoveryLabels(kind, 1000, now) } }) + +describe('expired experiment recovery ownership', () => { + it('preserves live controllers through their deadline and cleanup grace', () => { + expect(expiredOwnedContainer(valid(), now + 120999)).toBeNull() + expect(expiredOwnedContainer(valid(), now + 121000)).toBe(id) + expect(expiredOwnedContainer(valid('environment'), now + 121000)).toBe(id) + }) + it('requires exact version, host owner, kind and immutable full ID', () => { + const container = valid() + for (const [field, value] of [['version', '2'], ['owner', 'foreign'], ['kind', 'environment'], ['expires', 'NaN'], ['expires', '0'], ['expires', '9999999999999999']]) { + expect(expiredOwnedContainer({ ...container, Config: { Labels: { ...container.Config.Labels, + [`org.sleepypod.experiment.${field}`]: value } } }, now + 121000)).toBeNull() + } + for (const value of [null, {}, { ...container, Id: 'short' }, { ...container, Name: '/unrelated' }, + { ...container, Config: {} }, { ...container, Name: '/sleepypod-experiment-not-a-uuid' }]) { + expect(expiredOwnedContainer(value, now + 121000)).toBeNull() + } + }) + it('rejects invalid clock and budget inputs', () => { + for (const value of [-1, 0, NaN, Infinity, 3600001]) expect(() => recoveryLabels('experiment', value)).toThrow() + expect(() => recoveryLabels('experiment', 1000, -1)).toThrow() + expect(expiredOwnedContainer(valid(), NaN)).toBeNull() + }) +}) + +// Actual daemon test: an abandoned process remains alive until recovery, while +// an unexpired controller and an unlabelled lookalike survive the same sweep. +it.skipIf(!process.env.BIOMETRIC_EXPERIMENT_TEST_IMAGE)('removes an overdue running container and preserves other containers', async () => { + const { execFile } = await import('node:child_process') + const { promisify } = await import('node:util') + const { randomUUID } = await import('node:crypto') + const { recoverExpiredContainers } = await import('./recovery') + const exec = promisify(execFile) + const containers: string[] = [] + const clock = Date.now() + const image = process.env.BIOMETRIC_EXPERIMENT_TEST_IMAGE + if (!image) throw new Error('Test image required') + try { + for (const labels of [recoveryLabels('experiment', 1000, clock - 122000), recoveryLabels('experiment', 1000, clock), {}]) { + const result = await exec('docker', ['run', '-d', '--name', `sleepypod-experiment-${randomUUID()}`, + ...Object.entries(labels).flatMap(([key, value]) => ['--label', `${key}=${value}`]), + '--network', 'none', '--read-only', '--cap-drop', 'ALL', '--security-opt', 'no-new-privileges=true', + '--user', '65534:65534', '--pids-limit', '16', '--memory', '134217728', '--cpus', '1', + '--entrypoint', 'python', image, '-I', '-B', '-c', 'import time; time.sleep(120)'], + { timeout: 10000 }) + containers.push(result.stdout.trim()) + } + const report = await recoverExpiredContainers(clock) + expect(report.removed).toContain(containers[0]) + expect(report.removed).not.toContain(containers[1]) + expect(report.removed).not.toContain(containers[2]) + await expect(exec('docker', ['inspect', containers[0]])).rejects.toThrow() + for (const container of containers.slice(1)) { + expect((await exec('docker', ['inspect', '--format', '{{.State.Running}}', container])).stdout.trim()).toBe('true') + } + expect((await recoverExpiredContainers(clock)).removed).not.toContain(containers[0]) + } + finally { + for (const container of containers) await exec('docker', ['rm', '-f', container]).catch(() => undefined) + } +}, 60000) diff --git a/scripts/algorithm-experiments/recovery.ts b/scripts/algorithm-experiments/recovery.ts new file mode 100644 index 000000000..f1758ef33 --- /dev/null +++ b/scripts/algorithm-experiments/recovery.ts @@ -0,0 +1,68 @@ +import { execFile } from 'node:child_process' +import { promisify } from 'node:util' + +const exec = promisify(execFile) +const prefix = 'org.sleepypod.experiment' +const owner = `${process.getuid?.() ?? 'unknown'}` +const uuid = '[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}' +const namePattern = new RegExp(`^/sleepypod-(experiment|environment)-${uuid}$`) + +/** Immutable Docker labels survive a terminated controller. Leave two minutes + * beyond the hard deadline for startup, measurement and ordinary cleanup. */ +export function recoveryLabels(kind: 'experiment' | 'environment', maximumWallMs: number, now = Date.now()) { + if (!Number.isSafeInteger(maximumWallMs) || maximumWallMs <= 0 || maximumWallMs > 3600000 + || !Number.isSafeInteger(now) || now < 0 + || !Number.isSafeInteger(now + maximumWallMs + 120000)) throw new Error('Invalid recovery deadline') + return { [`${prefix}.version`]: '1', [`${prefix}.owner`]: owner, + [`${prefix}.kind`]: kind, [`${prefix}.expires`]: String(now + maximumWallMs + 120000) } +} +export function recoveryLabelArguments(kind: 'experiment' | 'environment', maximumWallMs: number) { + return Object.entries(recoveryLabels(kind, maximumWallMs)).flatMap(([key, value]) => ['--label', `${key}=${value}`]) +} + +/** Fail closed on malformed metadata; names alone never establish ownership. */ +export function expiredOwnedContainer(raw: unknown, now: number): string | null { + if (!Number.isSafeInteger(now) || now < 0 || !raw || typeof raw !== 'object') return null + const value = raw as { Id?: unknown, Name?: unknown, Config?: { Labels?: Record } } + if (typeof value.Id !== 'string' || !/^[a-f0-9]{64}$/.test(value.Id) + || typeof value.Name !== 'string' || !namePattern.test(value.Name)) return null + const labels = value.Config?.Labels + if (!labels || labels[`${prefix}.version`] !== '1' || labels[`${prefix}.owner`] !== owner) return null + const kind = value.Name.startsWith('/sleepypod-experiment-') ? 'experiment' : 'environment' + if (labels[`${prefix}.kind`] !== kind) return null + const expires = labels[`${prefix}.expires`] + if (typeof expires !== 'string' || !/^[1-9][0-9]{0,15}$/.test(expires) + || !Number.isSafeInteger(Number(expires)) || Number(expires) > now) return null + return value.Id +} + +/** Operates only on the selected Docker daemon and the invoking host UID. + * Remove by inspected immutable ID, never by a reusable container name. */ +export async function recoverExpiredContainers(now = Date.now()) { + if (!Number.isSafeInteger(now) || now < 0) throw new Error('Invalid recovery time') + const { stdout } = await exec('docker', ['ps', '-aq', '--no-trunc', '--filter', `label=${prefix}.version=1`, + '--filter', `label=${prefix}.owner=${owner}`], { timeout: 10000, maxBuffer: 1024 * 1024 }) + const ids = stdout.trim() ? stdout.trim().split(/\s+/) : [] + if (ids.length > 4096 || ids.some(id => !/^[a-f0-9]{64}$/.test(id))) throw new Error('Invalid recovery inventory') + const removed: string[] = [] + for (const id of ids) { + let inspection + try { + inspection = await exec('docker', ['inspect', '--type', 'container', id], { timeout: 10000, maxBuffer: 1024 * 1024 }) + } + catch (error) { + if (error instanceof Error && /No such (object|container):/.test(error.message)) continue + throw error + } + const values: unknown = JSON.parse(inspection.stdout) + if (!Array.isArray(values) || values.length !== 1 || expiredOwnedContainer(values[0], now) !== id) continue + try { + await exec('docker', ['rm', '-f', id], { timeout: 10000, maxBuffer: 65536 }) + removed.push(id) + } + catch (error) { + if (!(error instanceof Error) || !error.message.includes('No such container:')) throw error + } + } + return { schema: 'biometric-container-recovery-v1' as const, checkedAt: now, inspected: ids.length, removed } +} diff --git a/scripts/algorithm-experiments/resource-assessment.test.ts b/scripts/algorithm-experiments/resource-assessment.test.ts new file mode 100644 index 000000000..4d29364e6 --- /dev/null +++ b/scripts/algorithm-experiments/resource-assessment.test.ts @@ -0,0 +1,39 @@ +import { fixtureResourceTestPolicy } from './resource-test-policy' +import { describe, expect, it } from 'vitest' +import { assessPackageResources } from './resource-assessment' +import { ResourceObservations } from './resources' +function fixture() { + const observations = new ResourceObservations(0) + observations.record({ sourceTimestampMs: 1000, cpuMs: 10, memoryBytes: 1000000, processes: 1 }, 100) + return { measurements: observations.snapshot(200), limits: { cpuMs: 1000, wallMs: 1000, memoryBytes: 2000000, processes: 1, outputBytes: 10000 }, wallMs: 200 } +} +describe('package fixture resource assessment', () => { + it('binds explicit policy and accepts complete measurements within enforced limits', () => { + const result = assessPackageResources(fixture(), fixtureResourceTestPolicy) + expect(result.status).toBe('within_observed_limits') + expect(assessPackageResources(fixture(), { ...fixtureResourceTestPolicy, maximumCpuMs: 6000 }).policyHash).not.toBe(result.policyHash) + }) + it('does not turn missing, stale or inconsistent evidence into passing evidence', () => { + const input = fixture() + for (const measurements of [null, { ...input.measurements, attempts: 2, allAttemptsMeasured: false }, + { ...input.measurements, samples: 2 }, { ...input.measurements, source: 'candidate_claim' }]) { + expect(assessPackageResources({ ...input, measurements }, fixtureResourceTestPolicy).status).toBe('insufficient_evidence') + } + expect(assessPackageResources(input, { ...fixtureResourceTestPolicy, minimumSamples: 2 }).status).toBe('insufficient_evidence') + expect(assessPackageResources(input, { ...fixtureResourceTestPolicy, maximumObservationGapMs: 50 }).status).toBe('insufficient_evidence') + }) + it('fails measured overruns and limits broader than the reviewed policy', () => { + const input = fixture() + for (const changed of [{ ...input, wallMs: 6000 }, { ...input, measurements: { ...input.measurements, maximumObservedCpuMs: 6000 } }, + { ...input, measurements: { ...input.measurements, peakObservedMemoryBytes: 100000000 } }, + { ...input, measurements: { ...input.measurements, maximumObservedProcesses: 2 } }, + { ...input, limits: { ...input.limits, memoryBytes: 100000000 } }]) { + expect(assessPackageResources(changed, fixtureResourceTestPolicy).status).toBe('failed') + } + }) + it('retains a known budget failure even if observation coverage is incomplete', () => { + const input = fixture() + const result = assessPackageResources({ ...input, wallMs: 6000, measurements: { ...input.measurements, attempts: 2, allAttemptsMeasured: false } }, fixtureResourceTestPolicy) + expect(result).toMatchObject({ status: 'failed', failures: ['wall_budget_exceeded'], missing: ['unmeasured_attempts'] }) + }) +}) diff --git a/scripts/algorithm-experiments/resource-assessment.ts b/scripts/algorithm-experiments/resource-assessment.ts new file mode 100644 index 000000000..49d2c67e9 --- /dev/null +++ b/scripts/algorithm-experiments/resource-assessment.ts @@ -0,0 +1,48 @@ +import { z } from 'zod' +import { configHash } from '../../src/biometrics/algorithms/canonical' + +const positive = z.number().finite().positive() +const count = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) +const resourceLimitsSchema = z.object({ cpuMs: positive, wallMs: positive, memoryBytes: positive.int(), processes: positive.int(), outputBytes: positive.int() }).strict() +export const packageResourcePolicySchema = z.object({ + schema: z.literal('biometric-package-resource-policy-v1'), + maximumWallMs: positive, maximumCpuMs: positive, maximumMemoryBytes: positive.int(), maximumProcesses: positive.int(), + minimumSamples: positive.int(), maximumObservationGapMs: positive, + evidenceMode: z.literal('sampled_counters_and_enforced_limits'), +}).strict() +const measurementSchema = z.object({ + schema: z.literal('biometric-package-resources-v1'), source: z.literal('host_docker_engine_stats'), + attempts: count, samples: count, allAttemptsMeasured: z.boolean(), + maximumObservedCpuMs: z.number().finite().nonnegative().nullable(), peakObservedMemoryBytes: positive.nullable(), maximumObservedProcesses: positive.int().nullable(), + maximumObservationGapMs: z.number().finite().nonnegative(), elapsedMs: z.number().finite().nonnegative(), + exactLifetimeCpu: z.literal(false), exactPeakMemory: z.literal(false), memoryIncludesCache: z.literal(true), +}).strict().refine(value => value.samples <= value.attempts + && value.allAttemptsMeasured === (value.attempts > 0 && value.samples === value.attempts) + && value.maximumObservationGapMs <= value.elapsedMs + && (value.samples === 0 + ? value.maximumObservedCpuMs === null && value.peakObservedMemoryBytes === null && value.maximumObservedProcesses === null + : value.maximumObservedCpuMs !== null && value.peakObservedMemoryBytes !== null && value.maximumObservedProcesses !== null), 'Inconsistent resource observations') + +/** Assesses actual host evidence against explicit reviewed fixture criteria. + * No default thresholds or implicit target-pod fitness claim. */ +export function assessPackageResources(input: { measurements: unknown, limits: unknown, wallMs: unknown }, policyInput: unknown) { + const policy = packageResourcePolicySchema.parse(policyInput) + const policyHash = configHash(policy) + const measurements = measurementSchema.safeParse(input.measurements) + const limits = resourceLimitsSchema.safeParse(input.limits) + const wall = z.number().finite().nonnegative().safeParse(input.wallMs) + if (!measurements.success || !limits.success || !wall.success) return { status: 'insufficient_evidence' as const, policyHash, failures: ['invalid_or_missing_resource_evidence'] } + const evidence = measurements.data + const failures: string[] = [], missing: string[] = [] + if (!evidence.allAttemptsMeasured) missing.push('unmeasured_attempts') + if (evidence.samples < policy.minimumSamples) missing.push('insufficient_samples') + if (evidence.maximumObservationGapMs > policy.maximumObservationGapMs) missing.push('observation_gap') + if (wall.data > policy.maximumWallMs) failures.push('wall_budget_exceeded') + if (evidence.maximumObservedCpuMs !== null && evidence.maximumObservedCpuMs > policy.maximumCpuMs) failures.push('cpu_budget_exceeded') + if (evidence.peakObservedMemoryBytes !== null && evidence.peakObservedMemoryBytes > policy.maximumMemoryBytes) failures.push('memory_budget_exceeded') + if (evidence.maximumObservedProcesses !== null && evidence.maximumObservedProcesses > policy.maximumProcesses) failures.push('process_budget_exceeded') + if (limits.data.cpuMs > policy.maximumCpuMs || limits.data.wallMs > policy.maximumWallMs + || limits.data.memoryBytes > policy.maximumMemoryBytes || limits.data.processes > policy.maximumProcesses) failures.push('enforced_limits_exceed_policy') + return { status: failures.length ? 'failed' as const : missing.length ? 'insufficient_evidence' as const : 'within_observed_limits' as const, + policyHash, failures, missing } +} diff --git a/scripts/algorithm-experiments/resource-test-policy.ts b/scripts/algorithm-experiments/resource-test-policy.ts new file mode 100644 index 000000000..de77cec40 --- /dev/null +++ b/scripts/algorithm-experiments/resource-test-policy.ts @@ -0,0 +1,3 @@ +// Synthetic test criteria only; not target-pod qualification. +export const fixtureResourceTestPolicy = { schema: 'biometric-package-resource-policy-v1', maximumWallMs: 5000, maximumCpuMs: 5000, + maximumMemoryBytes: 64 * 1024 * 1024, maximumProcesses: 1, minimumSamples: 1, maximumObservationGapMs: 5000, evidenceMode: 'sampled_counters_and_enforced_limits' } diff --git a/scripts/algorithm-experiments/resources.test.ts b/scripts/algorithm-experiments/resources.test.ts new file mode 100644 index 000000000..1266363e3 --- /dev/null +++ b/scripts/algorithm-experiments/resources.test.ts @@ -0,0 +1,46 @@ +// @vitest-environment node +import { describe, expect, it } from 'vitest' +import { ResourceObservations, startResourceMonitor } from './resources' +const sample = { sourceTimestampMs: 1000, cpuMs: 10, memoryBytes: 1000000, processes: 1 } +describe('host experiment resource evidence', () => { + it('retains maxima and includes initial, interior and final observation gaps', () => { + const readings = new ResourceObservations(0) + readings.record(sample, 100) + readings.record({ ...sample, sourceTimestampMs: 2000, cpuMs: 20, memoryBytes: 900000, processes: 2 }, 400) + expect(readings.snapshot(1000)).toMatchObject({ attempts: 2, samples: 2, allAttemptsMeasured: true, + maximumObservedCpuMs: 20, peakObservedMemoryBytes: 1000000, maximumObservedProcesses: 2, + maximumObservationGapMs: 600, exactLifetimeCpu: false, exactPeakMemory: false }) + }) + it('marks failed, stale, regressing and invalid observations incomplete', () => { + const readings = new ResourceObservations(0) + readings.record(sample, 100) + readings.record(null, 200) + readings.record(sample, 300) + readings.record({ ...sample, sourceTimestampMs: 2000, cpuMs: 1 }, 400) + readings.record({ ...sample, sourceTimestampMs: 2000, memoryBytes: NaN }, 500) + expect(readings.snapshot(1000)).toMatchObject({ attempts: 5, samples: 1, allAttemptsMeasured: false, maximumObservationGapMs: 900 }) + expect(new ResourceObservations(0).snapshot(1000)).toMatchObject({ samples: 0, allAttemptsMeasured: false, + maximumObservedCpuMs: null, peakObservedMemoryBytes: null }) + }) + it('coalesces concurrent reads and waits for a pending observation at stop', async () => { + let resolve: (value: typeof sample) => void = () => undefined + let calls = 0 + const monitor = startResourceMonitor(() => { + calls++ + return new Promise((done) => { + resolve = done + }) + }, 0) + const first = monitor.sample(), second = monitor.sample() + expect(first).toBe(second) + const stopping = monitor.stop() + resolve(sample) + expect(await stopping).toMatchObject({ attempts: 1, samples: 1 }) + expect(calls).toBe(1) + }) + it('reports an unavailable transport rather than fabricated zero usage', async () => { + const monitor = startResourceMonitor(null, 0) + await monitor.sample() + expect(await monitor.stop()).toMatchObject({ attempts: 1, samples: 0, allAttemptsMeasured: false, maximumObservedCpuMs: null }) + }) +}) diff --git a/scripts/algorithm-experiments/resources.ts b/scripts/algorithm-experiments/resources.ts new file mode 100644 index 000000000..9413fc19f --- /dev/null +++ b/scripts/algorithm-experiments/resources.ts @@ -0,0 +1,121 @@ +import { execFile } from 'node:child_process' +import { get } from 'node:http' +import { promisify } from 'node:util' +import { performance } from 'node:perf_hooks' +import { z } from 'zod' + +const exec = promisify(execFile) +const counter = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) +const statsSchema = z.object({ read: z.string(), id: z.string().regex(/^[a-f0-9]{64}$/), + cpu_stats: z.object({ cpu_usage: z.object({ total_usage: counter }) }), + memory_stats: z.object({ usage: counter.positive() }), pids_stats: z.object({ current: counter.positive() }), +}) + +function dockerJson(socketPath: string, path: string): Promise { + return new Promise((resolve, reject) => { + const request = get({ socketPath, path }, (response) => { + if (response.statusCode !== 200) { + response.resume() + reject(new Error('Docker measurement unavailable')) + return + } + const chunks: Buffer[] = [] + let length = 0 + response.on('data', (chunk: Buffer) => { + length += chunk.length + if (length > 128 * 1024) request.destroy(new Error('Docker measurement byte limit')) + else chunks.push(chunk) + }) + response.on('error', reject) + response.on('end', () => { + try { + resolve(JSON.parse(Buffer.concat(chunks).toString('utf8'))) + } + catch (error) { reject(error) } + }) + }) + const timeout = setTimeout(() => request.destroy(new Error('Docker measurement timeout')), 1500) + request.on('error', reject) + request.on('close', () => clearTimeout(timeout)) + }) +} + +/** Independent host evidence. Never accepts counters emitted by candidate code. + * Unsupported remote Docker transports remain explicitly unmeasured. */ +export async function dockerResourceReader(containerId: string) { + if (!/^[a-f0-9]{64}$/.test(containerId)) throw new Error('Invalid measured container identity') + const host = !process.env.DOCKER_CONTEXT && process.env.DOCKER_HOST + ? process.env.DOCKER_HOST + : JSON.parse((await exec('docker', ['context', 'inspect', '--format', '{{json .Endpoints.docker.Host}}'], { timeout: 5000, maxBuffer: 16384 })).stdout) as string + if (!host.startsWith('unix:///')) throw new Error('Resource measurement requires a local Unix Docker endpoint') + const socket = host.slice('unix://'.length) + const version = z.object({ ApiVersion: z.string().regex(/^1\.\d+$/) }).parse(await dockerJson(socket, '/version')).ApiVersion + if (Number(version.split('.')[1]) < 41) throw new Error('Docker API does not support one-shot resource measurements') + return async () => { + const value = statsSchema.parse(await dockerJson(socket, `/v${version}/containers/${containerId}/stats?stream=false&one-shot=true`)) + if (value.id !== containerId) throw new Error('Measured container identity mismatch') + const sourceTimestampMs = Date.parse(value.read) + if (!Number.isFinite(sourceTimestampMs) || sourceTimestampMs <= 0) throw new Error('Invalid resource observation timestamp') + return { sourceTimestampMs, cpuMs: value.cpu_stats.cpu_usage.total_usage / 1e6, memoryBytes: value.memory_stats.usage, processes: value.pids_stats.current } + } +} + +interface Observation { sourceTimestampMs: number, cpuMs: number, memoryBytes: number, processes: number } +export class ResourceObservations { + private attempts = 0 + private samples = 0 + private peakMemoryBytes: number | null = null + private maximumCpuMs: number | null = null + private maximumProcesses: number | null = null + private sourceTimestampMs = 0 + private lastAt: number + private maximumGapMs = 0 + constructor(private readonly startedAt: number) { this.lastAt = startedAt } + record(value: Observation | null, at: number) { + this.attempts++ + if (!value || ![value.cpuMs, value.memoryBytes, value.processes].every(number => Number.isFinite(number) && number >= 0)) return + if (!Number.isFinite(value.sourceTimestampMs) || value.sourceTimestampMs <= this.sourceTimestampMs || !Number.isFinite(at) || at < this.lastAt) return + if (this.maximumCpuMs !== null && value.cpuMs < this.maximumCpuMs) return + this.sourceTimestampMs = value.sourceTimestampMs + this.samples++ + this.maximumGapMs = Math.max(this.maximumGapMs, at - this.lastAt) + this.lastAt = at + this.maximumCpuMs = Math.max(this.maximumCpuMs ?? 0, value.cpuMs) + this.peakMemoryBytes = Math.max(this.peakMemoryBytes ?? 0, value.memoryBytes) + this.maximumProcesses = Math.max(this.maximumProcesses ?? 0, value.processes) + } + + snapshot(endedAt: number) { + return { schema: 'biometric-package-resources-v1' as const, source: 'host_docker_engine_stats' as const, + attempts: this.attempts, samples: this.samples, allAttemptsMeasured: this.attempts > 0 && this.samples === this.attempts, + maximumObservedCpuMs: this.maximumCpuMs, peakObservedMemoryBytes: this.peakMemoryBytes, maximumObservedProcesses: this.maximumProcesses, + maximumObservationGapMs: Math.max(this.maximumGapMs, endedAt - this.lastAt), elapsedMs: endedAt - this.startedAt, + exactLifetimeCpu: false, exactPeakMemory: false, memoryIncludesCache: true } + } +} + +export function startResourceMonitor(read: (() => Promise) | null, startedAt: number) { + const observations = new ResourceObservations(startedAt) + let pending: Promise | null = null + let stopped = false + const sample = () => { + if (stopped) return Promise.resolve() + if (pending) return pending + pending = (async () => { + try { + observations.record(read ? await read() : null, performance.now()) + } + catch { observations.record(null, performance.now()) } + })().finally(() => { pending = null }) + return pending + } + const timer = setInterval(() => { + if (!stopped) void sample() + }, 250) + return { sample, stop: async () => { + stopped = true + clearInterval(timer) + if (pending) await pending + return observations.snapshot(performance.now()) + } } +} diff --git a/scripts/algorithm-experiments/staging-recovery.test.ts b/scripts/algorithm-experiments/staging-recovery.test.ts new file mode 100644 index 000000000..a2a6ef113 --- /dev/null +++ b/scripts/algorithm-experiments/staging-recovery.test.ts @@ -0,0 +1,66 @@ +// @vitest-environment node +import { mkdtemp, readFile, writeFile, rm, stat, symlink } from 'node:fs/promises' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import { recordExperimentStaging, recoverExperimentStaging } from './staging-recovery' +const docker = vi.hoisted(() => ({ mode: 'absent', calls: 0 })) +vi.mock('node:child_process', () => ({ execFile: (_file: string, _args: string[], _options: unknown, callback: (error: Error | null, stdout: string, stderr: string) => void) => { + docker.calls++ + callback(docker.mode === 'absent' ? new Error('No such container: test') : docker.mode === 'offline' ? new Error('Docker unavailable') : null, '[]', '') +} })) +let base: string +const deadPid = 2147483647 +beforeEach(async () => { + base = await mkdtemp(join(tmpdir(), 'staging-recovery-test-')) + docker.mode = 'absent' + docker.calls = 0 + vi.spyOn(process, 'kill').mockImplementation((pid) => { + if (pid === deadPid) throw Object.assign(new Error('gone'), { code: 'ESRCH' }) + return true + }) +}) +afterEach(async () => { + vi.restoreAllMocks() + await rm(base, { recursive: true, force: true }) +}) +async function fixture(changes: Record = {}) { + const root = await mkdtemp(join(base, 'sleepypod-package-')) + await recordExperimentStaging(root, 'sleepypod-experiment-12345678-1234-1234-1234-123456789abc', 1000, 1000) + const path = join(root, '.sleepypod-staging.json') + const receipt = JSON.parse(await readFile(path, 'utf8')) + await writeFile(path, JSON.stringify({ ...receipt, controllerPid: deadPid, ...changes })) + await writeFile(join(root, 'input.jsonl'), 'private input', { mode: 0o600 }) + return root +} +it('removes only an expired, owned staging directory after its controller and container disappear', async () => { + const root = await fixture() + expect((await recoverExperimentStaging(121999, base)).removed).toEqual([]) + expect(docker.calls).toBe(0) + expect((await recoverExperimentStaging(122000, base)).removed).toHaveLength(1) + await expect(stat(root)).rejects.toMatchObject({ code: 'ENOENT' }) + expect((await recoverExperimentStaging(122000, base)).removed).toEqual([]) +}) +it.each([{ controllerPid: process.pid }, { inode: 1 }, { owner: 999999 }, { schema: 'unknown' }])('preserves unproven ownership or a live controller: %j', async (changes) => { + const root = await fixture(changes) + expect((await recoverExperimentStaging(122000, base)).removed).toEqual([]) + expect((await stat(root)).isDirectory()).toBe(true) + expect(docker.calls).toBe(0) +}) +it('retains files when Docker still has the container or is unavailable', async () => { + const root = await fixture() + docker.mode = 'present' + expect((await recoverExperimentStaging(122000, base)).removed).toEqual([]) + docker.mode = 'offline' + await expect(recoverExperimentStaging(122000, base)).rejects.toThrow('Docker unavailable') + expect((await stat(root)).isDirectory()).toBe(true) +}) +it('does not follow symlinked ownership markers or infer legacy directory ownership', async () => { + const root = await fixture() + await rm(join(root, '.sleepypod-staging.json')) + const legacy = await mkdtemp(join(base, 'sleepypod-package-')) + await symlink('/etc/passwd', join(root, '.sleepypod-staging.json')) + expect((await recoverExperimentStaging(122000, base)).removed).toEqual([]) + expect((await stat(legacy)).isDirectory()).toBe(true) + expect(docker.calls).toBe(0) +}) diff --git a/scripts/algorithm-experiments/staging-recovery.ts b/scripts/algorithm-experiments/staging-recovery.ts new file mode 100644 index 000000000..bd16d7e99 --- /dev/null +++ b/scripts/algorithm-experiments/staging-recovery.ts @@ -0,0 +1,106 @@ +import { execFile } from 'node:child_process' +import { constants } from 'node:fs' +import { lstat, open, opendir, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { basename, join } from 'node:path' +import { promisify } from 'node:util' +import { z } from 'zod' + +const exec = promisify(execFile) +const marker = '.sleepypod-staging.json' +const directoryName = /^sleepypod-package-[a-zA-Z0-9]{6}$/ +const integer = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) +const receiptSchema = z.object({ schema: z.literal('biometric-private-staging-v1'), + directory: z.string().regex(directoryName), device: integer, inode: integer, owner: integer, + controllerPid: integer.positive().max(2147483647), expiresAtMs: integer, + containerName: z.string().regex(/^sleepypod-experiment-[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/), +}).strict() + +/** Written before input/package bytes. Legacy unmarked directories are never + * inferred to be owned from a matching name alone. */ +export async function recordExperimentStaging(root: string, containerName: string, maximumWallMs: number, now = Date.now()) { + z.number().int().min(1).max(3600000).parse(maximumWallMs) + integer.parse(now) + const info = await lstat(root) + if (!info.isDirectory() || info.isSymbolicLink() || info.mode & 0o077 || info.uid !== process.getuid?.()) throw new Error('Staging directory must be private and owned') + const receipt = receiptSchema.parse({ schema: 'biometric-private-staging-v1', directory: basename(root), device: info.dev, inode: info.ino, + owner: info.uid, controllerPid: process.pid, expiresAtMs: now + maximumWallMs + 120000, containerName }) + const file = await open(join(root, marker), constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600) + try { + await file.writeFile(JSON.stringify(receipt)) + await file.sync() + } + finally { await file.close() } +} +function processAbsent(pid: number) { + try { + process.kill(pid, 0) + } + catch (error) { return (error as NodeJS.ErrnoException).code === 'ESRCH' } + return false +} +async function containerAbsent(name: string) { + try { + await exec('docker', ['inspect', '--type', 'container', name], { timeout: 10000, maxBuffer: 1024 * 1024 }) + return false + } + catch (error) { + if (error instanceof Error && /No such (object|container):/.test(error.message)) return true + throw error // An unavailable daemon is not proof of absence. + } +} +/** Trusted host entry point; base is injectable for isolated tests. Container + * recovery must run first. Only an expired receipt, dead controller and absent + * named container permit deletion. PID reuse conservatively retains a directory. */ +export async function recoverExperimentStaging(now = Date.now(), base = tmpdir()) { + integer.parse(now) + const removed: string[] = [] + let scanned = 0, candidates = 0 + const entries = await opendir(base) + for await (const entry of entries) { + if (++scanned > 16384) throw new Error('Staging recovery inventory exceeds limit') + if (!directoryName.test(entry.name)) continue + if (++candidates > 128) throw new Error('Staging recovery candidate limit exceeded') + const root = join(base, entry.name) + let before, receipt, markerIdentity + try { + before = await lstat(root) + if (!before.isDirectory() || before.isSymbolicLink() || before.mode & 0o077 || before.uid !== process.getuid?.()) continue + const file = await open(join(root, marker), constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK) + try { + markerIdentity = await file.stat() + if (!markerIdentity.isFile() || markerIdentity.nlink !== 1 || markerIdentity.mode & 0o077 || markerIdentity.uid !== before.uid || markerIdentity.size > 4096) continue + const bytes = Buffer.alloc(4097) + const { bytesRead } = await file.read(bytes, 0, bytes.length, 0) + if (bytesRead !== markerIdentity.size) continue + receipt = receiptSchema.parse(JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes.subarray(0, bytesRead)))) + } + finally { await file.close() } + } + catch { continue } // Unknown or damaged ownership never authorizes deletion. + if (receipt.directory !== entry.name || receipt.device !== before.dev || receipt.inode !== before.ino || receipt.owner !== before.uid + || receipt.expiresAtMs > now || !processAbsent(receipt.controllerPid)) continue + if (!await containerAbsent(receipt.containerName)) continue + let current, currentMarker + try { + current = await lstat(root) + currentMarker = await lstat(join(root, marker)) + } + catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') continue + throw error + } + if (current.dev !== before.dev || current.ino !== before.ino || current.mode !== before.mode || current.uid !== before.uid + || currentMarker.dev !== markerIdentity.dev || currentMarker.ino !== markerIdentity.ino || currentMarker.size !== markerIdentity.size + || currentMarker.mtimeMs !== markerIdentity.mtimeMs || !processAbsent(receipt.controllerPid)) continue + try { + await rm(root, { recursive: true }) + } + catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') continue + throw error + } + removed.push(entry.name) + } + return { schema: 'biometric-staging-recovery-v1' as const, checkedAt: now, scanned, candidates, removed } +} diff --git a/scripts/algorithm-experiments/workspace.test.ts b/scripts/algorithm-experiments/workspace.test.ts new file mode 100644 index 000000000..75e5f7aa9 --- /dev/null +++ b/scripts/algorithm-experiments/workspace.test.ts @@ -0,0 +1,104 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import * as filesystem from 'node:fs/promises' +import { spawn } from 'node:child_process' +import { once } from 'node:events' +import { mkdtemp, readdir, rm, symlink, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import { withExperimentWorkspace } from './workspace' + +vi.mock('node:fs/promises', async (original) => { + const actual = await original() + return { ...actual, statfs: vi.fn(actual.statfs) } +}) + +let root: string +const policy = { maximumReportBytes: 80, maximumReportFiles: 2, maximumSingleReportBytes: 60, + maximumDatabaseBytes: 65536, minimumFreeBytes: 0 } +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'experiment-workspace-test-')) +}) +afterEach(async () => { + vi.restoreAllMocks() + await rm(root, { recursive: true, force: true }) +}) +it('rejects concurrent workspace owners and releases on ordinary failure', async () => { + await withExperimentWorkspace(root, policy, async () => { + await expect(withExperimentWorkspace(root, policy, async () => null)).rejects.toThrow('locked') + }) + await expect(withExperimentWorkspace(root, policy, async () => { + throw new Error('failed') + })).rejects.toThrow('failed') + expect(await withExperimentWorkspace(root, policy, async () => 'available')).toBe('available') +}) +it('bounds report bytes/counts before publication without deleting prior evidence', async () => { + await withExperimentWorkspace(root, policy, async (workspace) => { + await workspace.publishReport(join(workspace.reports, 'first.json'), { value: 'a'.repeat(30) }) + await expect(workspace.publishReport(join(workspace.reports, 'second.json'), { value: 'b'.repeat(30) })).rejects.toThrow('byte quota') + await expect(workspace.publishReport(join(workspace.reports, 'oversize.json'), { value: 'c'.repeat(60) })).rejects.toThrow('single-report') + await workspace.publishReport(join(workspace.reports, 'small.json'), {}) + await expect(workspace.publishReport(join(workspace.reports, 'third.json'), {})).rejects.toThrow('file quota') + expect((await readdir(workspace.reports)).sort()).toEqual(['first.json', 'small.json']) + await expect(workspace.publishReport(join(root, 'outside.json'), {})).rejects.toThrow('outside') + }) +}) +it('refuses unsafe existing report entries and a symlinked lock', async () => { + await withExperimentWorkspace(root, policy, async (workspace) => { + await symlink('/tmp', join(workspace.reports, 'link')) + await expect(workspace.checkCapacity()).rejects.toThrow('Unsafe') + }) + await rm(join(root, '.writer.sqlite')) + await symlink('/tmp', join(root, '.writer.sqlite')) + await expect(withExperimentWorkspace(root, policy, async () => null)).rejects.toThrow('Unsafe') +}) +it('enforces the ledger page ceiling and refuses a reduced limit below existing size', async () => { + const db = new Database(join(root, 'experiments.sqlite')) + try { + await withExperimentWorkspace(root, policy, async (workspace) => { + workspace.configureDatabase(db) + db.exec('CREATE TABLE payload(bytes BLOB)') + expect(() => db.prepare('INSERT INTO payload VALUES (?)').run(Buffer.alloc(100000))).toThrow(/full/) + expect(db.prepare('SELECT COUNT(*) AS count FROM payload').get()).toEqual({ count: 0 }) + }) + db.pragma('max_page_count = 1000') + db.prepare('INSERT INTO payload VALUES (?)').run(Buffer.alloc(100000)) + await expect(withExperimentWorkspace(root, policy, async workspace => workspace.configureDatabase(db))).rejects.toThrow('database quota') + } + finally { db.close() } +}) +it('fails admission before starting work when the free-space reserve is unavailable', async () => { + vi.mocked(filesystem.statfs).mockResolvedValueOnce({ type: 0n, bsize: 4096n, blocks: 1n, bfree: 0n, bavail: 0n, files: 0n, ffree: 0n }) + let entered = false + await expect(withExperimentWorkspace(root, { ...policy, minimumFreeBytes: 1024 ** 4 }, async () => { + entered = true + })).rejects.toThrow('free-space') + expect(entered).toBe(false) +}) +it('releases the OS writer lock after the owning controller is killed', async () => { + await writeFile(join(root, '.writer.sqlite'), '', { mode: 0o600 }) + const child = spawn(process.execPath, ['-e', `const Database=require('better-sqlite3');const db=new Database(process.argv[1]);db.exec('BEGIN IMMEDIATE');process.send('locked');setInterval(()=>{},1000)`, join(root, '.writer.sqlite')], { stdio: ['ignore', 'ignore', 'pipe', 'ipc'] }) + const closed = once(child, 'close') + try { + await once(child, 'message') + await expect(withExperimentWorkspace(root, policy, async () => null)).rejects.toThrow('locked') + child.kill('SIGKILL') + await closed + expect(await withExperimentWorkspace(root, policy, async () => 'recovered')).toBe('recovered') + } + finally { + child.kill('SIGKILL') + await closed + } +}) + +it('serializes concurrent publications under the same quota snapshot', async () => { + await withExperimentWorkspace(root, policy, async (workspace) => { + const first = workspace.publishReport(join(workspace.reports, 'first.json'), { value: 'a'.repeat(30) }) + const second = workspace.publishReport(join(workspace.reports, 'second.json'), { value: 'b'.repeat(30) }) + const results = await Promise.allSettled([first, second]) + expect(results.map(result => result.status)).toEqual(['fulfilled', 'rejected']) + expect(await readdir(workspace.reports)).toEqual(['first.json']) + }) +}) diff --git a/scripts/algorithm-experiments/workspace.ts b/scripts/algorithm-experiments/workspace.ts new file mode 100644 index 000000000..2597e865b --- /dev/null +++ b/scripts/algorithm-experiments/workspace.ts @@ -0,0 +1,100 @@ +import Database from 'better-sqlite3' +import { constants } from 'node:fs' +import { lstat, mkdir, open, opendir, statfs } from 'node:fs/promises' +import { dirname, join, resolve } from 'node:path' +import { z } from 'zod' +import { publishJson } from './files' + +const integer = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) +export const workspacePolicySchema = z.object({ + maximumReportBytes: integer.positive().max(64 * 1024 ** 3), + maximumReportFiles: integer.positive().max(100000), + maximumSingleReportBytes: integer.positive().max(32 * 1024 ** 2), + maximumDatabaseBytes: integer.min(65536).max(1024 ** 3), + minimumFreeBytes: integer.max(1024 ** 4), +}).strict() +/** Off-pod workstation limits, not pod production budgets. */ +export const defaultWorkspacePolicy = Object.freeze({ maximumReportBytes: 1024 ** 3, maximumReportFiles: 2048, + maximumSingleReportBytes: 32 * 1024 ** 2, maximumDatabaseBytes: 64 * 1024 ** 2, minimumFreeBytes: 1024 ** 3 }) + +async function privateDirectory(path: string) { + await mkdir(path, { recursive: true, mode: 0o700 }) + const info = await lstat(path) + if (!info.isDirectory() || info.isSymbolicLink() || (info.mode & 0o077) || (process.getuid && info.uid !== process.getuid())) throw new Error('Experiment workspace must be private and owned') +} +async function privateFile(path: string, missing = false) { + try { + const info = await lstat(path) + if (!info.isFile() || info.isSymbolicLink() || info.nlink !== 1 || (info.mode & 0o077) + || (process.getuid && info.uid !== process.getuid())) throw new Error('Unsafe experiment workspace file') + return info + } + catch (error) { + if (missing && (error as NodeJS.ErrnoException).code === 'ENOENT') return null + throw error + } +} + +/** A separate SQLite write lock survives async work and is released by the OS + * on controller death. Never remove its inode or steal ownership by timeout. */ +export async function withExperimentWorkspace(directory: string, rawPolicy: unknown, + action: (workspace: { reports: string, configureDatabase: (db: Database.Database) => void, publishReport: typeof publishJson, checkCapacity: () => Promise }) => Promise) { + const policy = workspacePolicySchema.parse(rawPolicy), root = resolve(directory) + await privateDirectory(root) + const lockPath = join(root, '.writer.sqlite') + try { + const file = await open(lockPath, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY | constants.O_NOFOLLOW, 0o600) + await file.close() + } + catch (error) { if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error } + for (const suffix of ['', '-wal', '-shm', '-journal']) { + const info = await privateFile(lockPath + suffix, suffix !== '') + if (info && info.size > 65536) throw new Error('Workspace lock exceeds its size bound') + } + const lock = new Database(lockPath, { timeout: 0 }) + let publication: Promise = Promise.resolve() + try { + lock.exec('BEGIN IMMEDIATE') + if ((lock.prepare('SELECT COUNT(*) AS count FROM sqlite_master').get() as { count: number }).count !== 0) throw new Error('Unexpected workspace lock database') + const reports = join(root, 'reports') + await privateDirectory(reports) + const capacity = async (additionalBytes = 0, additionalFiles = 0) => { + let count = additionalFiles, bytes = additionalBytes + const entries = await opendir(reports) + for await (const entry of entries) { + if (++count > policy.maximumReportFiles) throw new Error('Experiment report file quota exceeded') + const info = await privateFile(join(reports, entry.name)) + bytes += info?.size ?? 0 + if (bytes > policy.maximumReportBytes) throw new Error('Experiment report byte quota exceeded') + } + if (count > policy.maximumReportFiles || bytes > policy.maximumReportBytes) throw new Error('Experiment report quota exceeded') + const disk = await statfs(root, { bigint: true }) + // Reserve for a complete rollback journal and growth of the bounded ledger. + const required = BigInt(policy.minimumFreeBytes) + BigInt(2 * policy.maximumDatabaseBytes) + BigInt(additionalBytes) + if (disk.bavail < 0n || disk.bsize <= 0n || disk.bavail * disk.bsize < required) throw new Error('Experiment workspace free-space reserve unavailable') + } + await capacity() + return await action({ reports, checkCapacity: () => capacity(), configureDatabase: (db) => { + const pageSize = Number(db.pragma('page_size', { simple: true })) + if (!Number.isSafeInteger(pageSize) || pageSize < 512) throw new Error('Invalid experiment database page size') + const pages = Math.floor(policy.maximumDatabaseBytes / pageSize) + if (Number(db.pragma(`max_page_count = ${pages}`, { simple: true })) > pages) throw new Error('Experiment database quota exceeded') + if (db.pragma('journal_mode = DELETE', { simple: true }) !== 'delete') throw new Error('Bounded experiment rollback journal required') + }, publishReport: async (path, value) => { + if (dirname(resolve(path)) !== reports) throw new Error('Report path outside managed workspace') + const snapshot = structuredClone(value) + const bytes = Buffer.byteLength(JSON.stringify(snapshot, null, 2) + '\n') + if (bytes > policy.maximumSingleReportBytes) throw new Error('Experiment single-report quota exceeded') + const result = publication.then(async () => { + await capacity(bytes, 1) + await publishJson(path, snapshot) + }) + publication = result.catch(() => {}) + return result + } }) + } + finally { + await publication + lock.close() + } +} diff --git a/scripts/benchmarks/biometrics-live-shadow-runner.ts b/scripts/benchmarks/biometrics-live-shadow-runner.ts new file mode 100644 index 000000000..39d64eaca --- /dev/null +++ b/scripts/benchmarks/biometrics-live-shadow-runner.ts @@ -0,0 +1,133 @@ +/** Operational measurement only; never installs a qualification or changes profiles. + * Run on the pod: STAGE APP_ROOT PYTHON MODULES DATA_DIR [SECONDS=180] + * Live frames stay on the pod. Only summary metadata may be exported afterward. + */ +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { createHash } from 'node:crypto' +import { mkdtemp, readFile, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { setTimeout as delay } from 'node:timers/promises' +import WebSocket from 'ws' +import { defaultConfig, manifestHash } from '../../src/biometrics/algorithms/registry' +import { configHash, dataHash } from '../../src/biometrics/algorithms/canonical' +import { ShadowStore } from '../../src/biometrics/shadow/store' +import { runShadowAttempt } from '../../src/biometrics/shadow/attempt' +import { plannedShadowSourceFactory } from '../../src/biometrics/shadow/planned-sources' +import { startShadowHealthMonitor } from '../../src/biometrics/shadow/health-monitor' +import { collectAdmissionSample, evaluateAdmission, type AdmissionSample } from '../../src/biometrics/backtest/admission' +import { probeShadowBuild } from '../../src/biometrics/shadow/build' +import { publishProspectiveFrame, failProspectiveFrames } from '../../src/streaming/prospectiveFrameTap' + +async function main() { + const [stage, app, pythonPath, modulesPath, data, rawSeconds = '180'] = process.argv.slice(2) + const seconds = Number(rawSeconds) + if (!stage || !app || !pythonPath || !modulesPath || !data || !Number.isInteger(seconds) || seconds < 150 || seconds > 600) throw new Error('Invalid measurement arguments') + const directory = await mkdtemp(join(stage, 'live-shadow-')) + const configurationDatabase = join(data, 'sleepypod.db'), biometricsDatabase = join(data, 'biometrics.db') + const configuration = new Database(configurationDatabase, { readonly: true, fileMustExist: true, timeout: 100 }) + const sink = new Database(join(directory, 'isolated.db')) + sink.pragma('journal_mode = WAL') + migrate(drizzle(sink), { migrationsFolder: join(app, 'src/db/biometrics-migrations') }) + const controller = new AbortController(), socket = new WebSocket('ws://127.0.0.1:3001', { maxPayload: 128 * 1024 }) + let piezoAt: number | undefined, received = 0, closed = false + socket.on('open', () => socket.send(JSON.stringify({ type: 'subscribe', sensors: ['piezo-dual', 'frzHealth'] }))) + socket.on('message', (bytes) => { + try { + const frame = JSON.parse(bytes.toString()) + if (!['piezo-dual', 'frzHealth'].includes(frame.type)) return + const now = Date.now() + if (frame.type === 'piezo-dual') piezoAt = now + received++ + publishProspectiveFrame(frame, false, now) + } + catch { + controller.abort() + failProspectiveFrames() + } + }) + socket.on('error', () => { + controller.abort() + failProspectiveFrames() + }) + socket.on('close', () => { + if (!closed) { + controller.abort() + failProspectiveFrames() + } + }) + let monitor: Awaited> | undefined + let runId: string | undefined, result: unknown = null, failure: string | null = null + let lastHealth: AdmissionSample | null = null, refusal: unknown = null + const store = new ShadowStore(sink, { maximumRuns: 1, maximumPoints: 2000, maximumBytes: 8 * 1024 * 1024 }) + try { + const deadline = Date.now() + 30000 + while (!piezoAt && Date.now() < deadline && !controller.signal.aborted) await delay(100) + if (!piezoAt || controller.signal.aborted) throw new Error('No live piezo stream') + const assertDefaults = () => { + if (!configuration.readonly || (configuration.prepare('SELECT count(*) AS count FROM biometric_profile_bindings').get() as { count: number }).count !== 0) throw new Error('Measurement requires unchanged unpersonalized defaults') + } + assertDefaults() + const policy = { qualificationId: 'live-shadow-operational-measurement-only', minimumAvailableMemoryBytes: 768 * 1024 * 1024, + minimumFreeDiskBytes: 2 * 1024 * 1024 * 1024, maximumLoadPerCpu: 1.1, maximumEventLoopDelayMs: 100, + maximumPiezoAgeMs: 5000, maximumTemperatureC: 55, maximumSampleAgeMs: 2000 } + monitor = await startShadowHealthMonitor(async () => { + lastHealth = await collectAdmissionSample(directory, () => piezoAt) + return lastHealth + }, policy, controller.signal) + const hash = configHash(defaultConfig), python = await probeShadowBuild({ pythonPath, modulesPath }) + const runnerHash = createHash('sha256').update(await readFile(process.argv[1])).digest('hex') + const startMs = Date.now() + 2000, endMs = startMs + seconds * 1000 + const plan = { schema: 'biometric-shadow-plan-v1', side: 'left', subjectId: 'local-operational-measurement', startMs, endMs, + candidateConfigHash: hash, activeConfigHash: hash, qualificationHash: dataHash(policy), engineBuildHash: dataHash({ runnerHash, python: python.hash }), + maximumPoints: 2000, maximumBytes: 8 * 1024 * 1024, metrics: ['heart_rate'], + joining: { maximumPendingPoints: 200, maximumLatenessMs: 60000, drainGraceMs: 65000 }, + monitoring: { expectedIntervalMs: 60000, maximumGapMs: 120000, heartbeatStaleMs: 2000, minimumCandidateCoverage: 0.01, + maximumCoverageLoss: 1, minimumQuality: 0, alignment: { version: 'reference-alignment-v1', pointToleranceMs: 0, clockOffsetMs: 0, offsetSource: 'fixed' } }, + execution: { version: 'shadow-execution-policy-v1', configHashes: { left: hash, right: hash }, maximumRecords: 10000, maximumOutputs: 2000, + maximumCpuMs: 120000, maximumInputBytes: 64 * 1024 * 1024, maximumOutputBytes: 8 * 1024 * 1024, + budgets: { cpuSeconds: 120, addressSpaceBytes: 768 * 1024 * 1024, maximumRssBytes: 384 * 1024 * 1024, + wallMs: (seconds + 90) * 1000, acknowledgmentMs: 30000, idleMs: 10000, rssIntervalMs: 100, rssTimeoutMs: 1000 }, + leaseMs: 5000, heartbeatMs: 500, sources: { pollMs: 500, maximumActiveRows: 100, databaseBusyTimeoutMs: 100, + frames: { maximumFrames: 30, maximumBytes: 4 * 1024 * 1024, maximumEventAgeMs: 10000, maximumFutureSkewMs: 1000 }, + context: { presenceStaleMs: 300000, pumpStaleMs: 5000, maximumPresenceBytes: 65536 } } } } + const created = store.create(plan, 'measurement'), openSources = plannedShadowSourceFactory(plan, + { configurationDatabase, biometricsDatabase, presenceFile: join(data, 'sleep-detector-state.json') }) + runId = created.id + await delay(Math.max(0, startMs - Date.now())) + const attempt = await runShadowAttempt(store, runId, { location: { pythonPath, modulesPath }, + configs: { left: defaultConfig, right: defaultConfig }, openSources, admit: () => { + assertDefaults() + if (!monitor?.read()) { + refusal = { checkedAtMs: Date.now(), sample: lastHealth, decision: evaluateAdmission(lastHealth, policy) } + throw new Error('Measurement health admission refused') + } + } }, controller.signal) + result = { runnerHash, pythonBuildHash: python.hash, manifestHash, receipt: attempt.receipt } + } + catch (error) { + failure = error instanceof Error ? error.message : String(error) + process.exitCode = 1 + } + finally { + closed = true + controller.abort() + monitor?.close() + socket.terminate() + const run = runId ? store.get(runId) : null + const canonicalRows = (sink.prepare('SELECT count(*) AS count FROM vitals').get() as { count: number }).count + await writeFile(join(stage, 'live-shadow-summary.json'), JSON.stringify({ + version: 'live-shadow-operational-measurement-v1', scope: 'loopback live input and read-only active observations with isolated sink; not production registry admission or accuracy qualification', + seconds, receivedFrames: received, state: run?.state ?? null, reason: run?.reason ?? null, + pointCount: run?.point_count ?? 0, byteCount: run?.byte_count ?? 0, canonicalRowsInIsolatedSink: canonicalRows, + result, failure, refusal, workerReceipt: run?.workerReceipt ?? null, + }, null, 2)) + configuration.close() + sink.close() + } +} +void main().catch((error) => { + console.error(error) + process.exitCode = 1 +}) diff --git a/scripts/benchmarks/biometrics-probe.py b/scripts/benchmarks/biometrics-probe.py new file mode 100644 index 000000000..8d3036515 --- /dev/null +++ b/scripts/benchmarks/biometrics-probe.py @@ -0,0 +1,88 @@ +"""Independent, read-only loopback telemetry probe around a benchmark command. + +Usage: python3 biometrics-probe.py OUTPUT.json COMMAND [ARG ...] +Never retains response bodies or sensor samples. +""" +import json +import os +import signal +import pathlib +import subprocess +import sys +import threading +import time +import urllib.request + +samples = [] +phase = "baseline" +stop = threading.Event() +benchmark_pid = None + + +def process_tree_memory(root_pid): + if root_pid is None: + return 0, 0 + processes = {} + for path in pathlib.Path('/proc').glob('[0-9]*/status'): + try: + fields = dict(line.split(':', 1) for line in path.read_text().splitlines() if ':' in line) + processes[int(path.parent.name)] = (int(fields['PPid']), int(fields.get('VmRSS', '0 kB').split()[0])) + except (OSError, ValueError, KeyError): + continue + selected = {root_pid} + while True: + children = {pid for pid, (parent, _) in processes.items() if parent in selected} + if children <= selected: + break + selected |= children + return sum(processes.get(pid, (0, 0))[1] for pid in selected), len(selected & processes.keys()) + + +def sample(): + while not stop.is_set(): + row = {"phase": phase, "at": time.monotonic()} + cpu = list(map(int, pathlib.Path("/proc/stat").read_text().splitlines()[0].split()[1:])) + row["cpuTotal"] = sum(cpu[:8]) + row["cpuIdle"] = cpu[3] + cpu[4] + row['benchmarkTreeRssKiB'], row['benchmarkProcessCount'] = process_tree_memory(benchmark_pid) + memory = pathlib.Path('/proc/meminfo').read_text().splitlines() + row['memAvailableKiB'] = int(next(line.split()[1] for line in memory if line.startswith('MemAvailable:'))) + row['temperatureC'] = int(pathlib.Path('/sys/class/thermal/thermal_zone0/temp').read_text()) / 1000 + for endpoint in ["health/performance", "system/sensor-source", "device/status"]: + began = time.monotonic() + try: + with urllib.request.urlopen("http://127.0.0.1:3000/api/" + endpoint, timeout=2) as response: + response.read() + status = response.status + row[endpoint] = {"ms": (time.monotonic() - began) * 1000, "status": status} + except Exception as error: + row[endpoint] = {"ms": (time.monotonic() - began) * 1000, "error": type(error).__name__} + samples.append(row) + stop.wait(0.25) + + +if len(sys.argv) < 3: + raise SystemExit("OUTPUT.json COMMAND [ARG ...] required") +timeout_seconds = int(os.environ.get('BIOMETRICS_BENCHMARK_TIMEOUT_SECONDS', '600')) +if not 1 <= timeout_seconds <= 10800: + raise SystemExit('Benchmark timeout must be between 1 and 10800 seconds') +thread = threading.Thread(target=sample) +thread.start() +try: + time.sleep(4) + phase = "benchmark" + result = subprocess.Popen(sys.argv[2:], start_new_session=True) + benchmark_pid = result.pid + try: + result.wait(timeout=timeout_seconds) + except subprocess.TimeoutExpired: + os.killpg(result.pid, signal.SIGKILL) + result.wait() + benchmark_pid = None + phase = "after" + time.sleep(3) +finally: + stop.set() + thread.join() + pathlib.Path(sys.argv[1]).write_text(json.dumps({"version": "read-only-biometric-probe-v2", "timeoutSeconds": timeout_seconds, "samples": samples}, indent=2)) +raise SystemExit(result.returncode) diff --git a/scripts/benchmarks/biometrics-runner.ts b/scripts/benchmarks/biometrics-runner.ts new file mode 100644 index 000000000..db13a78a3 --- /dev/null +++ b/scripts/benchmarks/biometrics-runner.ts @@ -0,0 +1,158 @@ +/** Synthetic-only qualification harness. Bundle with esbuild (better-sqlite3 external). + * Pass NEW_DIRECTORY MIGRATIONS WORKER [mixed|dense] [COORDINATOR]. + * Omitting COORDINATOR measures the old, synchronous calling-process path. + * Reads only loopback sensor/API telemetry; never opens the application databases. + */ +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs' +import { resolve, join } from 'node:path' +import { monitorEventLoopDelay, performance } from 'node:perf_hooks' +import { createHash } from 'node:crypto' +import WebSocket from 'ws' +import * as schema from '../../src/db/biometrics-schema' +import { DatasetStore } from '../../src/biometrics/backtest/datasets' +import { BacktestJobStore } from '../../src/biometrics/backtest/jobs' +import { runCoordinatorTask } from '../../src/biometrics/backtest/coordinator' +import { executeNextBacktest } from '../../src/biometrics/backtest/runner' +import { workerBuildHash } from '../../src/biometrics/backtest/child' +import { defaultConfig, manifestHash } from '../../src/biometrics/algorithms/registry' +import { configHash } from '../../src/biometrics/algorithms/canonical' +import { ReferenceStore } from '../../src/biometrics/references/store' +import { previewReferenceImport } from '../../src/biometrics/references/normalize' + +async function main() { + const [directoryArg, migrationsArg, workerArg, fixture = 'mixed', coordinatorArg] = process.argv.slice(2) + if (!directoryArg || !migrationsArg || !workerArg) throw new Error('NEW_DIRECTORY MIGRATIONS WORKER required') + if (!['mixed', 'dense'].includes(fixture)) throw new Error('Fixture must be mixed or dense') + const dense = fixture === 'dense' + const directory = resolve(directoryArg) + // Exclusive creation prevents accidentally benchmarking against an existing DB. + mkdirSync(directory, { mode: 0o700 }) + const sqlite = new Database(join(directory, 'synthetic.db')) + sqlite.pragma('journal_mode = WAL') + const db = drizzle(sqlite, { schema }) + migrate(db, { migrationsFolder: resolve(migrationsArg) }) + const workerPath = resolve(workerArg), evaluatorBuildHash = await workerBuildHash(workerPath) + const sessionIds: number[] = [] + const references: Array> = [] + const day = 86400000, start = Date.UTC(2026, 0, 1, 20) + sqlite.transaction(() => { + for (let night = 0; night < 20; night++) { + const from = start + night * day, id = night + 1 + sessionIds.push(id) + db.insert(schema.sleepRecords).values({ id, side: 'left', enteredBedAt: new Date(from), leftBedAt: new Date(from + 12 * 3600000), sleepDurationSeconds: 43200, timesExitedBed: 0 }).run() + for (let row = 0; row < (dense ? 950 : 720); row++) { + db.insert(schema.vitals).values({ side: 'left', timestamp: new Date(from + row * (dense ? 45000 : 60000)), heartRate: row % 47 === 0 ? null : 55 + 12 * Math.sin(row / 30), hrv: 30 + row % 40, breathingRate: 12 + row % 7 }).run() + if (!dense && row % 4 === 0) db.insert(schema.movement).values({ side: 'left', timestamp: new Date(from + row * (dense ? 45000 : 60000)), totalMovement: row % 32 === 0 ? 250 : row % 60 }).run() + } + for (let row = 0; row < (dense ? 20 : 40); row++) { + const timestampMs = from + row * (dense ? 2160000 : 1080000) + references.push({ recordKey: `${night}:hr:${row}`, metric: 'heart_rate', value: 58 + row % 12, unit: 'bpm', timestampMs }) + references.push({ recordKey: `${night}:stage:${row}`, metric: 'sleep_stages', value: ['wake', 'light', 'deep', 'rem'][row % 4], unit: 'stage', startMs: timestampMs, endMs: timestampMs + (dense ? 2160000 : 1080000) }) + } + } + })() + const imported = { side: 'left' as const, subjectId: 'synthetic-qualification', source: 'synthetic-v1', rows: references } + const preview = previewReferenceImport(imported) + const reference = new ReferenceStore(db).commit({ import: imported, previewHash: preview.summary.contentHash }) + const coordinator = coordinatorArg ? { databasePath: join(directory, 'synthetic.db'), coordinatorPath: resolve(coordinatorArg), workerPath, modulePath: process.env.NODE_PATH } : null + const snapshotHistogram = monitorEventLoopDelay({ resolution: 20 }) + snapshotHistogram.enable() + await new Promise(resolve => setTimeout(resolve, 50)) + const snapshotStart = performance.now() + const datasetInput = { side: 'left' as const, subjectId: imported.subjectId, sessionIds, referenceImportIds: [reference.id], timezone: 'UTC', idempotencyKey: 'synthetic-v1' } + const dataset = coordinator ? await runCoordinatorTask(coordinator, { type: 'createDataset', input: datasetInput }) : new DatasetStore(db).create(datasetInput) + const snapshotMs = performance.now() - snapshotStart + await new Promise(resolve => setTimeout(resolve, 50)) + const snapshotEventLoopMaxMs = snapshotHistogram.max / 1e6 + snapshotHistogram.disable() + const originalHash = () => createHash('sha256').update(JSON.stringify(sqlite.prepare('SELECT * FROM vitals ORDER BY id').all())).digest('hex') + const beforeHash = originalHash() + const jobs = new BacktestJobStore(db) + const protocol = { version: 'descriptive-evaluation-v1' as const, mode: 'stage_replay' as const, assumedStageCalibrationQuality: 1, pointToleranceMs: 30000, clockOffsetMs: 0 } + const experiment = coordinator ? await runCoordinatorTask(coordinator, { type: 'createExperiment', input: { datasetId: dataset.id, protocol, idempotencyKey: 'synthetic-experiment' } }) : jobs.createExperiment(dataset.id, protocol) + const configs = (['default', 'current', 'candidate'] as const).map((label, index) => { + const resolved = structuredClone(defaultConfig) + // Synthetic comparison roles: this benchmark tests resources, not promotion. + resolved.sleep_stages.ruleset.rules[1].when[0].value = 0.90 + index * 0.02 + return { label, revisionId: null, manifestHash, resolvedHash: configHash(resolved), resolved } + }) + let phase = 'baseline', stopped = false, piezoAt: number | null = null, piezoSourceAt: number | null = null + const samples: unknown[] = [], runs: unknown[] = [] + const socket = new WebSocket('ws://127.0.0.1:3001') + socket.on('open', () => socket.send(JSON.stringify({ type: 'subscribe', sensors: ['piezo-dual'] }))) + socket.on('error', () => {}) + socket.on('message', (bytes) => { + try { + const frame = JSON.parse(bytes.toString()) + if (frame.type === 'piezo-dual') { + piezoAt = Date.now() + piezoSourceAt = typeof frame.ts === 'number' ? frame.ts * 1000 : null + } + } + catch { /* No frame contents are retained. */ } + }) + const histogram = monitorEventLoopDelay({ resolution: 20 }) + histogram.enable() + const sampler = (async () => { + while (!stopped) { + const sample: Record = { phase, atMs: Date.now(), parentRss: process.memoryUsage().rss, piezoAgeMs: piezoAt === null ? null : Date.now() - piezoAt, piezoSourceAgeMs: piezoSourceAt === null ? null : Date.now() - piezoSourceAt } + sample.memAvailableKiB = Number(/^MemAvailable:\s+(\d+)/m.exec(readFileSync('/proc/meminfo', 'utf8'))?.[1]) + sample.temperatureC = Number(readFileSync('/sys/class/thermal/thermal_zone0/temp', 'utf8')) / 1000 + for (const path of ['health/performance', 'device/status']) { + const began = performance.now() + try { + const response = await fetch(`http://127.0.0.1:3000/api/${path}`, { signal: AbortSignal.timeout(2000) }) + await response.arrayBuffer() + sample[path] = { ms: performance.now() - began, status: response.status } + } + catch { sample[path] = { failed: true, ms: performance.now() - began } } + } + samples.push(sample) + await new Promise(resolve => setTimeout(resolve, 250)) + } + })() + try { + await new Promise(resolve => setTimeout(resolve, 4000)) + for (let attempt = 0; attempt < 3; attempt++) { + const job = jobs.enqueue({ evaluatorBuildHash, experimentId: experiment.id, configs, split: 'tuning', idempotencyKey: `run-${attempt}` }) + phase = `replay-${attempt}` + histogram.reset() + const began = performance.now() + const outcome = coordinator + ? await runCoordinatorTask(coordinator, { type: 'execute', owner: 'synthetic-qualification' }, { admit: async () => true }) + : await executeNextBacktest(db, { owner: 'synthetic-qualification', workerPath, admit: async () => true, timeoutMs: 120000 }) + const elapsedMs = performance.now() - began + // Let timers observe the final synchronous publication before reading delay. + await new Promise(resolve => setTimeout(resolve, 50)) + const readStarted = performance.now(), result = coordinator ? await runCoordinatorTask(coordinator, { type: 'readResult', input: { id: job.id } }) : jobs.result(job.id) + const resultReadMs = performance.now() - readStarted + const report = result?.report as { operational?: unknown } | undefined + const reportBytes = Buffer.byteLength(JSON.stringify(report ?? null)) + // Include IPC deserialization and report serialization in the calling-loop maximum. + await new Promise(resolve => setTimeout(resolve, 50)) + runs.push({ outcome, elapsedMs, resultReadMs, available: result?.available, reportBytes, operational: report?.operational, + parentEventLoopP95Ms: histogram.percentile(95) / 1e6, parentEventLoopMaxMs: histogram.max / 1e6, parentPeakRssKiB: process.resourceUsage().maxRSS }) + if (outcome.state !== 'succeeded') throw new Error(`Qualification run failed: ${jobs.get(job.id).error}`) + } + phase = 'after' + await new Promise(resolve => setTimeout(resolve, 3000)) + } + finally { + stopped = true + await sampler + socket.terminate() + histogram.disable() + const output = { version: 'synthetic-runner-qualification-v1', fixture, coordinator: coordinator !== null, evaluatorBuildHash, datasetHash: dataset.contentHash, snapshotMs, snapshotEventLoopMaxMs, + canonicalObservationsUnchanged: beforeHash === originalHash(), runs, samples } + writeFileSync(join(directory, 'measurements.json'), JSON.stringify(output, null, 2)) + sqlite.close() + console.log(JSON.stringify({ directory, evaluatorBuildHash, runs, canonicalObservationsUnchanged: output.canonicalObservationsUnchanged })) + } +} +void main().catch((error) => { + console.error(error) + process.exitCode = 1 +}) diff --git a/scripts/benchmarks/biometrics-shadow-runner.ts b/scripts/benchmarks/biometrics-shadow-runner.ts new file mode 100644 index 000000000..e966b85ce --- /dev/null +++ b/scripts/benchmarks/biometrics-shadow-runner.ts @@ -0,0 +1,72 @@ +/** Synthetic-only target-pod shadow supervisor measurement. Bundle with esbuild. + * Arguments: OUTPUT_JSON PYTHON_PATH MODULES_PATH [SECONDS=1800] + * Uses no live inputs, databases, admission registry or hardware interfaces. + * Run alone: the shipped child requires the shared kernel replay/shadow slot. + */ +import { randomUUID, createHash } from 'node:crypto' +import { readFile, writeFile } from 'node:fs/promises' +import { monitorEventLoopDelay } from 'node:perf_hooks' +import { defaultConfig, manifestHash } from '../../src/biometrics/algorithms/registry' +import { configHash } from '../../src/biometrics/algorithms/canonical' +import { runShadowChild } from '../../src/biometrics/shadow/worker-child' +import { probeShadowBuild } from '../../src/biometrics/shadow/build' +import type { ShadowPartialReceipt } from '../../src/biometrics/shadow/receipt' + +async function main() { + const [output, pythonPath, modulesPath, duration = '1800'] = process.argv.slice(2) + const seconds = Number(duration) + if (!output || !pythonPath || !modulesPath || !Number.isInteger(seconds) || seconds < 150 || seconds > 28800) { + throw new Error('Expected OUTPUT_JSON PYTHON_PATH MODULES_PATH and 150–28800 synthetic seconds') + } + const pythonBuild = await probeShadowBuild({ pythonPath, modulesPath }) + const runnerHash = createHash('sha256').update(await readFile(process.argv[1])).digest('hex') + const startMs = 1700000000000, hash = configHash(defaultConfig) + // Measurement stop conditions, not deployment qualification or recommended limits. + const budgets = { cpuSeconds: 1200, addressSpaceBytes: 768 * 1024 * 1024, maximumRssBytes: 384 * 1024 * 1024, + wallMs: 1800000, acknowledgmentMs: 30000, idleMs: 30000, rssIntervalMs: 100, rssTimeoutMs: 1000 } + const init = { schema: 'biometric-shadow-worker-v1', runId: randomUUID(), attemptId: randomUUID(), side: 'left', + startMs, endMs: startMs + seconds * 1000, configs: { left: defaultConfig, right: defaultConfig }, + configHashes: { left: hash, right: hash }, maximumRecords: seconds, maximumOutputs: seconds, + maximumCpuMs: 1200000, maximumInputBytes: seconds * 6500 + 128 * 1024, maximumOutputBytes: seconds * 4096 } + async function* frames() { + for (let second = 0; second < seconds; second++) { + const channel = Buffer.alloc(2000) + for (let sample = 0; sample < 500; sample++) { + const t = second + sample / 500 + channel.writeInt32LE(Math.trunc(1000000 * Math.sin(2 * Math.PI * 1.2 * t)), sample * 4) + } + yield { type: 'frame', sequence: second, observedAtMs: startMs + second * 1000, + record: { type: 'piezo-dual', left1: channel.toString('base64'), right1: Buffer.alloc(2000).toString('base64') }, + context: { activeSides: ['left'], leftOccupied: true, rightOccupied: false, pumpContextKnown: true } } + } + } + const histogram = monitorEventLoopDelay({ resolution: 20 }), digest = createHash('sha256') + let outputCount = 0, partial: ShadowPartialReceipt | null = null, receipt: unknown = null, failure: string | null = null + histogram.enable() + const began = performance.now() + try { + receipt = await runShadowChild({ pythonPath, modulesPath }, init, frames(), (row) => { + outputCount++ + digest.update(JSON.stringify(row)).update('\n') + }, { budgets, onResources: (value) => { partial = value } }) + } + catch (error) { + failure = error instanceof Error ? error.message : String(error) + process.exitCode = 1 + } + finally { + histogram.disable() + await writeFile(output, JSON.stringify({ version: 'synthetic-shadow-supervisor-measurement-v1', + scope: 'accelerated synthetic frames through shipped supervisor and kernel-limited child; no live capture, database sink, held-out accuracy or installation qualification', + runnerHash, pythonBuildHash: pythonBuild.hash, pythonBuildIdentity: pythonBuild.identity, + node: { version: process.version, platform: process.platform, arch: process.arch }, + manifestHash, configHash: hash, seconds, sampleRateHz: 500, channels: 2, wallMs: performance.now() - began, + outputCount, outputHash: digest.digest('hex'), receipt, partial, failure, + parentEventLoopP95Ms: histogram.percentile(95) / 1e6, parentEventLoopMaxMs: histogram.max / 1e6, + parentPeakRssKiB: process.resourceUsage().maxRSS }, null, 2)) + } +} +void main().catch((error) => { + console.error(error) + process.exitCode = 1 +}) diff --git a/scripts/benchmarks/biometrics-waveform-fixture.py b/scripts/benchmarks/biometrics-waveform-fixture.py new file mode 100644 index 000000000..25629f77c --- /dev/null +++ b/scripts/benchmarks/biometrics-waveform-fixture.py @@ -0,0 +1,46 @@ +"""Create bounded deterministic synthetic dual-channel archives, never live data.""" +import argparse +from dataclasses import asdict +import gzip +import json +from pathlib import Path +import time + +import cbor2 +import numpy as np +from common.biometrics.raw_sources import ArchiveSource + +parser = argparse.ArgumentParser() +parser.add_argument('new_directory') +parser.add_argument('--seconds', type=int, default=1800) +args = parser.parse_args() +if not 600 <= args.seconds <= 28800: + raise SystemExit('Fixture duration must be between 600 seconds and 8 hours') +root = Path(args.new_directory) +root.mkdir(mode=0o700) # Never overwrite an existing archive directory. +start = 1767297600000 +began = time.monotonic() +rng = np.random.default_rng(814) +inspections = [] +for segment, offset in enumerate(range(0, args.seconds, 7200)): + name = '%03d.RAW.gz' % segment + with (root / name).open('xb') as output: + with gzip.GzipFile(filename='', fileobj=output, mode='wb', compresslevel=1, mtime=0) as stream: + for second in range(offset, min(offset + 7200, args.seconds)): + timestamp = start / 1000 + second + records = [] + if second % 30 == 0: + records.append({'type': 'frzHealth', 'ts': timestamp, + 'left': {'pumpRpm': 0}, 'right': {'pumpRpm': 0}}) + t = (np.arange(500) + second * 500) / 500 + left = (np.sin(t * 2 * np.pi * 1.2) * 1000000 + rng.normal(0, 100000, 500)).astype(' resolve(value)) + const fixture = z.object({ version: z.literal('synthetic-waveform-fixture-v1'), startMs: z.number(), endMs: z.number(), seconds: z.number(), + generationSeconds: z.number(), inspections: z.array(rawInspectionSchema).min(1).max(4) }).strict().parse(JSON.parse(readFileSync(fixturePath, 'utf8'))) + mkdirSync(directory, { mode: 0o700 }) + setPriority(process.pid, 19) + const sqlite = new Database(join(directory, 'synthetic.db')) + sqlite.pragma('journal_mode = WAL') + const db = drizzle(sqlite, { schema }) + migrate(db, { migrationsFolder: migrations }) + const replayBudget = process.env.SLEEPYPOD_BIOMETRICS_REPLAY_BUDGET === undefined + ? undefined + : replayBudgetSchema.parse(JSON.parse(process.env.SLEEPYPOD_BIOMETRICS_REPLAY_BUDGET)) + const options = { replayBudget, databasePath: join(directory, 'synthetic.db'), workerPath, coordinatorPath, + modulePath: process.env.NODE_PATH, rawLocation: { pythonPath, modulesPath, archiveDirectory } } + const coordinator = new CoordinatorQueue(options) + let retainedDatasetId: string | undefined + const start = fixture.startMs + 300000 + db.insert(schema.sleepRecords).values({ id: 1, side: 'left', enteredBedAt: new Date(start), leftBedAt: new Date(fixture.endMs), + sleepDurationSeconds: (fixture.endMs - start) / 1000, timesExitedBed: 0, presentIntervals: [[start / 1000, fixture.endMs / 1000]] }).run() + db.insert(schema.vitals).values({ side: 'left', timestamp: new Date(start), heartRate: 65 }).run() + const originalRows = JSON.stringify(sqlite.prepare('SELECT * FROM vitals').all()) + const sources = fixture.inspections.map(inspection => new RawSourceCatalog(db).record(inspection)) + let phase = 'baseline', stopped = false, piezoAt: number | undefined + let completed = false, readFailures = 0 + const samples: unknown[] = [], operations: unknown[] = [] + let lastHttpHealthy = false + const socket = new WebSocket('ws://127.0.0.1:3001') + socket.on('open', () => socket.send(JSON.stringify({ type: 'subscribe', sensors: ['piezo-dual'] }))) + socket.on('error', () => {}) + socket.on('message', (bytes) => { + try { + if (JSON.parse(bytes.toString()).type === 'piezo-dual') piezoAt = Date.now() + } + catch { /* Never retain sensor payloads. */ } + }) + const histogram = monitorEventLoopDelay({ resolution: 20 }) + histogram.enable() + const sampler = (async () => { + while (!stopped) { + const sample = await collectAdmissionSample(directory, () => piezoAt) + const requests = [] + for (const endpoint of ['health/performance', 'device/status']) { + const began = performance.now() + try { + const response = await fetch(`http://127.0.0.1:3000/api/${endpoint}`, { signal: AbortSignal.timeout(1500) }) + await response.arrayBuffer() + requests.push({ endpoint, status: response.status, ms: performance.now() - began }) + } + catch { requests.push({ endpoint, status: 0, ms: performance.now() - began }) } + } + let concurrentReadMs: number | undefined + if (phase === 'waveform-replay' && retainedDatasetId && samples.length % 10 === 0) { + const began = performance.now() + try { + await coordinator.request({ type: 'datasetDescriptor', input: { id: retainedDatasetId } }) + } + catch { readFailures++ } + concurrentReadMs = performance.now() - began + } + lastHttpHealthy = readFailures === 0 && requests.every(row => row.status === 200 && row.ms < 1000) + samples.push({ phase, sample, requests, concurrentReadMs, parentRss: process.memoryUsage().rss, loadPerCpu: loadavg()[0] / availableParallelism() }) + await new Promise(resolve => setTimeout(resolve, 500)) + } + })() + // Conservative stop conditions for measurement, not a production qualification policy. + const admit = async () => { + const sample = await collectAdmissionSample(directory, () => piezoAt) + return sample !== null && lastHttpHealthy && sample.availableMemoryBytes >= 512 * 1024 * 1024 + && sample.freeDiskBytes >= 1024 * 1024 * 1024 && sample.temperatureC <= 65 + && Date.now() - sample.piezoReceivedAtMs <= 10000 && sample.eventLoopDelayMs <= 100 + } + async function measure(name: string, action: () => Promise) { + phase = name + histogram.reset() + const began = performance.now() + let succeeded = false + try { + const result = await action() + succeeded = true + return result + } + finally { + await new Promise(resolve => setTimeout(resolve, 50)) + operations.push({ phase, succeeded, elapsedMs: performance.now() - began, eventLoopP95Ms: histogram.percentile(95) / 1e6, + eventLoopMaxMs: histogram.max / 1e6, parentPeakRssKiB: process.resourceUsage().maxRSS }) + } + } + try { + await new Promise(resolve => setTimeout(resolve, 8000)) + if (!await admit()) throw new Error('Live health is unavailable or outside conservative benchmark limits') + const parent = await measure('snapshot', () => coordinator.request({ type: 'createDataset', input: { side: 'left', subjectId: 'synthetic', + sessionIds: [1], referenceImportIds: [], timezone: 'UTC', idempotencyKey: 'synthetic-parent' } })) + const published = await measure('retain', () => coordinator.request({ type: 'createWaveformDataset', input: { + parentDatasetId: parent.id, expectedContentHash: parent.contentHash, idempotencyKey: 'synthetic-waveform', + cohorts: datasetManifestSchema.parse(parent.manifest).splitProtocol.assignments.map(cohort => ({ cohortId: cohort.cohortId, sourceIds: sources })), + } }, { admit })) + if (published.status !== 'published') throw new Error('Waveform retention was deferred') + retainedDatasetId = published.dataset.id + const build = await waveformEvaluatorBuildIdentity(coordinatorPath, workerPath, options.rawLocation) + const jobs = new BacktestJobStore(db) + const experiment = jobs.createExperiment(published.dataset.id, { version: 'descriptive-evaluation-v1', mode: 'signal_replay', waveformStages: true, + assumedStageCalibrationQuality: 0, pointToleranceMs: 30000, clockOffsetMs: 0 }) + const configs = (['default', 'current', 'candidate'] as const).map((label, index) => { + const resolved = structuredClone(defaultConfig) + resolved.heart_rate.trackingToleranceBpm = 15 + index * 2 + return { label, revisionId: null, manifestHash, resolvedHash: configHash(resolved), resolved } + }) + const job = jobs.enqueue({ experimentId: experiment.id, evaluatorBuildHash: build.hash, idempotencyKey: 'synthetic-signal', split: 'tuning', configs, + waveform: { pythonBuildHash: build.python.hash, otherSide: { manifestHash, resolvedHash: configHash(defaultConfig), resolved: defaultConfig } } }) + await measure('waveform-replay', async () => { + const outcome = await coordinator.request({ type: 'execute', owner: 'synthetic-qualification' }, { admit }) + if (outcome.state !== 'succeeded') throw new Error(`Waveform job stopped: ${jobs.get(job.id).error}`) + }) + const summary = await measure('summary', () => coordinator.request({ type: 'readSummary', input: { id: job.id } })) + if (!summary?.available || !summary.contentHash) throw new Error('Committed report summary unavailable') + const report = z.object({ version: z.literal('waveform-backtest-report-v1'), datasetHash: z.string(), + waveform: z.object({ outputs: z.array(z.object({ configHash: z.string(), run: z.object({ build_hash: z.string(), + counts: z.object({ scored_outputs: z.number().int().positive() }) }) })).length(3) }), + }).parse(summary.report) + if (report.datasetHash !== published.dataset.contentHash + || new Set(report.waveform.outputs.map(output => output.configHash)).size !== 3 + || report.waveform.outputs.some(output => output.run.build_hash !== build.python.hash || !configs.some(config => config.resolvedHash === output.configHash))) { + throw new Error('Qualification report identity or configuration coverage mismatch') + } + operations.push({ summaryBytes: Buffer.byteLength(JSON.stringify(summary)), resultHash: summary?.contentHash, + compoundBuildHash: build.hash, pythonBuildHash: build.python.hash }) + if (readFailures) throw new Error('Concurrent dataset reads failed') + completed = true + phase = 'after' + await new Promise(resolve => setTimeout(resolve, 3000)) + } + finally { + stopped = true + await sampler + await coordinator.stop() + socket.terminate() + histogram.disable() + const output = { version: 'synthetic-waveform-qualification-v1', completed, readFailures, replayBudget: replayBudget ?? null, fixtureSeconds: fixture.seconds, compressedBytes: fixture.inspections.reduce((sum, row) => sum + row.artifact.compressed_bytes, 0), + canonicalObservationsUnchanged: originalRows === JSON.stringify(sqlite.prepare('SELECT * FROM vitals').all()), operations, samples } + writeFileSync(join(directory, 'measurements.json'), JSON.stringify(output, null, 2), { mode: 0o600 }) + sqlite.close() + console.log(JSON.stringify({ directory, ...output, samples: output.samples.length })) + } +} +void main().catch((error) => { + console.error(error) + process.exitCode = 1 +}) diff --git a/scripts/build/biometrics-worker.mjs b/scripts/build/biometrics-worker.mjs new file mode 100644 index 000000000..d82a79cae --- /dev/null +++ b/scripts/build/biometrics-worker.mjs @@ -0,0 +1,51 @@ +import { build } from 'esbuild' +import { mkdir, writeFile } from 'node:fs/promises' + +const output = process.argv[2] ?? '.next/standalone/biometrics-worker' +await mkdir(output, { recursive: true }) +const result = await build({ + entryPoints: ['src/biometrics/backtest/worker-entry.ts'], + outfile: `${output}/worker.cjs`, + bundle: true, + platform: 'node', + target: 'node22', + format: 'cjs', + minify: false, + metafile: true, + treeShaking: true, +}) +// Fail packaging if the pure worker ever pulls in live application entrypoints. +const forbidden = Object.keys(result.metafile.inputs).filter(path => + /src\/(hardware|automation|scheduler|homekit|streaming)\//.test(path) + || /src\/db\//.test(path) || /drizzle-orm|better-sqlite3/.test(path), +) +if (forbidden.length) throw new Error(`Replay worker has live dependencies: ${forbidden.join(', ')}`) +await writeFile(`${output}/dependencies.json`, JSON.stringify(Object.keys(result.metafile.inputs).sort(), null, 2)) + +// The coordinator owns SQLite and artifact I/O, but never live application services. +const coordinator = await build({ + entryPoints: ['src/biometrics/backtest/coordinator-entry.ts'], + outfile: `${output}/coordinator.cjs`, bundle: true, platform: 'node', target: 'node22', + format: 'cjs', metafile: true, external: ['better-sqlite3'], +}) +const live = Object.keys(coordinator.metafile.inputs).filter(path => + /src\/(hardware|automation|scheduler|homekit|streaming)\//.test(path) + || /src\/db\/(index|biometrics)\.ts$/.test(path), +) +if (live.length) throw new Error(`Coordinator has live entrypoints: ${live.join(', ')}`) +await writeFile(`${output}/coordinator-dependencies.json`, JSON.stringify(Object.keys(coordinator.metafile.inputs).sort(), null, 2)) + +// The prospective host observes a shared passive tap. Live receive timestamps +// are injected by the service; bundling hardware/streaming entrypoints is forbidden. +const shadow = await build({ + entryPoints: ['src/biometrics/shadow/host-entry.ts'], + outfile: `${output}/shadow-host.cjs`, bundle: true, platform: 'node', target: 'node22', + format: 'cjs', metafile: true, external: ['better-sqlite3'], +}) +const shadowLive = Object.keys(shadow.metafile.inputs).filter(path => + /src\/(hardware|automation|scheduler|homekit)\//.test(path) + || (/src\/streaming\//.test(path) && !path.endsWith('/prospectiveFrameTap.ts')) + || /src\/db\/(index|biometrics)\.ts$/.test(path), +) +if (shadowLive.length) throw new Error(`Shadow host has live entrypoints: ${shadowLive.join(', ')}`) +await writeFile(`${output}/shadow-host-dependencies.json`, JSON.stringify(Object.keys(shadow.metafile.inputs).sort(), null, 2)) diff --git a/scripts/build/demo-biometric-worker.mjs b/scripts/build/demo-biometric-worker.mjs new file mode 100644 index 000000000..c62778d95 --- /dev/null +++ b/scripts/build/demo-biometric-worker.mjs @@ -0,0 +1,8 @@ +import { build } from 'esbuild' +import { createHash } from 'node:crypto' +import { writeFile } from 'node:fs/promises' +const result = await build({ entryPoints: ['src/demo/biometrics/worker.ts'], bundle: true, write: false, platform: 'browser', format: 'esm', metafile: true }) +const forbidden = Object.keys(result.metafile.inputs).filter(path => /src\/(hardware|db|automation|scheduler|homekit|streaming)\/|better-sqlite3|drizzle-orm/.test(path)) +if (forbidden.length) throw new Error(`Demo worker has live dependencies: ${forbidden.join(', ')}`) +const buildHash = createHash('sha256').update(result.outputFiles[0].contents).digest('hex') +await writeFile('src/demo/biometrics/worker-build.json', JSON.stringify({ buildHash, identity: 'esbuild-browser-evaluator-bundle', simulated: true }, null, 2) + '\n') diff --git a/scripts/deploy b/scripts/deploy index 69b656a2f..a840dcbaf 100755 --- a/scripts/deploy +++ b/scripts/deploy @@ -119,7 +119,7 @@ if [ -s .git-info ]; then cp -f .git-info .next/standalone/.git-info; fi # traced into .next/standalone). Keep the standalone server: it embeds the # compiled config and avoids loading build-only plugins at runtime. COPYFILE_DISABLE=1 tar czf "$LOCAL_STAGE/sleepypod-core.tar.gz" \ - --exclude='node_modules' --exclude='.next/cache' \ + --exclude='node_modules' --exclude='.next/cache' --exclude='.venv' --exclude='__pycache__' --exclude='.pytest_cache' \ --exclude='.git' --exclude='.env*' --exclude='*.db' --exclude='*.db-*' \ --exclude='.DS_Store' --exclude='test-results' --exclude='coverage' \ --exclude='.serena' --exclude='.claude' --exclude='.codex' --exclude='.ygg' \ diff --git a/scripts/tests/eslintConfig.test.ts b/scripts/tests/eslintConfig.test.ts new file mode 100644 index 000000000..0ac21c220 --- /dev/null +++ b/scripts/tests/eslintConfig.test.ts @@ -0,0 +1,15 @@ +// @vitest-environment node +import { ESLint } from 'eslint' +import { expect, it } from 'vitest' + +it('recognizes runtime references in JS and TypeScript config files while rejecting unused variables', async () => { + const eslint = new ESLint({ cwd: process.cwd() }) + for (const filePath of ['lint-fixture.mjs', 'lint-fixture.mts']) { + const [used] = await eslint.lintText('const value = 1\nexport default value\n', { filePath }) + expect(used.messages).toEqual([]) + const [unused] = await eslint.lintText('const forgotten = 1\n', { filePath }) + expect(unused.messages).toEqual(expect.arrayContaining([ + expect.objectContaining({ ruleId: '@typescript-eslint/no-unused-vars', severity: 2 }), + ])) + } +}, 20000) diff --git a/src/biometrics/algorithms/canonical-json.ts b/src/biometrics/algorithms/canonical-json.ts new file mode 100644 index 000000000..eddcd59fa --- /dev/null +++ b/src/biometrics/algorithms/canonical-json.ts @@ -0,0 +1,65 @@ +/** + * finite-binary64-json-v1: sorted UTF-16 keys, ordered arrays, exact decimal + * expansion of each finite IEEE-754 number, negative zero normalized to zero. + * Exact expansion avoids differences between Python and JS float printers. + * This is valid numeric JSON, not RFC 8785. Version it in every manifest. + */ +const utf8 = new TextEncoder() + +function numberToken(value: number): string { + if (!Number.isFinite(value)) throw new Error('Non-finite JSON number') + if (value === 0) return '0' + const view = new DataView(new ArrayBuffer(8)) + view.setFloat64(0, Math.abs(value)) + const bits = view.getBigUint64(0) + const exponent = Number((bits >> 52n) & 2047n) + let coefficient = bits & ((1n << 52n) - 1n) + if (exponent !== 0) coefficient += 1n << 52n + const power = (exponent || 1) - 1023 - 52 + let token: string + if (power >= 0) token = (coefficient << BigInt(power)).toString() + else { + const places = -power + const digits = (coefficient * 5n ** BigInt(places)).toString().padStart(places + 1, '0') + token = `${digits.slice(0, -places)}.${digits.slice(-places)}`.replace(/0+$/, '').replace(/\.$/, '') + } + return value < 0 ? `-${token}` : token +} + +export function canonicalWithLimit(value: unknown, maxNodes: number, maxBytes = Infinity): string { + let nodes = 0, bytes = 0 + const token = (text: string) => { + bytes += utf8.encode(text).byteLength + if (bytes > maxBytes) throw new Error('Canonical data byte limit') + return text + } + function visit(item: unknown, depth: number): string { + if (++nodes > maxNodes || depth > 32) throw new Error('JSON resource limit') + if (item === null) return token('null') + if (typeof item === 'number') return token(numberToken(item)) + if (typeof item === 'boolean') return token(String(item)) + if (typeof item === 'string') { + if (item.length > 65536 || /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(? maxNodes - nodes) throw new Error('JSON resource limit') + token('[' + ','.repeat(Math.max(0, item.length - 1)) + ']') + return `[${Array.from(item, child => visit(child, depth + 1)).join(',')}]` + } + if (typeof item === 'object' && (Object.getPrototypeOf(item) === Object.prototype || Object.getPrototypeOf(item) === null)) { + const keys = Object.keys(item) + if (keys.length * 2 > maxNodes - nodes) throw new Error('JSON resource limit') + keys.sort() + token('{' + ':'.repeat(keys.length) + ','.repeat(Math.max(0, keys.length - 1)) + '}') + return `{${keys.map(key => `${visit(key, depth + 1)}:${visit((item as Record)[key], depth + 1)}`).join(',')}}` + } + throw new Error('Expected finite JSON data') + } + return visit(value, 0) +} + +export function canonicalJson(value: unknown): string { + return canonicalWithLimit(value, 20000) +} + diff --git a/src/biometrics/algorithms/canonical.ts b/src/biometrics/algorithms/canonical.ts new file mode 100644 index 000000000..3fc4b0c2f --- /dev/null +++ b/src/biometrics/algorithms/canonical.ts @@ -0,0 +1,13 @@ +import { createHash } from 'node:crypto' +import { canonicalJson, canonicalWithLimit } from './canonical-json' +export { canonicalJson } from './canonical-json' + +/** Same canonical encoding; larger bounded budget for retained rows/reports. */ +export function dataHash(value: unknown): string { + const encoded = canonicalWithLimit(value, 2000000, 32 * 1024 * 1024) + return createHash('sha256').update(encoded).digest('hex') +} + +export function configHash(value: unknown): string { + return createHash('sha256').update(canonicalJson(value)).digest('hex') +} diff --git a/src/biometrics/algorithms/registry-core.ts b/src/biometrics/algorithms/registry-core.ts new file mode 100644 index 000000000..2f62a2f65 --- /dev/null +++ b/src/biometrics/algorithms/registry-core.ts @@ -0,0 +1,135 @@ +import contract from '../../../modules/common/biometrics/manifest.v1.json' +import type { StageClassifier } from '@/src/lib/sleep-stage-rules' +import { canonicalJson } from './canonical-json' + +interface ParameterSchema { + type?: string + properties?: Record + required?: string[] + additionalProperties?: boolean + items?: ParameterSchema + maxItems?: number + minimum?: number + maximum?: number + const?: unknown + enum?: unknown[] +} + +export interface BiometricConfig { + heart_rate: { trackingToleranceBpm: number, trackingHistoryLength: number } + breathing_rate: { intervalLowerRatio: number, intervalUpperRatio: number } + hrv_index: { harmonicTolerance: number, minimumWindows: number } + sleep_stages: StageClassifier +} +export interface Override { path: string, value: unknown } + +function deepFreeze(value: T): T { + if (value && typeof value === 'object') { + for (const child of Object.values(value)) deepFreeze(child) + Object.freeze(value) + } + return value +} + +export const biometricManifest = deepFreeze(contract) +export const defaultConfig = biometricManifest.defaults as BiometricConfig + +/** Validated adapters implement this deliberately small shared schema dialect. */ +function validateNode(schema: ParameterSchema, value: unknown, path: string): void { + const fail = (): never => { + throw new Error(`Invalid biometric parameter: ${path}`) + } + if (schema.const !== undefined && value !== schema.const) fail() + if (schema.enum && !schema.enum.includes(value)) fail() + switch (schema.type) { + case 'object': { + if (!value || typeof value !== 'object' || Array.isArray(value)) return fail() + const record = value as Record + const properties = schema.properties ?? {} + for (const key of schema.required ?? []) if (!Object.hasOwn(record, key)) fail() + for (const [key, child] of Object.entries(record)) { + if (!Object.hasOwn(properties, key)) fail() + validateNode(properties[key], child, `${path}.${key}`) + } + break + } + case 'array': + if (!Array.isArray(value) || value.length > (schema.maxItems ?? Infinity)) return fail() + if (!schema.items) throw new Error('Missing array schema') + for (const child of value) validateNode(schema.items, child, path) + break + case 'number': + case 'integer': + if (typeof value !== 'number' || !Number.isFinite(value) + || (schema.type === 'integer' && !Number.isSafeInteger(value)) + || value < (schema.minimum ?? -Infinity) || value > (schema.maximum ?? Infinity)) fail() + break + case 'string': + if (typeof value !== 'string') fail() + break + case 'boolean': + if (typeof value !== 'boolean') fail() + break + default: throw new Error(`Unsupported contract schema at ${path}`) + } +} + +function atPath(value: unknown, path: string[]): unknown { + return path.reduce((parent, key) => (parent as Record)[key], value) +} + +export function validateConfig(value: unknown): BiometricConfig { + canonicalJson(value) // finite JSON, depth/size bounds, no class instances + validateNode(biometricManifest.parameters, value, 'config') + for (const path of biometricManifest.orderedRanges) { + const range = atPath(value, path) as { min: number, max: number } + if (range.min > range.max) throw new Error(`Invalid ordered range: ${path.join('.')}`) + } + return value as BiometricConfig +} + +/** Scalars inherit independently; ordered arrays are replaced whole. */ +export function resolveOverrides(overrides: Override[]) { + if (overrides.length > 64) throw new Error('RESOURCE_LIMIT: maximum 64 overrides') + const resolved = structuredClone(defaultConfig) + const paths = new Set() + for (const override of overrides) { + if (Object.keys(override).some(k => k !== 'path' && k !== 'value')) throw new Error('Unknown override field') + const parts = override.path.split('.') + if (paths.has(override.path)) throw new Error('Duplicate override path') + paths.add(override.path) + let schema: ParameterSchema = biometricManifest.parameters + let parent = resolved as unknown as Record + for (const [i, part] of parts.entries()) { + if (schema.type !== 'object' || !schema.properties || !Object.hasOwn(schema.properties, part)) throw new Error(`Unknown override path: ${override.path}`) + schema = schema.properties[part] + if (i === parts.length - 1) { + if (schema.type === 'object') throw new Error('Override leaves or entire ordered arrays') + validateNode(schema, override.value, override.path) + parent[part] = structuredClone(override.value) + } + else parent = parent[part] as Record + } + } + validateConfig(resolved) + return { resolved: deepFreeze(resolved), overrides: structuredClone(overrides), provenance: Object.fromEntries(overrides.map(o => [o.path, 'override' as const])) } +} + +/** Removing a path restores inheritance; null is never a removal sentinel. */ +export function removeOverride(overrides: Override[], path: string): Override[] { + return overrides.filter(override => override.path !== path) +} + +/** Migrate every legacy leaf, including explicit defaults, to preserve intent. */ +export function legacyStageOverrides(classifier: StageClassifier): Override[] { + const result: Override[] = [] + function walk(value: unknown, path: string) { + if (value !== null && typeof value === 'object' && !Array.isArray(value)) { + for (const [key, child] of Object.entries(value)) walk(child, `${path}.${key}`) + } + else result.push({ path, value }) + } + walk(classifier, 'sleep_stages') + resolveOverrides(result) + return result +} diff --git a/src/biometrics/algorithms/registry.ts b/src/biometrics/algorithms/registry.ts new file mode 100644 index 000000000..0dc8371ff --- /dev/null +++ b/src/biometrics/algorithms/registry.ts @@ -0,0 +1,13 @@ +import { configHash } from './canonical' +import { biometricManifest, resolveOverrides, type Override } from './registry-core' +export { biometricManifest, defaultConfig, validateConfig, removeOverride, legacyStageOverrides } from './registry-core' +export type { BiometricConfig, Override } from './registry-core' + +export const manifestHash = configHash(biometricManifest) + +/** Resolve against the exact shipped manifest, retaining the canonical identity. */ +export function resolveConfig(overrides: Override[], baseHash = manifestHash) { + if (baseHash !== manifestHash) throw new Error('PROFILE_INCOMPATIBLE: unknown base manifest') + const result = resolveOverrides(overrides) + return { ...result, hash: configHash(result.resolved), manifestHash: baseHash } +} diff --git a/src/biometrics/algorithms/tests/data-hash.test.ts b/src/biometrics/algorithms/tests/data-hash.test.ts new file mode 100644 index 000000000..c5f411d87 --- /dev/null +++ b/src/biometrics/algorithms/tests/data-hash.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it } from 'vitest' +import { configHash, dataHash } from '../canonical' + +describe('bounded retained-data hashing', () => { + it('keeps canonical identity while permitting realistic multi-night data beyond config budgets', () => { + const small = { timestampMs: 1791673200000, heartRate: 60.1 } + expect(dataHash(small)).toBe(configHash(small)) + const large = Array.from({ length: 10000 }, (_, id) => ({ id, ...small })) + expect(() => configHash(large)).toThrow('resource limit') + expect(dataHash(large)).toMatch(/^[a-f0-9]{64}$/) + expect(dataHash(large)).toBe(dataHash(JSON.parse(JSON.stringify(large)))) + }) + it('continues to reject nonfinite and excessively deep data', () => { + expect(() => dataHash({ value: Infinity })).toThrow('Non-finite') + let nested: unknown = 0 + for (let index = 0; index < 40; index++) nested = [nested] + expect(() => dataHash(nested)).toThrow('resource limit') + expect(() => dataHash(new Array(3000000))).toThrow('resource limit') + expect(() => dataHash(Array(600).fill('x'.repeat(65536)))).toThrow('byte limit') + }) +}) diff --git a/src/biometrics/algorithms/tests/registry.test.ts b/src/biometrics/algorithms/tests/registry.test.ts new file mode 100644 index 000000000..35d690b9a --- /dev/null +++ b/src/biometrics/algorithms/tests/registry.test.ts @@ -0,0 +1,148 @@ +// @vitest-environment node +import { execFileSync } from 'node:child_process' +import { createHash } from 'node:crypto' +import { readFileSync } from 'node:fs' +import { describe, expect, it } from 'vitest' +import { canonicalJson, configHash } from '../canonical' +import { biometricManifest, defaultConfig, legacyStageOverrides, manifestHash, removeOverride, resolveConfig, validateConfig } from '../registry' +import { DEFAULT_STAGE_CLASSIFIER, stageClassifierSchema } from '@/src/lib/sleep-stage-rules' + +const fixtures = [ + [], + [{ path: 'heart_rate.trackingToleranceBpm', value: 20 }], + [{ path: 'sleep_stages.ruleset.smoothIsolatedEpochs', value: false }, { path: 'sleep_stages.ruleset.lowSignalBelow', value: 0 }], + [{ path: 'sleep_stages.ruleset.rules', value: [{ stage: 'wake', when: [] }, { stage: 'light', when: [] }] }], + [{ path: 'sleep_stages.ruleset.rules', value: [] }], + [{ path: 'heart_rate.trackingHistoryLength', value: 2.5 }], + [{ path: 'heart_rate.trackingToleranceBpm', value: true }], + [{ path: 'sleep_stages.ruleset.outliers.heartRate.min', value: 140 }], + [{ path: 'sleep_stages.ruleset.rules.0.stage', value: 'rem' }], + [{ path: 'sleep_stages.ruleset.rules', value: [{ stage: 'unknown', when: [] }] }], + [{ path: 'sleep_stages.ruleset.fallback', value: null }], + [{ path: 'heart_rate', value: { trackingToleranceBpm: 10 } }], + [{ path: 'heart_rate.trackingToleranceBpm', value: 10 }, { path: 'heart_rate.trackingToleranceBpm', value: 12 }], + [{ path: '__proto__.polluted', value: true }], + [{ path: 'sleep_stages.ruleset.lowSignalBelow', value: 1.01 }], + [{ path: 'sleep_stages.ruleset.rules', value: Array(33).fill({ stage: 'wake', when: [] }) }], +] + +describe('versioned biometric contract', () => { + it('pins the actual numerical source and locked dependency files', () => { + for (const [path, hash] of Object.entries({ ...biometricManifest.sourceHashes, ...biometricManifest.dependencyHashes })) { + expect(createHash('sha256').update(readFileSync(path)).digest('hex'), path).toBe(hash) + } + }) + + it('preserves the shipped classifier and labels HRV correctly', () => { + expect(resolveConfig([]).resolved.sleep_stages).toEqual(DEFAULT_STAGE_CLASSIFIER) + expect(stageClassifierSchema.parse(defaultConfig.sleep_stages)).toEqual(defaultConfig.sleep_stages) + expect(biometricManifest.components.hrv_index.metricId).toBe('window_ibi_variability_v1') + expect(biometricManifest.components.sleep_stages.causal).toBe(false) + expect(validateConfig(defaultConfig)).toEqual(defaultConfig) + }) + + it('keeps false/zero intent, replaces ordered arrays, and removes overrides explicitly', () => { + const overrides = fixtures[2] + const profile = resolveConfig(overrides) + expect(profile.resolved.sleep_stages.ruleset.lowSignalBelow).toBe(0) + expect(profile.resolved.sleep_stages.ruleset.smoothIsolatedEpochs).toBe(false) + expect(resolveConfig(removeOverride(overrides, 'sleep_stages.ruleset.lowSignalBelow')).resolved.sleep_stages.ruleset.lowSignalBelow).toBe(0.3) + expect(resolveConfig(fixtures[4]).resolved.sleep_stages.ruleset.rules).toEqual([]) + const reversed = structuredClone(fixtures[3]) + ;(reversed[0].value as unknown[]).reverse() + expect(resolveConfig(reversed).hash).not.toBe(resolveConfig(fixtures[3]).hash) + expect(() => { + profile.resolved.heart_rate.trackingToleranceBpm = 25 + }).toThrow() + }) + + it('preserves every legacy leaf and first-match rule ordering', () => { + const legacy = structuredClone(DEFAULT_STAGE_CLASSIFIER) + legacy.ruleset.rules.reverse() + legacy.ruleset.smoothIsolatedEpochs = false + const result = resolveConfig(legacyStageOverrides(legacy)) + expect(result.resolved.sleep_stages).toEqual(legacy) + expect(result.resolved.heart_rate).toEqual(defaultConfig.heart_rate) + expect(() => resolveConfig([], 'unknown-engine')).toThrow('PROFILE_INCOMPATIBLE') + }) + + it('rejects every invalid contract fixture', () => { + for (const fixture of fixtures.slice(5)) expect(() => resolveConfig(fixture)).toThrow() + expect(() => validateConfig({ ...defaultConfig, sensorCalibration: {} })).toThrow() + }) + + it('matches Python validation and configuration hashes on the same fixtures', () => { + const python = execFileSync('python3', ['-c', ` +import json, sys +sys.path.insert(0, 'modules') +from common.biometrics.config import resolve_config +results = [] +for fixture in json.load(sys.stdin): + try: + result = resolve_config(fixture) + results.append({'hash': result['hash'], 'manifestHash': result['manifestHash']}) + except ValueError: + results.append(None) +print(json.dumps(results)) +`], { input: JSON.stringify(fixtures), encoding: 'utf8' }) + expect(JSON.parse(python)).toEqual(fixtures.map((fixture) => { + try { + const result = resolveConfig(fixture) + return { hash: result.hash, manifestHash } + } + catch { return null } + })) + }) +}) + +describe('finite-binary64-json-v1', () => { + it('uses deterministic key order and finite numeric JSON, preserving array order', () => { + expect(canonicalJson({ b: -0, a: 0.5 })).toBe('{"a":0.5,"b":0}') + expect(configHash({ a: 1, b: 2 })).toBe(configHash({ b: 2, a: 1 })) + for (const invalid of [NaN, Infinity, undefined, new Date(), { a: undefined }, '\ud800', Array(2)]) { + expect(() => canonicalJson(invalid)).toThrow() + } + expect(() => canonicalJson(Array(20001).fill(null))).toThrow('resource limit') + let deep: unknown = null + for (let i = 0; i < 33; i++) deep = [deep] + expect(() => canonicalJson(deep)).toThrow('resource limit') + }) + + it('matches Python for float edge cases, control characters, and Unicode key order', () => { + const values = [0.1, -0, 1e-7, 1e21, Number.MIN_VALUE, Number.MAX_VALUE, 1.23456789012345, + { '\u{10000}': '🌙\n\t', '\ue000': 'é', '\u0000': '\b\f\r' }] + const output = execFileSync('python3', ['-c', ` +import json, sys +sys.path.insert(0, 'modules') +from common.biometrics.config import canonical_json +print(json.dumps([canonical_json(v) for v in json.load(sys.stdin)], ensure_ascii=False)) +`], { input: JSON.stringify(values), encoding: 'utf8' }) + expect(JSON.parse(output)).toEqual(values.map(canonicalJson)) + for (const value of values) expect(JSON.parse(canonicalJson(value))).toEqual(value === 0 ? 0 : value) + }) +}) + +it.each([ + ['wrong object type', { ...defaultConfig, heart_rate: [] }], + ['missing required property', { ...defaultConfig, heart_rate: { trackingToleranceBpm: 15 } }], + ['null component', { ...defaultConfig, heart_rate: null }], + ['wrong contract version', { ...defaultConfig, sleep_stages: { ...defaultConfig.sleep_stages, version: 99 } }], +])('rejects %s instead of silently filling an immutable snapshot', (_reason, config) => { + expect(() => validateConfig(config)).toThrow() +}) +it.each([ + { path: 'heart_rate.trackingToleranceBpm', value: 4 }, + { path: 'heart_rate.trackingToleranceBpm', value: 31 }, + { path: 'sleep_stages.ruleset.smoothIsolatedEpochs', value: 1 }, + { path: 'sleep_stages.ruleset.rules', value: {} }, + { path: 'sleep_stages.ruleset.rules', value: [null] }, + { path: 'sleep_stages.ruleset.rules', value: [{ stage: 'wake' }] }, + { path: '', value: 1 }, + { path: 'heart_rate.trackingToleranceBpm', value: 15, extra: true }, +])('rejects malformed override $path with value $value', (override) => { + expect(() => resolveConfig([override])).toThrow() +}) +it('bounds override intake before resolving or mutating defaults', () => { + expect(() => resolveConfig(Array.from({ length: 65 }, () => ({ path: 'heart_rate.trackingToleranceBpm', value: 15 })))).toThrow('maximum 64') + expect(defaultConfig.heart_rate.trackingToleranceBpm).toBe(15) +}) diff --git a/src/biometrics/background-admission.ts b/src/biometrics/background-admission.ts new file mode 100644 index 000000000..9299d2b75 --- /dev/null +++ b/src/biometrics/background-admission.ts @@ -0,0 +1,35 @@ +import { ReferenceInputError } from './references/time' + +type Lane = 'read' | 'work' +interface Slot { token: symbol, quarantined: boolean } +const shared = globalThis as typeof globalThis & { __biometricBackgroundSlotsV1?: Partial> } + +/** Application-process admission, shared across separately bundled modules. + * Kernel numerical-worker locking remains necessary across processes/restarts. */ +export function withBiometricBackgroundSlot(lane: Lane, action: () => Promise): Promise { + const slots = shared.__biometricBackgroundSlotsV1 ??= {} + if (slots[lane]) return Promise.reject(new ReferenceInputError('RESOURCE_LIMIT', slots[lane].quarantined + ? 'Biometrics background cleanup is unconfirmed; restart after verifying process cleanup' + : 'Biometrics background capacity is busy; retry later')) + const slot = { token: Symbol(lane), quarantined: false } + slots[lane] = slot + const release = (error?: unknown) => { + if (slots[lane]?.token !== slot.token) return + const code = error && typeof error === 'object' && 'code' in error ? error.code : null + if (code === 'SHADOW_CLEANUP_UNCONFIRMED' || code === 'BIOMETRIC_COORDINATOR_CLEANUP_UNCONFIRMED') slot.quarantined = true + else slots[lane] = undefined + } + try { + return action().then((result) => { + release() + return result + }, (error: unknown) => { + release(error) + throw error + }) + } + catch (error) { + release(error) + throw error + } +} diff --git a/src/biometrics/backtest/admission-service.ts b/src/biometrics/backtest/admission-service.ts new file mode 100644 index 000000000..6c19726a4 --- /dev/null +++ b/src/biometrics/backtest/admission-service.ts @@ -0,0 +1,59 @@ +import type { HealthAdmission } from './job-operations' +import { dirname } from 'node:path' +import { sql } from 'drizzle-orm' +import { z } from 'zod' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import type * as schema from '@/src/db/biometrics-schema' +import { admissionPolicySchema, collectAdmissionSample, evaluateAdmission } from './admission' + +export const replayAdmissionSchema = z.object({ + configured: z.boolean(), qualificationId: z.string().nullable(), admitted: z.boolean(), + reasons: z.array(z.enum(['unqualified', 'health_unavailable', 'stale_sample', 'clock_skew', + 'memory_pressure', 'disk_pressure', 'cpu_pressure', 'event_loop_delay', 'piezo_stale', 'temperature'])), +}).strict() + +/** Server-owned policy only. Missing or malformed configuration never enables + * background I/O. The qualification ID is operator-supplied evidence attribution, + * not an assertion that supplying JSON has qualified the hardware. + */ +async function measuredReplayAdmission(db: BetterSQLite3Database): Promise & { receipt?: Awaited> }> { + const policy = configuredReplayPolicy() + if (!policy) return { configured: false, qualificationId: null, admitted: false, reasons: ['unqualified'] } + try { + const path = db.all<{ name: string, file: string }>(sql`PRAGMA database_list`).find(row => row.name === 'main')?.file + if (!path) throw new Error('Missing persistent database') + const { getSensorFrameTimes } = await import('@/src/streaming/piezoStream') + const sample = await collectAdmissionSample(dirname(path), () => getSensorFrameTimes()['piezo-dual']) + const checkedAtMs = Date.now() + const decision = evaluateAdmission(sample, policy, checkedAtMs) + return { configured: true, qualificationId: policy.qualificationId, ...decision, + receipt: { allowed: decision.admitted, sample: decision.admitted && sample + ? { qualificationId: policy.qualificationId, checkedAtMs, ingestionAgeMs: checkedAtMs - sample.piezoReceivedAtMs } + : null } } + } + catch { return { configured: true, qualificationId: policy.qualificationId, admitted: false, reasons: ['health_unavailable'] } } +} + +export async function replayAdmission(db: BetterSQLite3Database) { + const { configured, qualificationId, admitted, reasons } = await measuredReplayAdmission(db) + return replayAdmissionSchema.parse({ configured, qualificationId, admitted, reasons }) +} + +export function replayHealthAdmission(db: BetterSQLite3Database): HealthAdmission { + return async () => { + const measured = await measuredReplayAdmission(db) + return measured.receipt ?? { allowed: false, sample: null } + } +} + +export function replayAdmitter(db: BetterSQLite3Database) { + return async () => (await replayAdmission(db)).admitted +} + +/** Reading configuration has no database, streaming or health side effects. */ +export function configuredReplayPolicy() { + try { + return admissionPolicySchema.parse(JSON.parse(process.env.SLEEPYPOD_BIOMETRICS_ADMISSION_POLICY ?? 'null')) + } + catch { return null } +} diff --git a/src/biometrics/backtest/admission.ts b/src/biometrics/backtest/admission.ts new file mode 100644 index 000000000..fdcaec507 --- /dev/null +++ b/src/biometrics/backtest/admission.ts @@ -0,0 +1,82 @@ +import { readFile, statfs } from 'node:fs/promises' +import { availableParallelism, loadavg } from 'node:os' +import { performance } from 'node:perf_hooks' +import { z } from 'zod' + +/** Limits must come from an explicitly qualified deployment policy. There is + * no fallback that silently enables replay on an unmeasured device. */ +export const admissionPolicySchema = z.object({ + qualificationId: z.string().min(1).max(200), + minimumAvailableMemoryBytes: z.number().int().positive(), + minimumFreeDiskBytes: z.number().int().positive(), + maximumLoadPerCpu: z.number().positive(), + maximumEventLoopDelayMs: z.number().positive(), + maximumPiezoAgeMs: z.number().int().positive(), + maximumTemperatureC: z.number().positive(), + maximumSampleAgeMs: z.number().int().positive(), +}).strict() +export type AdmissionPolicy = z.infer +const sampleSchema = z.object({ + measuredAtMs: z.number().int().nonnegative(), + availableMemoryBytes: z.number().int().nonnegative(), + freeDiskBytes: z.number().int().nonnegative(), + loadPerCpu: z.number().nonnegative(), + eventLoopDelayMs: z.number().nonnegative(), + piezoReceivedAtMs: z.number().int().nonnegative(), + temperatureC: z.number().min(-50).max(150), +}).strict() +export type AdmissionSample = z.infer +export type AdmissionReason = 'unqualified' | 'health_unavailable' | 'stale_sample' | 'clock_skew' + | 'memory_pressure' | 'disk_pressure' | 'cpu_pressure' | 'event_loop_delay' | 'piezo_stale' | 'temperature' + +export function evaluateAdmission(rawSample: unknown, rawPolicy: unknown, now = Date.now()): { admitted: boolean, reasons: AdmissionReason[] } { + const policy = admissionPolicySchema.safeParse(rawPolicy) + if (!policy.success) return { admitted: false, reasons: ['unqualified'] } + const sample = sampleSchema.safeParse(rawSample) + if (!sample.success || !Number.isSafeInteger(now) || now < 0) return { admitted: false, reasons: ['health_unavailable'] } + const s = sample.data, p = policy.data + const reasons: AdmissionReason[] = [] + if (s.measuredAtMs > now || s.piezoReceivedAtMs > now) reasons.push('clock_skew') + if (now - s.measuredAtMs > p.maximumSampleAgeMs) reasons.push('stale_sample') + if (s.availableMemoryBytes < p.minimumAvailableMemoryBytes) reasons.push('memory_pressure') + if (s.freeDiskBytes < p.minimumFreeDiskBytes) reasons.push('disk_pressure') + if (s.loadPerCpu > p.maximumLoadPerCpu) reasons.push('cpu_pressure') + if (s.eventLoopDelayMs > p.maximumEventLoopDelayMs) reasons.push('event_loop_delay') + if (now - s.piezoReceivedAtMs > p.maximumPiezoAgeMs) reasons.push('piezo_stale') + if (s.temperatureC > p.maximumTemperatureC) reasons.push('temperature') + return { admitted: reasons.length === 0, reasons } +} + +export function parseAvailableMemory(text: string) { + const match = /^MemAvailable:\s+(\d+)\s+kB\s*$/m.exec(text) + if (!match) throw new Error('Available memory is unavailable') + const bytes = Number(match[1]) * 1024 + if (!Number.isSafeInteger(bytes) || bytes < 0) throw new Error('Invalid available memory') + return bytes +} + +/** No hardware commands, database queries, or lifetime histogram. Measure the + * scheduling delay of this process and only the live piezo frame timestamp. */ +export async function collectAdmissionSample(databaseDirectory: string, piezoReceivedAt: () => number | undefined): Promise { + const measuredAtMs = Date.now() + try { + // Probe Linux health files directly. Turbopack folds both process.platform + // and os.platform() to the build host; missing runtime files fail closed. + const probeStarted = performance.now() + const [memory, disk, thermal, delay] = await Promise.all([ + readFile('/proc/meminfo', 'utf8'), + statfs(databaseDirectory), + readFile('/sys/class/thermal/thermal_zone0/temp', 'utf8'), + new Promise(resolve => setTimeout(() => resolve(Math.max(0, performance.now() - probeStarted - 20)), 20)), + ]) + const rawTemperature = thermal.trim() + if (!/^-?\d+$/.test(rawTemperature)) return null + const parsed = sampleSchema.safeParse({ + measuredAtMs, availableMemoryBytes: parseAvailableMemory(memory), freeDiskBytes: disk.bavail * disk.bsize, + loadPerCpu: loadavg()[0] / availableParallelism(), eventLoopDelayMs: delay, + piezoReceivedAtMs: piezoReceivedAt(), temperatureC: Number(rawTemperature) / 1000, + }) + return parsed.success ? parsed.data : null + } + catch { return null } +} diff --git a/src/biometrics/backtest/artifacts.ts b/src/biometrics/backtest/artifacts.ts new file mode 100644 index 000000000..b82330bb9 --- /dev/null +++ b/src/biometrics/backtest/artifacts.ts @@ -0,0 +1,161 @@ +import Database from 'better-sqlite3' +import { createHash, randomUUID } from 'node:crypto' +import { closeSync, fsyncSync, lstatSync, mkdirSync, openSync, readFileSync, readdirSync, renameSync, unlinkSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { z } from 'zod' +import { ReferenceInputError } from '../references/time' + +export const REPORT_BUDGET_BYTES = 64 * 1024 * 1024 +export const MAX_REPORT_BYTES = 16 * 1024 * 1024 +export const reportArtifactSchema = z.object({ kind: z.literal('biometric-report-artifact-v1'), + name: z.string().regex(/^[a-f0-9-]{36}\.[1-9][0-9]*\.[a-f0-9]{64}\.json$/), + byteLength: z.number().int().positive().max(MAX_REPORT_BYTES), sha256: z.string().regex(/^[a-f0-9]{64}$/), +}).strict() +export type ReportArtifact = z.infer +const ownedName = /^[a-f0-9-]{36}\.[1-9][0-9]*\.[a-f0-9]{64}\.json(?:\.[a-f0-9-]{36}\.tmp)?$/ + +/** Trusted evaluator artifacts only. The root comes from the server's DB path, + * never a request. Files are immutable; a fenced DB transaction publishes the + * descriptor only after fsync + atomic rename. Failed publication leaves an + * unreferenced artifact for bounded cleanup, never a partially visible result. */ +export class ReportArtifactStore { + private publishing = false + constructor(private readonly root: string) {} + + /** A separate SQLite file provides an OS-released cross-process lock. Long + * file writes never hold the live biometrics DB transaction. All publishers + * and collectors take this lock before reading live descriptor/quota state. */ + withPublication(operation: () => T): T { + if (this.publishing) return operation() + mkdirSync(this.root, { recursive: true, mode: 0o700 }) + const lock = new Database(join(this.root, '.publication.sqlite'), { timeout: 1000 }) + try { + return lock.transaction(() => { + this.publishing = true + try { + return operation() + } + finally { + this.publishing = false + } + }).immediate() + } + finally { + lock.close() + } + } + + usage() { + mkdirSync(this.root, { recursive: true, mode: 0o700 }) + const names = readdirSync(this.root) + if (names.length > 4096) throw new ReferenceInputError('RESOURCE_LIMIT', 'Too many report artifacts') + let bytes = 0 + for (const name of names) { + if (!ownedName.test(name)) continue + const stat = lstatSync(join(this.root, name)) + if (!stat.isFile()) throw new ReferenceInputError('RESULT_CORRUPT', 'Unexpected report artifact type') + bytes += stat.size + } + return bytes + } + + write(jobId: string, token: number, serialized: string, inlineBytes: number): ReportArtifact { + return this.withPublication(() => { + z.string().uuid().parse(jobId) + z.number().int().positive().parse(token) + z.number().int().nonnegative().parse(inlineBytes) + const byteLength = Buffer.byteLength(serialized) + if (!byteLength || byteLength > MAX_REPORT_BYTES) throw new ReferenceInputError('RESOURCE_LIMIT', 'Report exceeds 16 MiB') + const sha256 = createHash('sha256').update(serialized).digest('hex') + const name = `${jobId}.${token}.${sha256}.json` + const descriptor = reportArtifactSchema.parse({ kind: 'biometric-report-artifact-v1', name, byteLength, sha256 }) + const used = this.usage() + try { + const stat = lstatSync(join(this.root, name)) + if (stat.isFile() && stat.size === byteLength) { + const bytes = readFileSync(join(this.root, name)) + if (createHash('sha256').update(bytes).digest('hex') === sha256) { + if (used + inlineBytes > REPORT_BUDGET_BYTES) throw new ReferenceInputError('RESOURCE_LIMIT', 'Reports reached the 64 MiB storage budget') + return descriptor + } + } + } + catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error + } + if (used + byteLength + inlineBytes > REPORT_BUDGET_BYTES) throw new ReferenceInputError('RESOURCE_LIMIT', 'Reports reached the 64 MiB storage budget') + const temporary = join(this.root, `${name}.${randomUUID()}.tmp`) + let fd: number | undefined + try { + fd = openSync(temporary, 'wx', 0o600) + writeFileSync(fd, serialized) + fsyncSync(fd) + closeSync(fd) + fd = undefined + renameSync(temporary, join(this.root, name)) + const directory = openSync(this.root, 'r') + try { + fsyncSync(directory) + } + finally { + closeSync(directory) + } + return descriptor + } + finally { + if (fd !== undefined) closeSync(fd) + try { + unlinkSync(temporary) + } + catch { /* Atomic rename or cleanup after a failed write. */ } + } + }) + } + + read(raw: unknown): unknown { + const parsed = reportArtifactSchema.safeParse(raw) + if (!parsed.success) throw new ReferenceInputError('RESULT_CORRUPT', 'Invalid report artifact descriptor') + const descriptor = parsed.data + try { + const path = join(this.root, descriptor.name) + const stat = lstatSync(path) + if (!stat.isFile() || stat.size !== descriptor.byteLength) throw new Error('Artifact size/type mismatch') + const bytes = readFileSync(path) + if (createHash('sha256').update(bytes).digest('hex') !== descriptor.sha256) throw new Error('Artifact hash mismatch') + return JSON.parse(bytes.toString('utf8')) + } + catch { + throw new ReferenceInputError('RESULT_CORRUPT', 'Report artifact is missing or corrupt') + } + } + + /** Called only after the descriptor has been detached in a committed DB + * transaction under the publication lock. Failed unlink remains GC work. */ + removeReleased(raw: ReportArtifact) { + const descriptor = reportArtifactSchema.parse(raw) + if (!this.publishing) throw new Error('Artifact release requires the publication lock') + try { + unlinkSync(join(this.root, descriptor.name)) + } + catch { /* Retained orphan remains counted against the filesystem quota. */ } + } + + /** Protect freshly staged files from a concurrent publisher. Only files + * outside the committed descriptor set and older than one hour are removed. */ + collect(referenced: ReadonlySet, now = Date.now()) { + return this.withPublication(() => { + this.usage() + let removed = 0 + for (const name of readdirSync(this.root)) { + if (!ownedName.test(name) || referenced.has(name)) continue + const path = join(this.root, name) + const stat = lstatSync(path) + if (stat.isFile() && now - stat.mtimeMs > 3600000) { + unlinkSync(path) + removed++ + } + } + return removed + }) + } +} diff --git a/src/biometrics/backtest/child.ts b/src/biometrics/backtest/child.ts new file mode 100644 index 000000000..33e70ddf5 --- /dev/null +++ b/src/biometrics/backtest/child.ts @@ -0,0 +1,88 @@ +import { createHash } from 'node:crypto' +import { readFile } from 'node:fs/promises' +import { spawn } from 'node:child_process' +import { StringDecoder } from 'node:string_decoder' +import { tmpdir, setPriority } from 'node:os' +import { z } from 'zod' + +const eventSchema = z.discriminatedUnion('type', [ + z.object({ type: z.literal('progress'), completedNights: z.number().int().min(0).max(60), totalNights: z.number().int().min(0).max(60) }).strict(), + z.object({ type: z.literal('result'), report: z.object({ version: z.enum(['processed-backtest-report-v1', 'waveform-backtest-report-v1']) }).passthrough() }).strict(), +]) + +export async function workerBuildHash(workerPath: string) { + return createHash('sha256').update(await readFile(workerPath)).digest('hex') +} + +/** Executes only the shipped worker path selected by the server, never uploads. */ +export function runBacktestChild(workerPath: string, input: unknown, options: { + signal?: AbortSignal + timeoutMs?: number + onProgress?: (completed: number, total: number) => void +} = {}): Promise { + const serialized = JSON.stringify(input) + if (!serialized || Buffer.byteLength(serialized) > 10 * 1024 * 1024) return Promise.reject(new Error('Replay input exceeds 10 MiB')) + const timeoutMs = options.timeoutMs ?? 120000 + if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 10 || timeoutMs > 1800000) return Promise.reject(new Error('Invalid replay time budget')) + if (options.signal?.aborted) return Promise.reject(new Error('Replay cancelled')) + return new Promise((resolve, reject) => { + const child = spawn(process.execPath, ['--max-old-space-size=128', workerPath], { + cwd: tmpdir(), env: { NODE_ENV: 'production', LANG: 'C.UTF-8', TZ: 'UTC' }, stdio: ['pipe', 'pipe', 'pipe'], + }) + const decoder = new StringDecoder('utf8') + let stdout = '', outputBytes = 0, stderrBytes = 0 + let result: unknown, failure: Error | undefined, killTimer: ReturnType | undefined + const stop = (error: Error) => { + failure ??= error + child.kill('SIGTERM') + killTimer ??= setTimeout(() => child.kill('SIGKILL'), 500) + } + if (child.pid) { + try { + setPriority(child.pid, 19) + } + catch { + stop(new Error('Could not lower replay process priority')) + } + } + const timeout = setTimeout(() => stop(new Error('Replay exceeded its time budget')), timeoutMs) + const abort = () => stop(new Error('Replay cancelled')) + options.signal?.addEventListener('abort', abort, { once: true }) + child.on('error', error => stop(error)) + child.stdin.on('error', () => stop(new Error('Replay input pipe closed'))) + child.stderr.on('data', (chunk: Buffer) => { + stderrBytes += chunk.length + // Do not retain or log health exports through a worker error channel. + if (stderrBytes > 16384) stop(new Error('Replay diagnostic output exceeded its bound')) + }) + child.stdout.on('data', (chunk: Buffer) => { + outputBytes += chunk.length + if (outputBytes > 16 * 1024 * 1024) return stop(new Error('Replay output exceeded 16 MiB')) + stdout += decoder.write(chunk) + let newline: number + while ((newline = stdout.indexOf('\n')) >= 0) { + const line = stdout.slice(0, newline) + stdout = stdout.slice(newline + 1) + try { + const event = eventSchema.parse(JSON.parse(line)) + if (result !== undefined) throw new Error('Worker emitted data after its result') + if (event.type === 'progress') options.onProgress?.(event.completedNights, event.totalNights) + else result = event.report + } + catch { + stop(new Error('Replay emitted an invalid event or progress update')) + return + } + } + }) + child.on('close', (code) => { + clearTimeout(timeout) + if (killTimer) clearTimeout(killTimer) + options.signal?.removeEventListener('abort', abort) + if (failure) reject(failure) + else if (code !== 0 || result === undefined || stdout.length) reject(new Error('Replay worker failed or returned an incomplete report')) + else resolve(result) + }) + child.stdin.end(serialized) + }) +} diff --git a/src/biometrics/backtest/common-support.ts b/src/biometrics/backtest/common-support.ts new file mode 100644 index 000000000..102c12a33 --- /dev/null +++ b/src/biometrics/backtest/common-support.ts @@ -0,0 +1,34 @@ +export interface SupportInterval { start: number, end: number } + +function ordered(intervals: T[]) { + if (intervals.length > 100000) throw new Error('Too many support intervals') + const sorted = [...intervals].sort((a, b) => a.start - b.start) + if (sorted.some((row, index) => !Number.isSafeInteger(row.start) || !Number.isSafeInteger(row.end) || row.end <= row.start + || (index > 0 && sorted[index - 1].end > row.start))) throw new Error('Support intervals must be valid and nonoverlapping') + return sorted +} + +/** Linear sweep after validation/sorting; preserve labels from the left input. */ +export function restrictToSupport(input: T[], support: SupportInterval[]): T[] { + const left = ordered(input), right = ordered(support) + const result: T[] = [] + let i = 0, j = 0 + while (i < left.length && j < right.length) { + const a = left[i], b = right[j] + const start = Math.max(a.start, b.start), end = Math.min(a.end, b.end) + if (end > start) result.push({ ...a, start, end }) + if (a.end <= b.end) i++ + else j++ + } + return result +} + +/** One common denominator across every submitted configuration. An absent + * output empties the intersection; it never silently drops out of the set. */ +export function intersectOutputSupport(outputs: SupportInterval[][], start: number, end: number) { + if (!Number.isSafeInteger(start) || !Number.isSafeInteger(end) || end <= start || end - start > 86400000 + || outputs.length < 1 || outputs.length > 3) throw new Error('Invalid bounded common-support input') + let support: SupportInterval[] = [{ start, end }] + for (const output of outputs) support = restrictToSupport(support, output) + return support +} diff --git a/src/biometrics/backtest/contracts.ts b/src/biometrics/backtest/contracts.ts new file mode 100644 index 000000000..2e64f84b5 --- /dev/null +++ b/src/biometrics/backtest/contracts.ts @@ -0,0 +1,88 @@ +import { stageObjectivePolicySchema } from './stage-objective' +import { z } from 'zod' +import { objectivePolicySchema, type ObjectivePolicy } from './objective-policy' +import { waveformDatasetInputsSchema } from './waveform-contracts' +import { nightBootstrapSchema } from './uncertainty-policy' +import { normalizedMeasurementSchema } from '../references/contracts' +const key = z.string().min(1).max(200) +const hash = z.string().regex(/^[a-f0-9]{64}$/) +export const experimentProtocolSchema = z.object({ + version: z.literal('descriptive-evaluation-v1'), + mode: z.enum(['comparison', 'stage_replay', 'signal_replay']), + waveformStages: z.boolean().optional(), + // Explicit approximation, never represented as measured sensor calibration. + assumedStageCalibrationQuality: z.number().min(0).max(1), + pointToleranceMs: z.number().int().min(0).max(300000), + clockOffsetMs: z.number().int().min(-3600000).max(3600000), + nightUncertainty: nightBootstrapSchema.optional(), + objectivePolicy: objectivePolicySchema.optional(), + objectivePolicies: z.array(objectivePolicySchema).min(1).max(3).optional(), + stageObjectivePolicy: stageObjectivePolicySchema.optional(), +}).strict().refine(protocol => !(protocol.objectivePolicy && protocol.objectivePolicies), 'Use one numeric objective policy representation') + .refine(protocol => new Set(numericObjectivePolicies(protocol).map(policy => policy.metric)).size === numericObjectivePolicies(protocol).length, 'Numeric objective metrics must be unique') + .refine(protocol => !numericObjectivePolicies(protocol).length || protocol.mode === 'signal_replay', 'Numeric objectives require waveform replay') + .refine(protocol => !protocol.stageObjectivePolicy || protocol.mode === 'stage_replay' || (protocol.mode === 'signal_replay' && protocol.waveformStages === true), 'Stage objectives require stage replay') +/** Preserve old protocol hashes: normalize at consumption, never mutate snapshots. */ +export function numericObjectivePolicies(protocol: { objectivePolicy?: ObjectivePolicy, objectivePolicies?: ObjectivePolicy[] }): ObjectivePolicy[] { + return protocol.objectivePolicies ?? (protocol.objectivePolicy ? [protocol.objectivePolicy] : []) +} +export const replayBaselineProvenanceSchema = z.object({ source: z.enum(['acknowledged_runtime', 'unacknowledged_defaults']), + generation: z.number().int().nonnegative().nullable(), effectiveSampleTimeMs: z.number().int().nonnegative().nullable(), warning: z.string().max(1000).nullable() }).strict() +export const enqueueBacktestSchema = z.object({ + experimentId: z.string().uuid(), idempotencyKey: key, evaluatorBuildHash: hash, submissionHash: hash.optional(), + waveform: z.object({ pythonBuildHash: hash, baselineProvenance: z.object({ current: replayBaselineProvenanceSchema, otherSide: replayBaselineProvenanceSchema }).strict().optional(), otherSide: z.object({ manifestHash: hash, resolvedHash: hash, resolved: z.unknown() }).strict() }).strict().optional(), + split: z.enum(['tuning', 'validation', 'holdout']), + configs: z.array(z.object({ label: z.enum(['default', 'current', 'candidate']), revisionId: z.string().uuid().nullable(), + manifestHash: hash, resolvedHash: hash, resolved: z.unknown() }).strict()).min(1).max(3), +}).strict() + +export const backtestJobSnapshotSchema = enqueueBacktestSchema.extend({ version: z.literal('backtest-job-v1'), datasetId: z.string().uuid(), + datasetHash: z.string(), protocol: experimentProtocolSchema, protocolHash: z.string(), holdoutUse: z.number().int().nonnegative(), + evidence: z.enum(['descriptive_only', 'reused_holdout_is_validation']), +}) + +const vitalSchema = z.object({ id: z.number().int(), timestampMs: z.number().int(), heartRate: z.number().nullable(), + hrv: z.number().nullable(), breathingRate: z.number().nullable(), processingRunId: z.string().nullable() }) +const movementSchema = z.object({ id: z.number().int(), timestampMs: z.number().int(), totalMovement: z.number() }) +const processedDatasetSnapshotSchema = z.object({ + version: z.literal('processed-dataset-v1'), side: z.enum(['left', 'right']), subjectId: z.string(), + sessions: z.array(z.object({ id: z.number().int(), startMs: z.number().int(), endMs: z.number().int(), + sleepDurationSeconds: z.number(), timesExitedBed: z.number(), presentIntervals: z.unknown(), notPresentIntervals: z.unknown(), + vitals: z.array(vitalSchema), movement: z.array(movementSchema), + })), + references: z.array(z.object({ importId: z.string(), revision: z.number(), contentHash: z.string(), source: z.string(), + timePolicy: z.unknown(), measurements: z.array(normalizedMeasurementSchema) })), + processingRuns: z.array(z.object({ id: z.string(), side: z.string(), kind: z.string(), bundle: z.unknown(), + configHash: z.string(), calibrationHash: z.string().nullable(), inputIdentity: z.unknown(), startedAtMs: z.number(), endedAtMs: z.number().nullable() })), +}) +export const datasetSnapshotSchema = z.discriminatedUnion('version', [ + processedDatasetSnapshotSchema, + processedDatasetSnapshotSchema.extend({ version: z.literal('waveform-dataset-v1'), waveformInputs: waveformDatasetInputsSchema }), +]) +export type DatasetSnapshot = z.infer + +export const coverageSchema = z.object({ count: z.number(), duplicateTimestamps: z.number(), firstMs: z.number().nullable(), lastMs: z.number().nullable(), + expectedCadenceMs: z.number(), gaps: z.array(z.object({ startMs: z.number(), endMs: z.number() })), gapDurationMs: z.number(), + interpretation: z.literal('cadence_availability_not_measured_support') }) +export const datasetManifestSchema = z.object({ + version: z.enum(['processed-dataset-manifest-v1', 'waveform-dataset-manifest-v1']), + parentDataset: z.object({ id: z.string().uuid(), contentHash: z.string().regex(/^[a-f0-9]{64}$/) }).optional(), + splitProtocol: z.object({ version: z.literal('chronological-whole-night-v1'), timezone: z.string(), + timezoneRuntime: z.object({ icu: z.string(), tz: z.string() }), grouping: z.literal('local_noon_connected_sessions'), + assignments: z.array(z.object({ cohortId: z.string(), nightLabels: z.array(z.string()), sessionIds: z.array(z.number()), + startMs: z.number(), endMs: z.number(), split: z.enum(['tuning', 'validation', 'holdout']) })), + evidence: z.enum(['partitioned_not_validated', 'descriptive_only']), limitations: z.array(z.string()), + }), + coverage: z.array(z.object({ sessionId: z.number(), vitals: coverageSchema, movement: coverageSchema })), + capabilities: z.object({ comparison: z.boolean(), stageReplay: z.boolean(), signalReplay: z.boolean() }), limitations: z.array(z.string()), +}) +export const datasetDescriptorSchema = z.object({ id: z.string().uuid(), side: z.enum(['left', 'right']), subjectId: z.string(), + contentHash: z.string(), manifest: datasetManifestSchema, byteLength: z.number(), createdAtMs: z.number(), releasedAtMs: z.number().nullable(), available: z.boolean() }) + +export const createDatasetSchema = z.object({ + side: z.enum(['left', 'right']), subjectId: z.string().min(1).max(200), + sessionIds: z.array(z.number().int().positive()).min(1).max(60), + referenceImportIds: z.array(z.string().uuid()).max(20), + timezone: z.string().min(1).max(100), + idempotencyKey: z.string().min(1).max(200), +}).strict() diff --git a/src/biometrics/backtest/coordinator-entry.ts b/src/biometrics/backtest/coordinator-entry.ts new file mode 100644 index 000000000..9a0759424 --- /dev/null +++ b/src/biometrics/backtest/coordinator-entry.ts @@ -0,0 +1,76 @@ +import { admissionReceiptSchema, type AdmissionReceipt } from './job-operations' +import { replayBudgetSchema } from './execution-budget' +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import * as schema from '@/src/db/biometrics-schema' +import { ReferenceInputError } from '../references/time' +import { performCoordinatorTask } from './coordinator-tasks' + +// One operation per process. The supervisor owns this process group, including +// the evaluator grandchild; losing the supervisor IPC channel cancels all work. +const [databasePath, workerPath, rawLocationJson, replayBudgetJson] = process.argv.slice(2) +let started = false, completed = false, sequence = 0 +const controller = new AbortController() +let lastAdmission: AdmissionReceipt | null = null +const pending = new Map void>() +process.on('disconnect', () => { + if (completed) return + controller.abort() + // The launcher creates this process group. If it dies, clean up our child too. + if (process.platform !== 'win32') { + try { + process.kill(-process.pid, 'SIGKILL') + } + catch { /* Fall through if the supervisor already removed the group. */ } + } + process.exit(1) +}) +process.on('message', async (message: unknown) => { + const data = message as { type?: string, sequence?: number, allowed?: boolean, sample?: unknown, memory?: unknown, task?: unknown } + if (data?.type === 'cancel') { + controller.abort() + return + } + if (data?.type === 'admission' && typeof data.sequence === 'number') { + const receipt = admissionReceiptSchema.safeParse({ allowed: data.allowed, sample: data.sample ?? null, ...(data.memory === undefined ? {} : { memory: data.memory }) }) + pending.get(data.sequence)?.(receipt.success ? receipt.data : { allowed: false, sample: null }) + return + } + if (data?.type !== 'run' || started) return + started = true + let sqlite: Database.Database | undefined + try { + const replayBudget = replayBudgetJson && replayBudgetJson !== 'null' ? replayBudgetSchema.parse(JSON.parse(replayBudgetJson)) : undefined + sqlite = new Database(databasePath, { fileMustExist: true, timeout: 1000 }) + sqlite.pragma('foreign_keys = ON') + const db = drizzle(sqlite, { schema }) + const result = await performCoordinatorTask(db, data.task, { workerPath, replayBudget, rawLocation: rawLocationJson ? JSON.parse(rawLocationJson) ?? undefined : undefined, signal: controller.signal, admissionReceipt: () => lastAdmission, admit: () => new Promise((resolve) => { + const id = ++sequence + const finish = (receipt: AdmissionReceipt) => { + clearTimeout(timer) + pending.delete(id) + lastAdmission = receipt + resolve(receipt.allowed) + } + const timer = setTimeout(() => finish({ allowed: false, sample: null }), 2000) + pending.set(id, finish) + process.send?.({ type: 'admission', sequence: id }) + }) }) + if (Buffer.byteLength(JSON.stringify(result)) > 17 * 1024 * 1024) throw new Error('Coordinator response exceeds byte limit') + process.send?.({ type: 'result', result }, () => { + completed = true + process.disconnect?.() + }) + } + catch (error) { + // Known input errors preserve their codes; generic errors never log exports. + process.send?.({ type: 'error', code: error instanceof ReferenceInputError ? error.code : 'COORDINATOR_FAILED', + message: error instanceof ReferenceInputError ? error.message.slice(0, 1000) : 'Biometrics coordinator failed' }, () => { + completed = true + process.disconnect?.() + }) + } + finally { + sqlite?.close() + } +}) diff --git a/src/biometrics/backtest/coordinator-lanes.ts b/src/biometrics/backtest/coordinator-lanes.ts new file mode 100644 index 000000000..14477eec2 --- /dev/null +++ b/src/biometrics/backtest/coordinator-lanes.ts @@ -0,0 +1,8 @@ +import type { CoordinatorTask } from './coordinator-tasks' +export type CoordinatorLane = 'read' | 'work' +export const coordinatorTaskLanes: Record = { + readShadowReport: 'read', + pageRawSources: 'read', datasetDescriptor: 'read', readResult: 'read', readSummary: 'read', readTrace: 'read', readNumericTrace: 'read', readReferenceTrace: 'read', + createWaveformDataset: 'work', inspectRawSource: 'work', createDataset: 'work', releaseDataset: 'work', + releaseResult: 'work', createExperiment: 'work', collectArtifacts: 'work', execute: 'work', +} diff --git a/src/biometrics/backtest/coordinator-queue.ts b/src/biometrics/backtest/coordinator-queue.ts new file mode 100644 index 000000000..e39683dcd --- /dev/null +++ b/src/biometrics/backtest/coordinator-queue.ts @@ -0,0 +1,91 @@ +import { coordinatorTaskLanes as taskLanes, type CoordinatorLane as Lane } from './coordinator-lanes' +import { ReferenceInputError } from '../references/time' +import { runCoordinatorTask, type CoordinatorOptions } from './coordinator' +import type { CoordinatorTask, CoordinatorResults } from './coordinator-tasks' + +interface Control { signal?: AbortSignal, admit?: () => Promise } +interface Entry { lane: Lane, start: () => Promise, reject: (error: Error) => void } + +/** One serialized work slot plus one read slot, so replay cannot starve report + * access. No writes or scans use the read slot. Eight requests total and seven + * per lane reserve capacity for the other lane; pending waits remain bounded. + * Both processes count toward the deployment's admission/resource qualification. + */ +export class CoordinatorQueue { + private pending: Entry[] = [] + private active = new Map>() + private stopped = false + private abort = new AbortController() + constructor(private readonly options: CoordinatorOptions, private readonly execute = runCoordinatorTask) {} + + request(task: T, control: Control = {}): Promise { + if (this.stopped) return Promise.reject(new Error('Coordinator stopped')) + if (control.signal?.aborted) return Promise.reject(new Error('Coordinator cancelled')) + const lane = taskLanes[task.type] ?? 'work' + if (this.pending.length + this.active.size >= 8 + || this.pending.filter(entry => entry.lane === lane).length + (this.active.has(lane) ? 1 : 0) >= 7) return Promise.reject(new ReferenceInputError('RESOURCE_LIMIT', 'Biometrics coordinator queue is full')) + return new Promise((resolve, reject) => { + const controller = new AbortController() + let started = false, finished = false + const cleanup = () => { + clearTimeout(timer) + control.signal?.removeEventListener('abort', cancel) + this.abort.signal.removeEventListener('abort', cancel) + } + const fail = (error: Error) => { + if (finished) return + finished = true + cleanup() + this.pending = this.pending.filter(row => row !== entry) + reject(error) + } + const cancel = () => { + controller.abort() + if (!started) fail(new Error('Coordinator cancelled')) + } + const timer = setTimeout(() => fail(new ReferenceInputError('RESOURCE_LIMIT', 'Biometrics coordinator queue wait exceeded')), 30000) + const entry: Entry = { + lane, + reject: fail, + start: async () => { + started = true + clearTimeout(timer) + try { + const result = await this.execute(this.options, task, { ...control, signal: controller.signal }) + if (!finished) resolve(result) + } + catch (error) { if (!finished) reject(error) } + finally { + finished = true + cleanup() + } + }, + } + control.signal?.addEventListener('abort', cancel, { once: true }) + this.abort.signal.addEventListener('abort', cancel, { once: true }) + this.pending.push(entry) + this.pump() + }) + } + + private pump() { + if (this.stopped) return + for (const lane of ['read', 'work'] as const) { + if (this.active.has(lane)) continue + const index = this.pending.findIndex(entry => entry.lane === lane) + if (index === -1) continue + const [next] = this.pending.splice(index, 1) + this.active.set(lane, next.start().finally(() => { + this.active.delete(lane) + this.pump() + })) + } + } + + async stop() { + this.stopped = true + this.abort.abort() + for (const entry of [...this.pending]) entry.reject(new Error('Coordinator stopped')) + await Promise.all(this.active.values()) + } +} diff --git a/src/biometrics/backtest/coordinator-service.ts b/src/biometrics/backtest/coordinator-service.ts new file mode 100644 index 000000000..280884c99 --- /dev/null +++ b/src/biometrics/backtest/coordinator-service.ts @@ -0,0 +1,32 @@ +import type { HealthAdmission } from './job-operations' +import { replayBudgetSchema } from './execution-budget' +import { sql } from 'drizzle-orm' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import type * as schema from '@/src/db/biometrics-schema' +import { ReferenceInputError } from '../references/time' +import { CoordinatorQueue } from './coordinator-queue' +import { packagedCoordinatorPath, packagedWorkerPath } from './location' +import type { CoordinatorTask } from './coordinator-tasks' + +const shared = globalThis as typeof globalThis & { __biometricCoordinator?: { path: string, queue: CoordinatorQueue } } +export function coordinatorRequest(db: BetterSQLite3Database, task: T, control: { + signal?: AbortSignal + admit?: () => Promise + healthAdmission?: HealthAdmission +} = {}) { + const path = db.all<{ name: string, file: string }>(sql`PRAGMA database_list`).find(row => row.name === 'main')?.file + if (!path) throw new ReferenceInputError('WORKER_UNAVAILABLE', 'Coordinator requires a persistent biometrics database') + if (shared.__biometricCoordinator && shared.__biometricCoordinator.path !== path) throw new Error('Stop the coordinator before changing its database') + shared.__biometricCoordinator ??= { path, queue: new CoordinatorQueue({ databasePath: path, + coordinatorPath: packagedCoordinatorPath(), workerPath: packagedWorkerPath(), + replayBudget: process.env.SLEEPYPOD_BIOMETRICS_REPLAY_BUDGET === undefined + ? undefined + : replayBudgetSchema.parse(JSON.parse(process.env.SLEEPYPOD_BIOMETRICS_REPLAY_BUDGET)) }) } + return shared.__biometricCoordinator.queue.request(task, control) +} +export async function stopBiometricCoordinator() { + const current = shared.__biometricCoordinator + if (!current) return + await current.queue.stop() + if (shared.__biometricCoordinator === current) delete shared.__biometricCoordinator +} diff --git a/src/biometrics/backtest/coordinator-tasks.ts b/src/biometrics/backtest/coordinator-tasks.ts new file mode 100644 index 000000000..bcc8e6140 --- /dev/null +++ b/src/biometrics/backtest/coordinator-tasks.ts @@ -0,0 +1,170 @@ +import type { AdmissionReceipt } from './job-operations' +import type Database from 'better-sqlite3' +import { shadowReportRequestSchema } from '../shadow/report-contracts' +import { readShadowReport } from '../shadow/report-operation' +import { referenceTraceRequestSchema } from './reference-trace-page' +import { sql } from 'drizzle-orm' +import { dirname, join } from 'node:path' +import { runRawCollectionChild } from './raw-collection-child' +import type { ReplayBudget } from './execution-budget' +import { z } from 'zod' +import { summarizeBacktestResult } from './report-summary' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import type * as schema from '@/src/db/biometrics-schema' +import { DatasetStore, createDatasetSchema } from './datasets' +import { BacktestJobStore, experimentProtocolSchema } from './jobs' +import { readStageTraces, readNumericTraces, readReferenceTraces } from './traces' +import { traceRequestSchema } from './trace-contracts' +import { executeNextBacktest } from './runner' + +import { RawSourceCatalog, rawSourcePageSchema } from './raw-catalog' +import { rawScanInputSchema, runRawInspectionChild, podRawInspectionLocation, type RawInspectionLocation } from './raw-inspection-child' + +import { createWaveformDatasetSchema } from './waveform-selection' +import { publishSelectedWaveformDataset } from './waveform-publication' + +const idInput = z.object({ id: z.string().uuid() }).strict() +export const coordinatorTaskSchema = z.discriminatedUnion('type', [ + z.object({ type: z.literal('readShadowReport'), input: shadowReportRequestSchema }).strict(), + z.object({ type: z.literal('createWaveformDataset'), input: createWaveformDatasetSchema }).strict(), + z.object({ type: z.literal('inspectRawSource'), input: rawScanInputSchema }).strict(), + z.object({ type: z.literal('pageRawSources'), input: rawSourcePageSchema }).strict(), + z.object({ type: z.literal('createDataset'), input: createDatasetSchema }).strict(), + z.object({ type: z.literal('datasetDescriptor'), input: idInput }).strict(), + z.object({ type: z.literal('readResult'), input: idInput }).strict(), + z.object({ type: z.literal('readSummary'), input: idInput }).strict(), + z.object({ type: z.literal('readReferenceTrace'), input: referenceTraceRequestSchema }).strict(), + z.object({ type: z.literal('readNumericTrace'), input: traceRequestSchema }).strict(), + z.object({ type: z.literal('readTrace'), input: traceRequestSchema }).strict(), + z.object({ type: z.literal('releaseDataset'), input: idInput }).strict(), + z.object({ type: z.literal('releaseResult'), input: idInput.extend({ expectedContentHash: z.string().regex(/^[a-f0-9]{64}$/) }) }).strict(), + z.object({ type: z.literal('createExperiment'), input: z.object({ datasetId: z.string().uuid(), protocol: experimentProtocolSchema, idempotencyKey: z.string().min(1).max(200) }).strict() }).strict(), + z.object({ type: z.literal('collectArtifacts') }).strict(), + z.object({ type: z.literal('execute'), owner: z.string().min(1).max(200) }).strict(), +]) +export type CoordinatorTask = z.input +export interface CoordinatorResults { + readShadowReport: ReturnType + createWaveformDataset: Awaited> + inspectRawSource: Awaited> + pageRawSources: ReturnType + createDataset: ReturnType + datasetDescriptor: ReturnType + readResult: ReturnType + readSummary: ReturnType>>> + readReferenceTrace: ReturnType + readNumericTrace: ReturnType + readTrace: ReturnType + releaseDataset: ReturnType + releaseResult: ReturnType + createExperiment: ReturnType + collectArtifacts: null + execute: Awaited> +} + +/** Fixed shipped operations; neither executable code nor paths come from RPC input. */ +export async function performCoordinatorTask(db: BetterSQLite3Database & { $client: Database.Database }, raw: unknown, options: { + workerPath: string + rawLocation?: RawInspectionLocation + replayBudget?: ReplayBudget + signal: AbortSignal + admit: () => Promise + admissionReceipt?: () => AdmissionReceipt | null +}) { + const task = coordinatorTaskSchema.parse(raw) + switch (task.type) { + case 'readShadowReport': return readShadowReport(db.$client, task.input) + case 'createWaveformDataset': return publishSelectedWaveformDataset(db, task.input, options) + case 'inspectRawSource': return inspectRawSource(db, task.input, options) + case 'pageRawSources': return new RawSourceCatalog(db).page(task.input) + case 'createDataset': return new DatasetStore(db).create(task.input) + case 'datasetDescriptor': return new DatasetStore(db).descriptor(task.input.id) + case 'readResult': return new BacktestJobStore(db).result(task.input.id) + case 'readSummary': return summarizeBacktestResult(new BacktestJobStore(db).result(task.input.id)) + case 'readReferenceTrace': return readReferenceTraces(db, task.input) + case 'readNumericTrace': return readNumericTraces(db, task.input) + case 'readTrace': return readStageTraces(db, task.input) + case 'releaseDataset': return new DatasetStore(db).release(task.input.id) + case 'releaseResult': return new BacktestJobStore(db).releaseResult(task.input.id, task.input.expectedContentHash) + case 'createExperiment': return new BacktestJobStore(db).createExperiment(task.input.datasetId, task.input.protocol, task.input.idempotencyKey) + case 'collectArtifacts': + new BacktestJobStore(db).collectOrphanedArtifacts() + await collectRawArtifacts(db, options) + return null + case 'execute': return executeNextBacktest(db, { owner: task.owner, ...options }) + } +} + +/** Recheck resource health during decompression and before committing evidence. */ +async function inspectRawSource(db: BetterSQLite3Database, input: z.input, options: { + signal: AbortSignal + admit: () => Promise + rawLocation?: RawInspectionLocation + replayBudget?: ReplayBudget +}) { + const deferred = { status: 'deferred' as const, nextCursor: input.afterName } + if (options.signal.aborted || !await options.admit()) return deferred + const controller = new AbortController() + const abort = () => controller.abort() + options.signal.addEventListener('abort', abort, { once: true }) + if (options.signal.aborted) abort() + let checking: Promise | undefined, denied = false + const timer = setInterval(() => { + if (checking) return + checking = Promise.resolve().then(() => options.admit()).catch(() => false).then((allowed) => { + if (!allowed) { + denied = true + controller.abort() + } + }).finally(() => { checking = undefined }) + }, 5000) + try { + const result = await runRawInspectionChild(options.rawLocation ?? podRawInspectionLocation, input, { signal: controller.signal }) + clearInterval(timer) + await checking + if (denied || options.signal.aborted || !await options.admit() || options.signal.aborted) return deferred + if (result.status !== 'inspected') return result + const id = new RawSourceCatalog(db).record(result.inspection) + return { status: 'inspected' as const, nextCursor: result.nextCursor, source: new RawSourceCatalog(db).read(id) } + } + catch (error) { + if (denied) return deferred + throw error + } + finally { + clearInterval(timer) + await checking + options.signal.removeEventListener('abort', abort) + } +} + +/** Same work lane as publication; Python additionally locks across processes. */ +async function collectRawArtifacts(db: BetterSQLite3Database, options: { + signal: AbortSignal + admit: () => Promise + rawLocation?: RawInspectionLocation +}) { + if (options.signal.aborted || !await options.admit()) return + const databasePath = db.all<{ name: string, file: string }>(sql`PRAGMA database_list`).find(row => row.name === 'main')?.file + if (!databasePath) return + const controller = new AbortController() + const abort = () => controller.abort() + options.signal.addEventListener('abort', abort, { once: true }) + if (options.signal.aborted) abort() + let checking: Promise | undefined + const timer = setInterval(() => { + if (checking) return + checking = Promise.resolve().then(options.admit).catch(() => false).then((allowed) => { + if (!allowed) controller.abort() + }).finally(() => { checking = undefined }) + }, 5000) + try { + await runRawCollectionChild({ ...(options.rawLocation ?? podRawInspectionLocation), databasePath, + retainedDirectory: join(dirname(databasePath), 'biometric-raw') }, { signal: controller.signal }) + } + finally { + clearInterval(timer) + await checking + options.signal.removeEventListener('abort', abort) + } +} diff --git a/src/biometrics/backtest/coordinator.ts b/src/biometrics/backtest/coordinator.ts new file mode 100644 index 000000000..07f92761c --- /dev/null +++ b/src/biometrics/backtest/coordinator.ts @@ -0,0 +1,123 @@ +import { withBiometricBackgroundSlot } from '../background-admission' +import { coordinatorTaskLanes } from './coordinator-lanes' +import { confirmShadowProcessCleanup } from '../shadow/process-cleanup' +import { monitorProcessGroupMemory } from './process-group-memory' +import { admissionReceiptSchema, type HealthAdmission } from './job-operations' +import { fork } from 'node:child_process' +import { setPriority } from 'node:os' +import { dirname, resolve } from 'node:path' +import { ReferenceInputError } from '../references/time' +import { coordinatorTaskSchema, type CoordinatorTask, type CoordinatorResults } from './coordinator-tasks' + +import { replayBudgetSchema, type ReplayBudget } from './execution-budget' +import type { RawInspectionLocation } from './raw-inspection-child' + +export interface CoordinatorOptions { + databasePath: string + coordinatorPath: string + workerPath: string + rawLocation?: RawInspectionLocation + modulePath?: string + timeoutMs?: number + replayBudget?: ReplayBudget +} + +/** Own an OS process group so crashes/timeouts cannot orphan the evaluator. */ +export function runCoordinatorTask(options: CoordinatorOptions, task: T, control: { + signal?: AbortSignal + admit?: () => Promise + healthAdmission?: HealthAdmission +} = {}): Promise { + const parsed = coordinatorTaskSchema.parse(task) + if (Buffer.byteLength(JSON.stringify(parsed)) > 65536) return Promise.reject(new ReferenceInputError('RESOURCE_LIMIT', 'Coordinator input exceeds 64 KiB')) + const replayBudget = options.replayBudget === undefined ? undefined : replayBudgetSchema.parse(options.replayBudget) + const timeoutMs = options.timeoutMs ?? (parsed.type === 'execute' ? replayBudget?.jobWallMs : undefined) ?? 150000 + const maximumTimeout = parsed.type === 'execute' && replayBudget ? replayBudget.jobWallMs : 180000 + if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 10 || timeoutMs > maximumTimeout) throw new Error('Invalid coordinator timeout') + if (control.signal?.aborted) return Promise.reject(new Error('Coordinator cancelled')) + return withBiometricBackgroundSlot(coordinatorTaskLanes[parsed.type], () => new Promise((resolveResult, reject) => { + const child = fork(resolve(options.coordinatorPath), [resolve(options.databasePath), resolve(options.workerPath), JSON.stringify(options.rawLocation ?? null), JSON.stringify(parsed.type === 'execute' ? replayBudget ?? null : null)], { + cwd: dirname(resolve(options.databasePath)), detached: process.platform !== 'win32', serialization: 'advanced', + execArgv: ['--max-old-space-size=384'], silent: true, + env: { NODE_ENV: 'production', LANG: 'C.UTF-8', TZ: 'UTC', NODE_PATH: options.modulePath ?? resolve('node_modules') }, + }) + const memory = parsed.type === 'execute' && child.pid ? monitorProcessGroupMemory(child.pid) : null + let settled = false, received = false, result: CoordinatorResults[T['type']], failure: Error | undefined + let grace: ReturnType | undefined + let cancellationGrace: ReturnType | undefined + let diagnostics = 0, admissionActive = false + const killGroup = (signal: NodeJS.Signals) => { + if (!child.pid) return + try { + if (process.platform === 'win32') child.kill(signal) + else process.kill(-child.pid, signal) + } + catch { /* Already exited. */ } + } + const stop = (error: Error) => { + failure ??= error + killGroup('SIGTERM') + grace ??= setTimeout(() => killGroup('SIGKILL'), 500) + } + const timer = setTimeout(() => stop(new Error('Coordinator time budget exceeded')), timeoutMs) + const cancel = () => { + failure ??= new Error('Coordinator cancelled') + if (child.connected) child.send({ type: 'cancel' }, () => {}) + cancellationGrace ??= setTimeout(() => stop(new Error('Coordinator cancellation grace expired')), 2000) + } + control.signal?.addEventListener('abort', cancel, { once: true }) + if (control.signal?.aborted) cancel() + const count = (chunk: Buffer) => { + diagnostics += chunk.length + if (diagnostics > 16384) stop(new Error('Coordinator diagnostics exceeded byte limit')) + } + child.stdout?.on('data', count) + child.stderr?.on('data', count) + child.on('error', stop) + child.on('message', (message: unknown) => { + const data = message as { type?: string, sequence?: number, result?: CoordinatorResults[T['type']], code?: string, message?: string } + if (data?.type === 'admission' && (parsed.type === 'execute' || parsed.type === 'inspectRawSource' || parsed.type === 'createWaveformDataset' || parsed.type === 'collectArtifacts') && !received && !admissionActive && Number.isSafeInteger(data.sequence)) { + admissionActive = true + Promise.resolve().then(async () => control.healthAdmission + ? admissionReceiptSchema.parse(await control.healthAdmission()) + : { allowed: await control.admit?.() ?? false, sample: null }).catch(() => ({ allowed: false, sample: null })).then(async (receipt) => { + await memory?.sample() + if (child.connected && !failure) child.send({ type: 'admission', sequence: data.sequence, ...receipt, ...(memory ? { memory: memory.snapshot() } : {}) }, () => {}) + }).finally(() => { admissionActive = false }) + } + else if (data?.type === 'result' && !received) { + received = true + result = data.result as CoordinatorResults[T['type']] + } + else if (data?.type === 'error' && typeof data.code === 'string' && typeof data.message === 'string') { + stop(new ReferenceInputError(data.code, data.message)) + } + else stop(new Error('Invalid coordinator response')) + }) + child.on('close', async (code) => { + if (settled) return + settled = true + void memory?.stop() + clearTimeout(timer) + if (grace) clearTimeout(grace) + if (cancellationGrace) clearTimeout(cancellationGrace) + control.signal?.removeEventListener('abort', cancel) + // Also kill descendants if the coordinator crashed before reporting them. + killGroup('SIGKILL') + if (process.platform !== 'win32' && !await confirmShadowProcessCleanup(child.pid, () => true)) { + reject(Object.assign(new Error('Biometrics coordinator process-group cleanup unconfirmed', { cause: failure }), { code: 'BIOMETRIC_COORDINATOR_CLEANUP_UNCONFIRMED' })) + return + } + if (failure) reject(failure) + else if (!received || code !== 0) reject(new Error('Coordinator exited without a complete result')) + else resolveResult(result) + }) + try { + if (child.pid) setPriority(child.pid, 19) + child.send({ type: 'run', task: parsed }, (error) => { + if (error) stop(error) + }) + } + catch { stop(new Error('Could not start coordinator')) } + })) +} diff --git a/src/biometrics/backtest/coverage.ts b/src/biometrics/backtest/coverage.ts new file mode 100644 index 000000000..988c3625b --- /dev/null +++ b/src/biometrics/backtest/coverage.ts @@ -0,0 +1,18 @@ +/** No interpolation: expose actual timestamps and gaps under an explicit cadence. */ +export function describeCoverage(timestamps: number[], startMs: number, endMs: number, expectedCadenceMs: number) { + if (![startMs, endMs, expectedCadenceMs].every(Number.isSafeInteger) || endMs <= startMs || expectedCadenceMs <= 0 + || timestamps.length > 100000 || timestamps.some(time => !Number.isSafeInteger(time))) throw new Error('Invalid bounded coverage input') + const times = [...new Set(timestamps.filter(time => time >= startMs && time < endMs))].sort((a, b) => a - b) + const gaps: Array<{ startMs: number, endMs: number }> = [] + let cursor = startMs + for (const time of times) { + if (time > cursor) gaps.push({ startMs: cursor, endMs: time }) + cursor = Math.min(endMs, time + expectedCadenceMs) + } + if (cursor < endMs) gaps.push({ startMs: cursor, endMs }) + return { count: times.length, duplicateTimestamps: timestamps.filter(time => time >= startMs && time < endMs).length - times.length, + firstMs: times[0] ?? null, lastMs: times.at(-1) ?? null, + expectedCadenceMs, gaps, gapDurationMs: gaps.reduce((sum, gap) => sum + gap.endMs - gap.startMs, 0), + // Cadence-based availability is not sensor support duration. + interpretation: 'cadence_availability_not_measured_support' as const } +} diff --git a/src/biometrics/backtest/datasets.ts b/src/biometrics/backtest/datasets.ts new file mode 100644 index 000000000..b86ce0d2c --- /dev/null +++ b/src/biometrics/backtest/datasets.ts @@ -0,0 +1,194 @@ +import { randomUUID } from 'node:crypto' +import { and, desc, eq, gte, inArray, isNull, lt, or, sql } from 'drizzle-orm' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import type { z } from 'zod' +import * as schema from '@/src/db/biometrics-schema' +import { configHash, dataHash } from '../algorithms/canonical' +import { normalizedMeasurementSchema } from '../references/contracts' +import { ReferenceStore } from '../references/store' +import { ReferenceInputError } from '../references/time' +import { assignWholeNightSplits } from './splits' +import { describeCoverage } from './coverage' + +import { createDatasetSchema } from './contracts' +export { createDatasetSchema } from './contracts' + +export { datasetSnapshotSchema } from './contracts' +import { datasetSnapshotSchema } from './contracts' + +const LIMITS = { rows: 20000, datasetBytes: 8 * 1024 * 1024, totalBytes: 64 * 1024 * 1024 } + +/** Creates processed snapshots; reads and releases both retained dataset formats. */ +export class DatasetStore { + constructor(private readonly db: BetterSQLite3Database, private readonly now = Date.now) {} + + create(raw: z.input) { + const input = createDatasetSchema.parse(raw) + if (new Set(input.sessionIds).size !== input.sessionIds.length || new Set(input.referenceImportIds).size !== input.referenceImportIds.length) throw new ReferenceInputError('DUPLICATE_SELECTION', 'Select each session and import once') + input.sessionIds.sort((a, b) => a - b) + input.referenceImportIds.sort() + const requestHash = configHash(input) + const existing = this.db.select().from(schema.biometricDatasets).where(eq(schema.biometricDatasets.idempotencyKey, input.idempotencyKey)).get() + if (existing) { + if (existing.requestHash !== requestHash) throw new ReferenceInputError('REVISION_CONFLICT', 'Dataset idempotency key was already used for a different selection') + return this.describe(existing) + } + const captured = this.db.transaction(() => { + const sessions = this.db.select().from(schema.sleepRecords).where(and(eq(schema.sleepRecords.side, input.side), inArray(schema.sleepRecords.id, input.sessionIds))) + .orderBy(schema.sleepRecords.enteredBedAt, schema.sleepRecords.id).all() + if (sessions.length !== input.sessionIds.length) throw new ReferenceInputError('NOT_FOUND', 'A selected session is missing or belongs to the other side') + const splits = assignWholeNightSplits(sessions.map(row => ({ id: row.id, startMs: row.enteredBedAt.getTime(), endMs: row.leftBedAt.getTime() })), input.timezone) + let remaining = LIMITS.rows + const snapshotSessions = sessions.map((session) => { + const vitals = this.db.select().from(schema.vitals).where(and(eq(schema.vitals.side, input.side), + gte(schema.vitals.timestamp, session.enteredBedAt), lt(schema.vitals.timestamp, session.leftBedAt))) + .orderBy(schema.vitals.timestamp).limit(remaining + 1).all() + remaining -= vitals.length + if (remaining < 0) throw new ReferenceInputError('RESOURCE_LIMIT', 'Select fewer sessions; processed snapshot row budget exceeded') + const movement = this.db.select().from(schema.movement).where(and(eq(schema.movement.side, input.side), + gte(schema.movement.timestamp, session.enteredBedAt), lt(schema.movement.timestamp, session.leftBedAt))) + .orderBy(schema.movement.timestamp, schema.movement.id).limit(remaining + 1).all() + remaining -= movement.length + if (remaining < 0) throw new ReferenceInputError('RESOURCE_LIMIT', 'Select fewer sessions; processed snapshot row budget exceeded') + return { id: session.id, startMs: session.enteredBedAt.getTime(), endMs: session.leftBedAt.getTime(), + sleepDurationSeconds: session.sleepDurationSeconds, timesExitedBed: session.timesExitedBed, + presentIntervals: session.presentIntervals, notPresentIntervals: session.notPresentIntervals, + vitals: vitals.map(row => ({ id: row.id, timestampMs: row.timestamp.getTime(), heartRate: row.heartRate, hrv: row.hrv, breathingRate: row.breathingRate, processingRunId: row.processingRunId })), + movement: movement.map(row => ({ id: row.id, timestampMs: row.timestamp.getTime(), totalMovement: row.totalMovement })), + } + }) + const store = new ReferenceStore(this.db) + const families = new Set() + const references = input.referenceImportIds.map((importId) => { + const descriptor = store.get(importId) + if (descriptor.summary.side !== input.side || descriptor.summary.subjectId !== input.subjectId) throw new ReferenceInputError('SUBJECT_MISMATCH', 'All references must match the selected side and sleeper') + if (families.has(descriptor.familyId)) throw new ReferenceInputError('REVISION_CONFLICT', 'Select only one revision from each reference family') + families.add(descriptor.familyId) + const measurements = [] + let cursor = 0 + do { + const page = store.measurements(importId, cursor) + remaining -= page.items.length + if (remaining < 0) throw new ReferenceInputError('RESOURCE_LIMIT', 'Reference selection exceeds snapshot row budget') + measurements.push(...page.items.map(row => normalizedMeasurementSchema.parse(Object.fromEntries(Object.entries(row).filter(([key]) => key !== 'id'))))) + if (page.nextCursor === null) break + cursor = page.nextCursor + } while (remaining >= 0) + return { importId, revision: descriptor.revision, contentHash: descriptor.summary.contentHash, + source: descriptor.summary.source, timePolicy: descriptor.summary.timePolicy, measurements } + }) + const runIds = [...new Set(snapshotSessions.flatMap(session => session.vitals.flatMap(row => row.processingRunId ? [row.processingRunId] : [])))] + const processingRuns: Array = [] + for (let index = 0; index < runIds.length; index += 100) { + processingRuns.push(...this.db.select().from(schema.biometricProcessingRuns) + .where(and(eq(schema.biometricProcessingRuns.side, input.side), inArray(schema.biometricProcessingRuns.id, runIds.slice(index, index + 100)))).all()) + } + processingRuns.sort((a, b) => a.id.localeCompare(b.id)) + return { snapshot: datasetSnapshotSchema.parse({ version: 'processed-dataset-v1', side: input.side, subjectId: input.subjectId, + sessions: snapshotSessions, references, processingRuns }), splits } + }) + // No write lock is held while computing hashes and coverage. + const snapshotBytes = Buffer.byteLength(JSON.stringify(captured.snapshot)) + if (snapshotBytes > LIMITS.datasetBytes) throw new ReferenceInputError('RESOURCE_LIMIT', 'Processed dataset exceeds the 8 MiB snapshot limit') + const manifest = { version: 'processed-dataset-manifest-v1', splitProtocol: captured.splits, + coverage: captured.snapshot.sessions.map(session => ({ sessionId: session.id, + vitals: describeCoverage(session.vitals.map(row => row.timestampMs), session.startMs, session.endMs, 60000), + movement: describeCoverage(session.movement.map(row => row.timestampMs), session.startMs, session.endMs, 60000), + })), + capabilities: { comparison: true, stageReplay: captured.snapshot.sessions.some(session => session.vitals.length > 0), signalReplay: false }, + limitations: ['Raw channels, pump context and measured calibration bytes are not retained in this processed dataset; signal replay is unavailable.', + 'Historical stage replay must declare its calibration quality assumption when original calibration is unavailable.', + 'Reference deletion does not erase this separately retained dataset.', + 'Session side is known; historical occupant identity is asserted by the dataset creator, not measured by the pod.'], + } + const byteLength = snapshotBytes + Buffer.byteLength(JSON.stringify(manifest)) + if (byteLength > LIMITS.datasetBytes) throw new ReferenceInputError('RESOURCE_LIMIT', 'Dataset and manifest exceed the 8 MiB snapshot limit') + const contentHash = dataHash({ manifest, snapshot: captured.snapshot }) + return this.db.transaction((tx) => { + const raced = tx.select().from(schema.biometricDatasets).where(eq(schema.biometricDatasets.idempotencyKey, input.idempotencyKey)).get() + if (raced) { + if (raced.requestHash !== requestHash) throw new ReferenceInputError('REVISION_CONFLICT', 'Dataset idempotency key changed') + return this.describe(raced) + } + const total = tx.select({ bytes: sql`coalesce(sum(${schema.biometricDatasets.byteLength}), 0)` }).from(schema.biometricDatasets).get()?.bytes ?? 0 + if (total + byteLength > LIMITS.totalBytes) throw new ReferenceInputError('RESOURCE_LIMIT', 'Processed datasets have reached their 64 MiB retained storage budget') + const row = tx.insert(schema.biometricDatasets).values({ id: randomUUID(), side: input.side, subjectId: input.subjectId, + idempotencyKey: input.idempotencyKey, requestHash, contentHash, manifest, snapshot: captured.snapshot, byteLength, createdAtMs: this.now() }).returning().get() + return this.describe(row) + }, { behavior: 'immediate' }) + } + + private describe(row: typeof schema.biometricDatasets.$inferSelect) { + return { id: row.id, side: row.side, subjectId: row.subjectId, contentHash: row.contentHash, + manifest: row.manifest, byteLength: row.byteLength, createdAtMs: row.createdAtMs, releasedAtMs: row.releasedAtMs, available: row.releasedAtMs === null } + } + + read(id: string) { + const row = this.db.select().from(schema.biometricDatasets).where(eq(schema.biometricDatasets.id, id)).get() + if (!row) throw new ReferenceInputError('NOT_FOUND', 'Dataset is unavailable') + if (row.releasedAtMs !== null) throw new ReferenceInputError('DATASET_RELEASED', 'Retained dataset inputs were explicitly released') + const snapshot = datasetSnapshotSchema.parse(row.snapshot) + if (dataHash({ manifest: row.manifest, snapshot }) !== row.contentHash) throw new ReferenceInputError('DATASET_CORRUPT', 'Retained dataset integrity check failed') + return { ...this.describe(row), snapshot } + } + + release(id: string) { + return this.db.transaction((tx) => { + const dataset = tx.select().from(schema.biometricDatasets).where(eq(schema.biometricDatasets.id, id)).get() + if (!dataset) throw new ReferenceInputError('NOT_FOUND', 'Dataset not found') + if (dataset.releasedAtMs !== null) return this.describe(dataset) + const active = tx.select({ id: schema.biometricBacktestJobs.id }).from(schema.biometricBacktestJobs) + .innerJoin(schema.biometricExperiments, eq(schema.biometricBacktestJobs.experimentId, schema.biometricExperiments.id)) + .where(and(eq(schema.biometricExperiments.datasetId, id), inArray(schema.biometricBacktestJobs.state, ['queued', 'running', 'cancel_requested', 'interrupted']))).get() + if (active) throw new ReferenceInputError('REVISION_CONFLICT', 'Cancel or finish jobs using this dataset before releasing its inputs') + const releasedAtMs = this.now() + const references = schema.biometricDatasetRawArtifacts + const artifacts = schema.biometricRawArtifacts + tx.update(references).set({ releasedAtMs }) + .where(and(eq(references.datasetId, id), isNull(references.releasedAtMs))).run() + // Mark storage eligible for collection only after its last live reference + // is released. The file collector separately serializes with publication. + tx.update(artifacts).set({ releasedAtMs }).where(and(isNull(artifacts.releasedAtMs), + inArray(artifacts.contentHash, tx.select({ hash: references.artifactHash }).from(references).where(eq(references.datasetId, id))), + sql`NOT EXISTS (SELECT 1 FROM biometric_dataset_raw_artifacts AS live + WHERE live.artifact_hash = ${artifacts.contentHash} AND live.released_at_ms IS NULL)`, + )).run() + const tombstone = { released: true } + const byteLength = Buffer.byteLength(JSON.stringify(dataset.manifest)) + Buffer.byteLength(JSON.stringify(tombstone)) + const released = tx.update(schema.biometricDatasets).set({ snapshot: tombstone, byteLength, releasedAtMs }) + .where(eq(schema.biometricDatasets.id, id)).returning().get() + return this.describe(released) + }, { behavior: 'immediate' }) + } + + descriptor(id: string) { + const row = this.db.select().from(schema.biometricDatasets).where(eq(schema.biometricDatasets.id, id)).get() + if (!row) throw new ReferenceInputError('NOT_FOUND', 'Dataset not found') + if (row.releasedAtMs === null) { + const snapshot = datasetSnapshotSchema.parse(row.snapshot) + if (dataHash({ manifest: row.manifest, snapshot }) !== row.contentHash) throw new ReferenceInputError('DATASET_CORRUPT', 'Retained dataset integrity check failed') + } + return this.describe(row) + } + + page(side: 'left' | 'right', cursor?: { createdAtMs: number, id: string }) { + const rows = this.db.select({ id: schema.biometricDatasets.id, side: schema.biometricDatasets.side, + subjectId: schema.biometricDatasets.subjectId, contentHash: schema.biometricDatasets.contentHash, + manifest: schema.biometricDatasets.manifest, byteLength: schema.biometricDatasets.byteLength, createdAtMs: schema.biometricDatasets.createdAtMs, releasedAtMs: schema.biometricDatasets.releasedAtMs, + }).from(schema.biometricDatasets).where(and(eq(schema.biometricDatasets.side, side), cursor + ? or(lt(schema.biometricDatasets.createdAtMs, cursor.createdAtMs), and(eq(schema.biometricDatasets.createdAtMs, cursor.createdAtMs), lt(schema.biometricDatasets.id, cursor.id))) + : undefined)) + .orderBy(desc(schema.biometricDatasets.createdAtMs), desc(schema.biometricDatasets.id)).limit(51).all() + const items = rows.slice(0, 50).map(row => ({ ...row, available: row.releasedAtMs === null })) + const last = items.at(-1) + return { items, nextCursor: rows.length > 50 && last ? { createdAtMs: last.createdAtMs, id: last.id } : null } + } + + list(side: 'left' | 'right') { + return this.db.select({ id: schema.biometricDatasets.id, side: schema.biometricDatasets.side, + subjectId: schema.biometricDatasets.subjectId, contentHash: schema.biometricDatasets.contentHash, + manifest: schema.biometricDatasets.manifest, byteLength: schema.biometricDatasets.byteLength, createdAtMs: schema.biometricDatasets.createdAtMs, releasedAtMs: schema.biometricDatasets.releasedAtMs, + }).from(schema.biometricDatasets).where(eq(schema.biometricDatasets.side, side)) + .orderBy(desc(schema.biometricDatasets.createdAtMs), desc(schema.biometricDatasets.id)).limit(100).all() + } +} diff --git a/src/biometrics/backtest/descriptors.ts b/src/biometrics/backtest/descriptors.ts new file mode 100644 index 000000000..7cf6dd0b7 --- /dev/null +++ b/src/biometrics/backtest/descriptors.ts @@ -0,0 +1,23 @@ +import { z } from 'zod' +import type { biometricBacktestJobs } from '@/src/db/biometrics-schema' + +export const jobDescriptorSchema = z.object({ id: z.string().uuid(), experimentId: z.string().uuid(), + state: z.enum(['queued', 'running', 'cancel_requested', 'cancelled', 'succeeded', 'failed', 'interrupted']), attempt: z.number(), + progress: z.object({ phase: z.string(), completedNights: z.number(), totalNights: z.number() }), + cancellationRequestedAtMs: z.number().nullable(), error: z.string().nullable(), createdAtMs: z.number(), updatedAtMs: z.number(), + datasetId: z.string().uuid(), datasetHash: z.string(), evaluatorBuildHash: z.string(), split: z.enum(['tuning', 'validation', 'holdout']), + configurations: z.array(z.object({ label: z.string(), revisionId: z.string().uuid().nullable(), hash: z.string() })), + evaluationScope: z.enum(['original_readings_comparison', 'stage_rules_on_original_readings', 'retained_waveform_replay']), +}) +export function describeJob(row: typeof biometricBacktestJobs.$inferSelect) { + const input = z.object({ datasetId: z.string(), datasetHash: z.string(), evaluatorBuildHash: z.string(), split: z.string(), + protocol: z.object({ mode: z.enum(['comparison', 'stage_replay', 'signal_replay']) }), + configs: z.array(z.object({ label: z.string(), revisionId: z.string().nullable(), resolvedHash: z.string() })), + }).parse(row.input) + return jobDescriptorSchema.parse({ ...row, datasetId: input.datasetId, datasetHash: input.datasetHash, evaluatorBuildHash: input.evaluatorBuildHash, + split: input.split, configurations: input.configs.map(config => ({ label: config.label, revisionId: config.revisionId, hash: config.resolvedHash })), + evaluationScope: input.protocol.mode === 'signal_replay' ? 'retained_waveform_replay' : input.protocol.mode === 'comparison' ? 'original_readings_comparison' : 'stage_rules_on_original_readings' }) +} + +export const resultDescriptorSchema = z.object({ jobId: z.string(), fencingToken: z.number(), report: z.unknown(), + contentHash: z.string(), completedAtMs: z.number(), releasedAtMs: z.number().nullable(), available: z.boolean(), currentHoldoutUseCount: z.number() }) diff --git a/src/biometrics/backtest/discovery.ts b/src/biometrics/backtest/discovery.ts new file mode 100644 index 000000000..7f4f1ffda --- /dev/null +++ b/src/biometrics/backtest/discovery.ts @@ -0,0 +1,60 @@ +import { and, desc, eq, gt, gte, lt, or, sql } from 'drizzle-orm' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import { z } from 'zod' +import * as schema from '@/src/db/biometrics-schema' +import { ReferenceInputError } from '../references/time' +import { coverageSchema } from './contracts' +import { describeCoverage } from './coverage' + +const side = z.enum(['left', 'right']) +const sessionSchema = z.object({ id: z.number(), side, startMs: z.number(), endMs: z.number(), sleepDurationSeconds: z.number(), timesExitedBed: z.number() }) +export const sessionPageSchema = z.object({ items: z.array(sessionSchema), nextCursor: z.object({ startMs: z.number(), id: z.number() }).nullable() }) +export const sessionCapabilitiesSchema = z.object({ + session: sessionSchema, + processed: z.object({ vitals: coverageSchema, movement: coverageSchema, legacyUnknownProvenanceCount: z.number() }), + referenceMetrics: z.array(z.object({ metric: z.string(), count: z.number() })), + comparison: z.boolean(), stageReplay: z.boolean(), signalReplay: z.literal(false), rawWaveforms: z.literal('not_inventoried'), + limitations: z.array(z.string()), +}) +const describeSession = (row: typeof schema.sleepRecords.$inferSelect) => ({ id: row.id, side: row.side, startMs: row.enteredBedAt.getTime(), + endMs: row.leftBedAt.getTime(), sleepDurationSeconds: row.sleepDurationSeconds, timesExitedBed: row.timesExitedBed }) + +export function listReplaySessions(db: BetterSQLite3Database, selectedSide: 'left' | 'right', cursor?: { startMs: number, id: number }) { + const records = db.select().from(schema.sleepRecords).where(and(eq(schema.sleepRecords.side, selectedSide), cursor + ? or(lt(schema.sleepRecords.enteredBedAt, new Date(cursor.startMs)), and(eq(schema.sleepRecords.enteredBedAt, new Date(cursor.startMs)), lt(schema.sleepRecords.id, cursor.id))) + : undefined)) + .orderBy(desc(schema.sleepRecords.enteredBedAt), desc(schema.sleepRecords.id)).limit(21).all() + const items = records.slice(0, 20).map(describeSession) + const last = items.at(-1) + return { items, nextCursor: records.length > 20 && last ? { startMs: last.startMs, id: last.id } : null } +} + +export function sessionCapabilities(db: BetterSQLite3Database, selectedSide: 'left' | 'right', sessionId: number) { + return db.transaction(() => { + const record = db.select().from(schema.sleepRecords).where(and(eq(schema.sleepRecords.side, selectedSide), eq(schema.sleepRecords.id, sessionId))).get() + if (!record) throw new ReferenceInputError('NOT_FOUND', 'Session not found for this side') + const session = describeSession(record) + if (session.endMs <= session.startMs || session.endMs - session.startMs > 36 * 3600000) throw new ReferenceInputError('RESOURCE_LIMIT', 'Session boundaries exceed the inspection budget') + const vitals = db.select({ timestamp: schema.vitals.timestamp, run: schema.vitals.processingRunId }).from(schema.vitals).where(and(eq(schema.vitals.side, selectedSide), + gte(schema.vitals.timestamp, record.enteredBedAt), lt(schema.vitals.timestamp, record.leftBedAt))).orderBy(schema.vitals.timestamp).limit(20001).all() + const movement = db.select({ timestamp: schema.movement.timestamp }).from(schema.movement).where(and(eq(schema.movement.side, selectedSide), + gte(schema.movement.timestamp, record.enteredBedAt), lt(schema.movement.timestamp, record.leftBedAt))).orderBy(schema.movement.timestamp).limit(20001).all() + if (vitals.length + movement.length > 20000) throw new ReferenceInputError('RESOURCE_LIMIT', 'Session exceeds the retained processed-data row budget') + const referenceMetrics = db.select({ metric: schema.referenceMeasurements.metric, count: sql`count(*)` }).from(schema.referenceMeasurements) + .where(and(eq(schema.referenceMeasurements.side, selectedSide), lt(schema.referenceMeasurements.startMs, session.endMs), + or(gt(schema.referenceMeasurements.endMs, session.startMs), and(eq(schema.referenceMeasurements.temporalKind, 'point'), gte(schema.referenceMeasurements.startMs, session.startMs))))) + .groupBy(schema.referenceMeasurements.metric).all() + return { session, + processed: { vitals: describeCoverage(vitals.map(row => row.timestamp.getTime()), session.startMs, session.endMs, 60000), + movement: describeCoverage(movement.map(row => row.timestamp.getTime()), session.startMs, session.endMs, 60000), + legacyUnknownProvenanceCount: vitals.filter(row => row.run === null).length }, + referenceMetrics, comparison: vitals.length > 0, stageReplay: vitals.length > 0 && session.endMs - session.startMs <= 86400000, + signalReplay: false as const, rawWaveforms: 'not_inventoried' as const, + limitations: ['Reference counts include available revisions; select one revision per source family when freezing inputs.', + 'Raw archives and pump/occupancy context have not been retained by this processed-data service.', + 'Movement and vitals availability use nominal one-minute cadence, not inferred continuous signal support.', + ...(session.endMs - session.startMs > 86400000 ? ['Stage replay currently requires a merged window of at most one day.'] : []), + ...(!vitals.length ? ['No stored vitals are available for comparison or stage replay.'] : [])], + } + }) +} diff --git a/src/biometrics/backtest/evaluator-core.ts b/src/biometrics/backtest/evaluator-core.ts new file mode 100644 index 000000000..1db9c3fde --- /dev/null +++ b/src/biometrics/backtest/evaluator-core.ts @@ -0,0 +1,245 @@ +import { assessStageObjective } from './stage-objective' +import { numericSubgroupNames, subgroupReferences, subgroupObjectiveEvidence, subgroupBoundaryExclusions } from './numeric-subgroups' +import { summarizeWaveformResources } from './waveform-resources' +import { assessNumericObjective } from './objective-policy' +import { createNumericNightSummary } from './numeric-night-summary' +import { compareNumericConfigurations } from './numeric-comparison' +import { z } from 'zod' +import { waveformEvaluationSchema } from './waveform-output' +import { compareStageOutputs } from './stage-comparison' +import { createNightSummary } from './night-summary-core' +import { createNightUncertainty } from './uncertainty-core' +import { intersectOutputSupport } from './common-support' +import { classifySleepStages, type SleepStageBlock } from '@/src/lib/sleep-stages' +import { biometricManifest, validateConfig } from '../algorithms/registry-core' +import { alignReferences, DEFAULT_ALIGNMENT } from '../references/alignment' +import { scoreStageDurations } from '../references/stage-scoring' +import { datasetSnapshotSchema, backtestJobSnapshotSchema, numericObjectivePolicies } from './contracts' + +export { waveformEvaluationSchema } from './waveform-output' + +export const backtestWorkSchema = z.object({ + job: backtestJobSnapshotSchema, + dataset: z.object({ manifest: z.unknown(), snapshot: datasetSnapshotSchema }), + waveform: waveformEvaluationSchema.optional(), +}) +const assignmentsSchema = z.object({ splitProtocol: z.object({ assignments: z.array(z.object({ + cohortId: z.string(), sessionIds: z.array(z.number()), split: z.enum(['tuning', 'validation', 'holdout']), startMs: z.number(), endMs: z.number(), +})) }) }) + +function mergeWindows(windows: Array<{ startMs: number, endMs: number }>) { + const merged: typeof windows = [] + for (const window of [...windows].sort((a, b) => a.startMs - b.startMs)) { + const last = merged.at(-1) + if (last && last.endMs >= window.startMs) last.endMs = Math.max(last.endMs, window.endMs) + else merged.push({ ...window }) + } + return merged +} + +/** Pure retained-data evaluator. No DB, network, filesystem or hardware writes. */ +export function createBacktestEvaluator(dependencies: { configHash: (value: unknown) => string, dataHash: (value: unknown) => string, digest: (text: string) => Uint8Array }) { + const { configHash, dataHash, digest } = dependencies + const manifestHash = configHash(biometricManifest) + const summarizeNumericNights = createNumericNightSummary(createNightUncertainty(digest, 'equal_night_mean_common_mae_difference')) + const summarizeStageNights = createNightSummary(createNightUncertainty(digest)) + return function evaluateBacktest(raw: z.input, progress: (completed: number, total: number) => void = () => {}) { + const { job, dataset, waveform } = backtestWorkSchema.parse(raw) + const signalReplay = job.protocol.mode === 'signal_replay' + const stages = job.protocol.mode === 'stage_replay' || (signalReplay && job.protocol.waveformStages === true) + if (dataHash(dataset) !== job.datasetHash || configHash(job.protocol) !== job.protocolHash) throw new Error('Evaluation input integrity mismatch') + const assignments = assignmentsSchema.parse(dataset.manifest).splitProtocol.assignments.filter(row => row.split === job.split) + if (!assignments.length) throw new Error('No nights in selected split') + const configs = job.configs.map((config) => { + const resolved = validateConfig(config.resolved) + if (config.manifestHash !== manifestHash || configHash(resolved) !== config.resolvedHash) throw new Error('Evaluation engine/configuration mismatch') + return { ...config, resolved } + }) + const waveformOutputs = new Map['outputs'][number]>() + if (signalReplay) { + if (!job.waveform || !waveform || dataset.snapshot.version !== 'waveform-dataset-v1' + || dataHash(waveform.outputs) !== waveform.hash) throw new Error('Missing or corrupt waveform evaluation inputs') + const other = validateConfig(job.waveform.otherSide.resolved) + if (job.waveform.otherSide.manifestHash !== manifestHash || configHash(other) !== job.waveform.otherSide.resolvedHash) throw new Error('Invalid other-side waveform configuration') + const expectedConfigs = new Set(configs.map(config => config.resolvedHash)) + const otherSide = dataset.snapshot.side === 'left' ? 'right' : 'left' + for (const output of waveform.outputs) { + const retained = dataset.snapshot.waveformInputs.find(row => row.cohortId === output.cohortId) + const key = `${output.cohortId}:${output.configHash}` + if (!retained || !assignments.some(row => row.cohortId === output.cohortId) || !expectedConfigs.has(output.configHash) + || waveformOutputs.has(key) || output.run.build_hash !== job.waveform.pythonBuildHash + || output.run.config_hashes[dataset.snapshot.side] !== output.configHash + || output.run.config_hashes[otherSide] !== job.waveform.otherSide.resolvedHash + || output.run.counts.scored_outputs !== output.readings.length + || output.run.raw_records < output.run.decoded_records || output.run.decoded_records < output.run.counts.records + || JSON.stringify(output.run.artifact_hashes) !== JSON.stringify(retained.input.artifacts.map(row => row.artifact.sha256)) + || output.readings.some(row => row.configHash !== output.run.config_hashes[row.side] + || row.timestampMs < retained.input.score_start_ms || row.timestampMs >= retained.input.score_end_ms)) throw new Error('Waveform output identity mismatch') + waveformOutputs.set(key, output) + } + if (waveformOutputs.size !== assignments.length * expectedConfigs.size) throw new Error('Missing waveform cohort/configuration output') + } + else if (waveform || job.waveform) throw new Error('Unexpected waveform inputs for processed replay') + const references = dataset.snapshot.references.flatMap(source => source.measurements.map(row => ({ ...row, recordKey: `${source.importId}:${row.recordKey}` }))) + const nights = assignments.map((assignment, nightIndex) => { + const sessions = assignment.sessionIds.map((id) => { + const session = dataset.snapshot.sessions.find(row => row.id === id) + if (!session) throw new Error('Retained session missing') + return session + }) + const windows = mergeWindows(sessions.map(row => ({ startMs: row.startMs, endMs: row.endMs }))) + const vitals = [...new Map(sessions.flatMap(session => session.vitals).map(row => [row.id, row])).values()].sort((a, b) => a.timestampMs - b.timestampMs) + const movement = [...new Map(sessions.flatMap(session => session.movement).map(row => [row.id, row])).values()].sort((a, b) => a.timestampMs - b.timestampMs || a.id - b.id) + // Each unique configuration executes once even when default/current match. + const computed = new Map>() + const evaluateConfig = (config: typeof configs[number]) => windows.map((window) => { + if (window.endMs - window.startMs > 86400000) throw new Error('Merged replay window exceeds the one-day evaluation budget') + const output = waveformOutputs.get(`${assignment.cohortId}:${config.resolvedHash}`) + const observations = signalReplay + ? (output?.readings.filter(row => row.side === dataset.snapshot.side).map((row, id) => ({ + id, timestampMs: row.timestampMs, heartRate: row.heartRate, hrv: row.hrvIndex, breathingRate: row.breathingRate, processingRunId: null, + })) ?? []) + : vitals + const rows = observations.filter(row => row.timestampMs >= window.startMs && row.timestampMs < window.endMs) + const moves = movement.filter(row => row.timestampMs >= window.startMs && row.timestampMs < window.endMs) + const numeric = (['heart_rate', 'breathing_rate', 'window_ibi_variability_v1'] as const).map((metric) => { + const aligned = alignReferences(metric, metric === 'window_ibi_variability_v1' ? 300000 : null, + rows.map(row => ({ timestampMs: row.timestampMs, value: metric === 'heart_rate' ? row.heartRate : metric === 'breathing_rate' ? row.breathingRate : row.hrv })), + references.filter(row => row.metric === metric && row.startMs >= window.startMs && (row.temporalKind === 'point' ? row.startMs < window.endMs : row.endMs <= window.endMs)), + { ...DEFAULT_ALIGNMENT, pointToleranceMs: job.protocol.pointToleranceMs, clockOffsetMs: job.protocol.clockOffsetMs }) + // The report stores summaries; full input samples remain in the dataset. + return { metric, status: aligned.status, metrics: aligned.metrics, pairedCount: aligned.pairedCount, + referenceCoverage: aligned.referenceCoverage, outputCoverage: aligned.outputCoverage, unmatchedReferences: aligned.unmatchedReferences, + unmatchedObservations: aligned.unmatchedObservations, exclusions: aligned.exclusions } + }) + let blocks: SleepStageBlock[] = [] + if (stages) { + const epochs = classifySleepStages(rows.map(row => ({ ...row, timestamp: new Date(row.timestampMs) })), + moves.map(row => ({ ...row, timestamp: new Date(row.timestampMs) })), job.protocol.assumedStageCalibrationQuality, config.resolved.sleep_stages.ruleset) + blocks = epochs.map(epoch => ({ start: epoch.start, end: Math.min(window.endMs, epoch.start + epoch.duration), stage: epoch.stage })).filter(row => row.end > row.start) + } + return { ...window, observationCount: rows.length, numeric, + stage: stages + ? scoreStageDurations(blocks.map(block => ({ ...block, start: block.start + job.protocol.clockOffsetMs, end: block.end + job.protocol.clockOffsetMs })), references.filter(row => row.metric === 'sleep_stages'), window.startMs, window.endMs) + : null, + // Bounded complete traces (the dataset has at most 20k rows). + blocks, + } + }) + const evaluations = configs.map((config) => { + let windows = computed.get(config.resolvedHash) + if (!windows) { + windows = evaluateConfig(config) + computed.set(config.resolvedHash, windows) + } + return { label: config.label, revisionId: config.revisionId, configHash: config.resolvedHash, windows } + }) + const shift = (blocks: SleepStageBlock[]) => blocks.map(block => ({ ...block, + start: block.start + job.protocol.clockOffsetMs, end: block.end + job.protocol.clockOffsetMs })) + const sharedSupport = stages + ? windows.map((window, index) => + intersectOutputSupport(evaluations.map(evaluation => shift(evaluation.windows[index].blocks)), window.startMs, window.endMs)) + : [] + const comparisons = stages + ? evaluations.flatMap(candidate => evaluations + .filter(baseline => (candidate.label === 'candidate' && baseline.label !== 'candidate') || (candidate.label === 'current' && baseline.label === 'default')) + .map(baseline => ({ baseline: baseline.label, candidate: candidate.label, + baselineHash: baseline.configHash, candidateHash: candidate.configHash, + windows: candidate.windows.map((window, index) => { + return { startMs: window.startMs, endMs: window.endMs, + ...compareStageOutputs(shift(baseline.windows[index].blocks), shift(window.blocks), + references.filter(row => row.metric === 'sleep_stages'), window.startMs, window.endMs, sharedSupport[index]) } + }), + }))) + : [] + const compareWindow = (window: typeof windows[number], metric: 'heart_rate' | 'breathing_rate' | 'window_ibi_variability_v1', selectedReferences: typeof references) => compareNumericConfigurations( + metric, metric === 'window_ibi_variability_v1' ? 300000 : null, + configs.map(config => ({ label: config.label, configHash: config.resolvedHash, + observations: (waveformOutputs.get(`${assignment.cohortId}:${config.resolvedHash}`)?.readings ?? []) + .filter(row => row.side === dataset.snapshot.side && row.timestampMs >= window.startMs && row.timestampMs < window.endMs) + .map(row => ({ timestampMs: row.timestampMs, value: metric === 'heart_rate' ? row.heartRate : metric === 'breathing_rate' ? row.breathingRate : row.hrvIndex })), + })), selectedReferences, + { ...DEFAULT_ALIGNMENT, pointToleranceMs: job.protocol.pointToleranceMs, clockOffsetMs: job.protocol.clockOffsetMs }) + const windowReferences = (window: typeof windows[number], metric: string) => references.filter(row => row.metric === metric && row.startMs >= window.startMs + && (row.temporalKind === 'point' ? row.startMs < window.endMs : row.endMs <= window.endMs)) + const numericComparisons = signalReplay + ? windows.map(window => ({ ...window, + metrics: (['heart_rate', 'breathing_rate', 'window_ibi_variability_v1'] as const).flatMap(metric => compareWindow(window, metric, windowReferences(window, metric))), + })) + : [] + const subgroupMetrics = numericObjectivePolicies(job.protocol).map(policy => policy.metric) + const numericSubgroups = signalReplay && subgroupMetrics.length + ? numericSubgroupNames.map(name => ({ name, numericComparisons: windows.map(window => ({ ...window, + metrics: subgroupMetrics.flatMap(metric => compareWindow(window, metric, subgroupReferences(name, windowReferences(window, metric), window))), + excludedBoundaryReferenceCount: subgroupMetrics.reduce((sum, metric) => sum + subgroupBoundaryExclusions(name, windowReferences(window, metric), window), 0), + excludedByMetric: Object.fromEntries(subgroupMetrics.map(metric => [metric, subgroupBoundaryExclusions(name, windowReferences(window, metric), window)])), + })) })) + : [] + progress(nightIndex + 1, assignments.length) + return { cohortId: assignment.cohortId, sessionIds: assignment.sessionIds, + uniqueObservationCount: signalReplay + ? new Set([...waveformOutputs.values()].filter(output => output.cohortId === assignment.cohortId) + .flatMap(output => output.readings.filter(row => row.side === dataset.snapshot.side + && windows.some(window => row.timestampMs >= window.startMs && row.timestampMs < window.endMs)).map(row => row.timestampMs))).size + : vitals.length, + ...(signalReplay ? { numericComparisons, numericSubgroups, recordedObservationCount: vitals.length, observationCountBasis: 'union_of_derived_timestamps' } : {}), evaluations, comparisons } + }) + const stageNightSummary = summarizeStageNights(nights, job.protocol.nightUncertainty) + const stagePolicy = job.protocol.stageObjectivePolicy + const stageObjectiveAssessments = stagePolicy + ? stageNightSummary.filter(row => row.candidate === 'candidate').map(summary => ({ + baseline: summary.baseline, candidate: summary.candidate, baselineHash: summary.baselineHash, candidateHash: summary.candidateHash, + ...assessStageObjective(stagePolicy, { split: job.split, holdoutUse: job.holdoutUse, intervalLower: summary.uncertainty.interval?.lower ?? null, operational: null, + nights: summary.nights.map(night => ({ cohortId: night.cohortId, pairedDurationMs: night.commonSupport.candidate.pairedDurationMs, + agreementDifference: night.commonAgreementDifference, macroF1Difference: night.commonMacroF1Difference, + candidateOutputCoverage: night.ownSupport.candidate.outputCoverage, baselineOutputMs: night.ownSupport.baseline.outputMs, commonOutputMs: night.commonOutputDurationMs, + perStage: night.commonSupport.candidate.perStage.map((row) => { + const baseline = night.commonSupport.baseline.perStage.find(item => item.stage === row.stage) + return { stage: row.stage, referenceMs: row.referenceMs, f1Difference: row.f1 === null || baseline?.f1 == null ? null : row.f1 - baseline.f1 } + }), + })), + }, null), + })) + : [] + const numericNightSummary = signalReplay ? summarizeNumericNights(nights, job.protocol.nightUncertainty) : [] + const numericSubgroupSummary = signalReplay && numericObjectivePolicies(job.protocol).length + ? numericSubgroupNames.flatMap(name => summarizeNumericNights(nights.map(night => ({ cohortId: night.cohortId, + numericComparisons: night.numericSubgroups?.find(group => group.name === name)?.numericComparisons }))) + .map(summary => ({ name, version: 'reference-defined-numeric-subgroups-v1' as const, + referenceMembershipBasis: 'native_reference_time' as const, observationCoverageScope: 'complete_scoring_windows' as const, + excludedBoundaryReferenceCount: nights.reduce((sum, night) => sum + (night.numericSubgroups?.find(group => group.name === name)?.numericComparisons.reduce((count, window) => count + (window.excludedByMetric[summary.metric] ?? 0), 0) ?? 0), 0), summary }))) + : [] + const objectiveAssessments = numericObjectivePolicies(job.protocol).flatMap(objectivePolicy => numericNightSummary + .filter(summary => summary.metric === objectivePolicy.metric && summary.candidate === 'candidate') + .map(summary => ({ baseline: summary.baseline, candidate: summary.candidate, baselineHash: summary.baselineHash, candidateHash: summary.candidateHash, + ...assessNumericObjective(objectivePolicy, { + metricCompatible: !nights.some(night => night.numericComparisons?.some(window => window.metrics.some(row => row.metric === summary.metric + && row.baseline === summary.baseline && row.candidate === summary.candidate && row.commonSupport.candidate.status === 'incompatible'))), + split: job.split, holdoutUse: job.holdoutUse, + nights: summary.nights.map(night => ({ cohortId: night.cohortId, pairedReferences: night.commonSupport.candidate.pairedCount, + maeDifference: night.commonMaeDifference, lostObservations: night.lostObservationTimestamps, + baselineObservationCount: nights.find(row => row.cohortId === night.cohortId)?.numericComparisons?.reduce((sum, window) => sum + + (window.metrics.find(row => row.metric === summary.metric && row.baseline === summary.baseline && row.candidate === summary.candidate)?.ownSupport.baseline.validObservationCount ?? 0), 0) ?? 0 })), + intervalUpper: summary.uncertainty.interval?.upper ?? null, + subgroups: subgroupObjectiveEvidence(numericSubgroupSummary.map(group => ({ ...group.summary, name: group.name })), summary, objectivePolicy.minimumPairedReferencesPerNight), operational: null, + }, null), + }))) + return { side: dataset.snapshot.side, version: signalReplay ? 'waveform-backtest-report-v1' as const : 'processed-backtest-report-v1' as const, + ...(signalReplay ? { waveform: { ...(job.waveform?.baselineProvenance ? { baselineProvenance: job.waveform.baselineProvenance } : {}), pythonBuildHash: job.waveform?.pythonBuildHash, otherSideConfigHash: job.waveform?.otherSide.resolvedHash, ...waveform, resources: summarizeWaveformResources(waveform?.outputs ?? []) } } : {}), datasetId: job.datasetId, datasetHash: job.datasetHash, + evaluatorBuildHash: job.evaluatorBuildHash, protocol: job.protocol, protocolHash: job.protocolHash, manifestHash, support: signalReplay ? 'identical_retained_waveform_inputs' as const : 'identical_retained_epoch_intervals' as const, split: job.split, holdoutUse: job.holdoutUse, + ...(signalReplay ? { numericNightSummary, numericSubgroupSummary } : {}), + objectiveAssessments, stageObjectiveAssessments, + status: 'insufficient_evidence' as const, evidence: job.evidence, nights, stageNightSummary, + calibration: { source: 'explicit_assumption' as const, quality: job.protocol.assumedStageCalibrationQuality }, + limitations: ['Only a qualified, complete independent evaluation can establish improvement in reference agreement.', + ...(signalReplay ? ['Numeric comparisons retain own-support scores and scores on observation timestamps shared by all configurations; coverage loss and ambiguous timestamps are reported separately. Common-support differences require qualified objective and coverage assessment before they support promotion.'] : []), + signalReplay + ? 'Numeric readings are replayed from retained waveforms; missing historical context, event-time substitution and cold starts constrain parity.' + : 'Numeric readings are original observations; parameter changes to waveform processing are not evaluated by processed-data replay.', + 'Stages use the shared retrospective classifier, including its bounded gap-duration semantics; they are not causal predictions.', + 'Calibration quality is an explicit assumption, not recovered historical sensor calibration.', + 'Overlapping sessions share one input set; disconnected sessions are classified separately.', + 'Holdout exposure must be rechecked before any later promotion decision.'], + } + } +} diff --git a/src/biometrics/backtest/evaluator.ts b/src/biometrics/backtest/evaluator.ts new file mode 100644 index 000000000..4ba5ebb8a --- /dev/null +++ b/src/biometrics/backtest/evaluator.ts @@ -0,0 +1,5 @@ +import { createHash } from 'node:crypto' +import { configHash, dataHash } from '../algorithms/canonical' +import { createBacktestEvaluator } from './evaluator-core' +export { backtestWorkSchema, waveformEvaluationSchema } from './evaluator-core' +export const evaluateBacktest = createBacktestEvaluator({ configHash, dataHash, digest: text => createHash('sha256').update(text).digest() }) diff --git a/src/biometrics/backtest/execution-budget.ts b/src/biometrics/backtest/execution-budget.ts new file mode 100644 index 000000000..539bcdf49 --- /dev/null +++ b/src/biometrics/backtest/execution-budget.ts @@ -0,0 +1,14 @@ +import { z } from 'zod' + +/** Trusted deployment settings, never an experiment/configuration or RPC field. + * These ceilings permit qualification of whole nights; they do not enable a + * worker host or replace ongoing resource admission and lease fencing. + */ +export const replayBudgetSchema = z.object({ + jobWallMs: z.number().int().min(1000).max(48 * 60 * 60 * 1000), + waveformWallMs: z.number().int().min(1000).max(60 * 60 * 1000), + waveformCpuSeconds: z.number().int().min(1).max(3600), +}).strict().refine(budget => budget.waveformWallMs <= budget.jobWallMs + && budget.waveformCpuSeconds * 1000 <= budget.waveformWallMs, +'Waveform CPU and wall budgets must fit within the job budget') +export type ReplayBudget = z.infer diff --git a/src/biometrics/backtest/host-startup.ts b/src/biometrics/backtest/host-startup.ts new file mode 100644 index 000000000..e0f257bff --- /dev/null +++ b/src/biometrics/backtest/host-startup.ts @@ -0,0 +1,11 @@ +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import type * as schema from '@/src/db/biometrics-schema' +import { configuredReplayPolicy, replayAdmitter, replayHealthAdmission } from './admission-service' +import { startBacktestHost } from './host' + +/** Enable polling only for an explicitly configured deployment. Startup does + * not substitute for qualification or fresh health checks on every operation. */ +export function startConfiguredBacktestHost(db: BetterSQLite3Database) { + if (!configuredReplayPolicy()) return null + return startBacktestHost(db, replayAdmitter(db), replayHealthAdmission(db)) +} diff --git a/src/biometrics/backtest/host.ts b/src/biometrics/backtest/host.ts new file mode 100644 index 000000000..ce3cd189e --- /dev/null +++ b/src/biometrics/backtest/host.ts @@ -0,0 +1,101 @@ +import type { HealthAdmission } from './job-operations' +import { randomUUID } from 'node:crypto' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import type * as schema from '@/src/db/biometrics-schema' +import { coordinatorRequest } from './coordinator-service' + +interface HostOptions { + execute: (signal: AbortSignal) => Promise + collect: (signal: AbortSignal) => void | Promise + onError?: (phase: 'execution' | 'cleanup') => void + intervalMs?: number + clock?: () => number +} + +/** One serialized host loop. The runner still owns the database lease, so + * multiple processes cannot bypass fencing by each creating a local host. */ +export class BacktestHost { + private stopped = false + private active: Promise | null = null + private timer: ReturnType | null = null + private controller = new AbortController() + private lastCollection = -Infinity + private readonly clock: () => number + constructor(private readonly options: HostOptions) { + this.clock = options.clock ?? Date.now + if (!Number.isSafeInteger(options.intervalMs ?? 5000) || (options.intervalMs ?? 5000) < 10) throw new Error('Invalid backtest host interval') + } + + start() { + if (!this.stopped && !this.active && !this.timer) this.tick() + } + + private report(phase: 'execution' | 'cleanup') { + try { + this.options.onError?.(phase) + } + catch { /* Observability must not stop recovery. */ } + } + + private tick() { + if (this.stopped || this.active) return + this.timer = null + this.active = this.run().finally(() => { + this.active = null + if (!this.stopped) { + this.timer = setTimeout(() => this.tick(), this.options.intervalMs ?? 5000) + this.timer.unref() + } + }) + } + + private async run() { + if (this.clock() - this.lastCollection >= 1800000) { + this.lastCollection = this.clock() + try { + await this.options.collect(this.controller.signal) + } + catch { + this.report('cleanup') + } + } + if (this.stopped) return + try { + await this.options.execute(this.controller.signal) + } + catch { + this.report('execution') + } + } + + /** Abort the child, then await its lease interruption before DB closure. */ + async stop() { + this.stopped = true + if (this.timer) clearTimeout(this.timer) + this.timer = null + this.controller.abort() + await this.active + } +} + +const shared = globalThis as typeof globalThis & { __biometricBacktestHost?: BacktestHost } +/** Admission is mandatory and supplied by the deployment's resource policy. + * There is deliberately no permissive fallback for missing health evidence. */ +export function startBacktestHost(db: BetterSQLite3Database, admit: () => Promise, healthAdmission?: HealthAdmission) { + if (shared.__biometricBacktestHost) return shared.__biometricBacktestHost + const owner = `core:${process.pid}:${randomUUID()}` + const host = new BacktestHost({ + execute: signal => coordinatorRequest(db, { type: 'execute', owner }, { signal, admit, healthAdmission }), + collect: signal => coordinatorRequest(db, { type: 'collectArtifacts' }, { signal, admit, healthAdmission }).then(() => {}), + onError: phase => console.warn(`[biometrics] Backtest ${phase} failed; host will retry`), + }) + shared.__biometricBacktestHost = host + host.start() + return host +} +export async function stopBacktestHost() { + const host = shared.__biometricBacktestHost + if (!host) return + await host.stop() + if (shared.__biometricBacktestHost === host) delete shared.__biometricBacktestHost +} diff --git a/src/biometrics/backtest/job-operations.ts b/src/biometrics/backtest/job-operations.ts new file mode 100644 index 000000000..fbc554110 --- /dev/null +++ b/src/biometrics/backtest/job-operations.ts @@ -0,0 +1,64 @@ +import { processGroupMemorySchema, type ProcessGroupMemory } from './process-group-memory-contract' +import { z } from 'zod' + +/** Internal supervisor evidence, never accepted from experiment/RPC payloads. */ +export const admissionReceiptSchema = z.object({ + allowed: z.boolean(), + memory: processGroupMemorySchema.optional(), + sample: z.object({ qualificationId: z.string().min(1).max(200), checkedAtMs: z.number().int().nonnegative(), + ingestionAgeMs: z.number().int().nonnegative() }).strict().nullable(), +}).strict() +export const jobOperationsSchema = z.object({ + version: z.literal('backtest-job-operations-v1'), wallMs: z.number().finite().nonnegative(), + scope: z.literal('admission_through_prepublication'), + memory: processGroupMemorySchema.nullable().optional(), + ingestion: z.object({ source: z.literal('supervisor_live_piezo_receipt_age'), + checks: z.number().int().nonnegative(), samples: z.number().int().nonnegative(), allChecksMeasured: z.boolean(), + qualificationId: z.string().nullable(), maximumObservedAgeMs: z.number().int().nonnegative().nullable(), + maximumObservationGapMs: z.number().finite().nonnegative(), nominalIntervalMs: z.literal(5000), continuousMeasurement: z.literal(false), + }).strict(), +}).strict() + +export type AdmissionReceipt = z.infer +export type HealthAdmission = () => Promise + +/** Bounded accumulation; no health frames or unbounded sample arrays. Clock + * gaps are measured monotonically, independently of wall-clock adjustments. */ +export function createJobOperations(now: () => number = () => performance.now()) { + const start = now() + let last = start, checks = 0, samples = 0, maximumGapMs = 0, maximumIngestionAgeMs = 0 + let memory: ProcessGroupMemory | null = null + let qualificationId: string | null = null, consistent = true + return { + record(raw: unknown) { + const at = now() + maximumGapMs = Math.max(maximumGapMs, at - last) + last = at + checks++ + const receipt = admissionReceiptSchema.safeParse(raw) + if (receipt.success && receipt.data.memory) memory = receipt.data.memory + if (!receipt.success || !receipt.data.allowed || !receipt.data.sample) { + consistent = false + return + } + const sample = receipt.data.sample + if (qualificationId !== null && qualificationId !== sample.qualificationId) consistent = false + qualificationId ??= sample.qualificationId + maximumIngestionAgeMs = Math.max(maximumIngestionAgeMs, sample.ingestionAgeMs) + samples++ + }, + finish() { + const end = now() + return { version: 'backtest-job-operations-v1' as const, wallMs: Math.max(0, end - start), + scope: 'admission_through_prepublication' as const, memory, + ingestion: { source: 'supervisor_live_piezo_receipt_age' as const, checks, samples, + allChecksMeasured: consistent && checks >= 2 && samples === checks, + qualificationId: consistent ? qualificationId : null, + maximumObservedAgeMs: samples > 0 ? maximumIngestionAgeMs : null, + maximumObservationGapMs: Math.max(maximumGapMs, end - last), + nominalIntervalMs: 5000, + continuousMeasurement: false as const }, + } + }, + } +} diff --git a/src/biometrics/backtest/jobs.ts b/src/biometrics/backtest/jobs.ts new file mode 100644 index 000000000..afdd33f19 --- /dev/null +++ b/src/biometrics/backtest/jobs.ts @@ -0,0 +1,340 @@ +import { randomUUID } from 'node:crypto' +import { dirname, join } from 'node:path' +import { and, asc, desc, eq, gt, inArray, lt, lte, or, sql } from 'drizzle-orm' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import { z } from 'zod' +import * as schema from '@/src/db/biometrics-schema' +import { configHash, dataHash } from '../algorithms/canonical' +import { manifestHash, validateConfig } from '../algorithms/registry' +import { ReferenceInputError } from '../references/time' +import { DatasetStore } from './datasets' +import { MAX_REPORT_BYTES, REPORT_BUDGET_BYTES, ReportArtifactStore, reportArtifactSchema } from './artifacts' + +const jobs = schema.biometricBacktestJobs +const experiments = schema.biometricExperiments +const results = schema.biometricBacktestResults +const exposure = schema.biometricHoldoutAccesses +const manifestSchema = z.object({ splitProtocol: z.object({ assignments: z.array(z.object({ split: z.string(), startMs: z.number(), endMs: z.number() })) }) }) +const key = z.string().min(1).max(200) +export { experimentProtocolSchema, enqueueBacktestSchema } from './contracts' +import { experimentProtocolSchema, enqueueBacktestSchema } from './contracts' +const progressSchema = z.object({ phase: z.enum(['queued', 'loading', 'evaluating', 'reporting', 'complete']), + completedNights: z.number().int().min(0).max(60), totalNights: z.number().int().min(0).max(60) }).strict() +const leaseSchema = z.object({ id: z.string().uuid(), owner: key, token: z.number().int().positive() }).strict() +export type JobLease = z.infer + +/** Leases fence every publication; all expensive work happens outside SQLite. */ +export class BacktestJobStore { + constructor(private readonly db: BetterSQLite3Database, private readonly now = Date.now, + private readonly artifactStore?: ReportArtifactStore) {} + + private storage() { + if (this.artifactStore) return this.artifactStore + const main = this.db.all<{ name: string, file: string }>(sql`PRAGMA database_list`).find(row => row.name === 'main') + return main?.file ? new ReportArtifactStore(join(dirname(main.file), 'biometric-report-artifacts')) : undefined + } + + private artifacts() { + const store = this.storage() + if (!store) throw new ReferenceInputError('RESOURCE_LIMIT', 'Large reports require a persistent artifact directory') + return store + } + + collectOrphanedArtifacts() { + const store = this.artifacts() + return store.withPublication(() => { + const referenced = new Set(this.db.select({ report: results.report }).from(results).all().flatMap((row) => { + // Preserve named future-version descriptors during downgrade. + const artifact = z.object({ name: z.string() }).safeParse(row.report) + return artifact.success ? [artifact.data.name] : [] + })) + return store.collect(referenced, this.now()) + }) + } + + createExperiment(datasetId: string, rawProtocol: z.input, idempotencyKey: string = randomUUID()) { + const protocol = experimentProtocolSchema.parse(rawProtocol) + if (protocol.waveformStages === undefined) delete protocol.waveformStages + key.parse(idempotencyKey) + const requestHash = configHash({ datasetId, protocol }) + const dataset = new DatasetStore(this.db).read(datasetId) + if (protocol.mode === 'signal_replay' && dataset.snapshot.version !== 'waveform-dataset-v1') throw new ReferenceInputError('INSUFFICIENT_RAW_DATA', 'Signal replay requires retained waveform inputs') + return this.db.transaction((tx) => { + const existing = tx.select().from(experiments).where(eq(experiments.idempotencyKey, idempotencyKey)).get() + if (existing) { + if (existing.requestHash !== requestHash) throw new ReferenceInputError('REVISION_CONFLICT', 'Experiment idempotency key changed') + return existing + } + const total = tx.select({ count: sql`count(*)` }).from(experiments).get()?.count ?? 0 + if (total >= 1000) throw new ReferenceInputError('RESOURCE_LIMIT', 'Experiment history has reached its 1000-entry bound') + return tx.insert(experiments).values({ id: randomUUID(), datasetId, idempotencyKey, requestHash, side: dataset.side, + protocol, protocolHash: configHash(protocol), createdAtMs: this.now() }).returning().get() + }, { behavior: 'immediate' }) + } + + getExperiment(id: string) { + const experiment = this.db.select().from(experiments).where(eq(experiments.id, id)).get() + if (!experiment) throw new ReferenceInputError('NOT_FOUND', 'Experiment not found') + return { ...experiment, protocol: experimentProtocolSchema.parse(experiment.protocol) } + } + + pageExperiments(side: 'left' | 'right', cursor?: { createdAtMs: number, id: string }) { + const rows = this.db.select().from(experiments).where(and(eq(experiments.side, side), cursor + ? or(lt(experiments.createdAtMs, cursor.createdAtMs), and(eq(experiments.createdAtMs, cursor.createdAtMs), lt(experiments.id, cursor.id))) + : undefined)) + .orderBy(desc(experiments.createdAtMs), desc(experiments.id)).limit(51).all() + const items = rows.slice(0, 50).map(row => ({ ...row, protocol: experimentProtocolSchema.parse(row.protocol) })) + const last = items.at(-1) + return { items, nextCursor: rows.length > 50 && last ? { createdAtMs: last.createdAtMs, id: last.id } : null } + } + + holdoutUseCount(datasetId: string) { + const dataset = this.db.select({ side: schema.biometricDatasets.side, manifest: schema.biometricDatasets.manifest }) + .from(schema.biometricDatasets).where(eq(schema.biometricDatasets.id, datasetId)).get() + if (!dataset) throw new ReferenceInputError('NOT_FOUND', 'Dataset unavailable') + const windows = manifestSchema.parse(dataset.manifest).splitProtocol.assignments.filter(row => row.split === 'holdout') + if (!windows.length) return 0 + return this.db.selectDistinct({ jobId: exposure.jobId }).from(exposure).where(and(eq(exposure.side, dataset.side), + or(...windows.map(window => and(lt(exposure.startMs, window.endMs), gt(exposure.endMs, window.startMs)))), + )).all().length + } + + enqueue(raw: z.input) { + const input = enqueueBacktestSchema.parse(raw) + if (input.waveform === undefined) delete input.waveform + if (new Set(input.configs.map(config => config.label)).size !== input.configs.length) throw new ReferenceInputError('INVALID_CONFIG', 'Baseline labels must be unique') + for (const config of input.configs) { + const resolved = validateConfig(config.resolved) + if (config.manifestHash !== manifestHash || configHash(resolved) !== config.resolvedHash) throw new ReferenceInputError('INVALID_CONFIG', 'Resolved config or engine hash does not match this build') + } + if (Buffer.byteLength(JSON.stringify(input)) > 64 * 1024) throw new ReferenceInputError('RESOURCE_LIMIT', 'Backtest configuration snapshot exceeds 64 KiB') + const requestHash = configHash(input) + return this.db.transaction((tx) => { + const existing = tx.select().from(jobs).where(eq(jobs.idempotencyKey, input.idempotencyKey)).get() + if (existing) { + if (existing.requestHash !== requestHash) throw new ReferenceInputError('REVISION_CONFLICT', 'Backtest idempotency key changed') + return existing + } + const total = tx.select({ count: sql`count(*)` }).from(jobs).get()?.count ?? 0 + if (total >= 1000) throw new ReferenceInputError('RESOURCE_LIMIT', 'Job history has reached its 1000-entry bound') + const experiment = tx.select().from(experiments).where(eq(experiments.id, input.experimentId)).get() + if (!experiment) throw new ReferenceInputError('NOT_FOUND', 'Experiment not found') + const protocol = experimentProtocolSchema.parse(experiment.protocol) + if ((protocol.mode === 'signal_replay') !== Boolean(input.waveform)) throw new ReferenceInputError('INVALID_CONFIG', 'Waveform replay requires pinned Python and other-side configuration') + if (input.waveform && (input.waveform.otherSide.manifestHash !== manifestHash + || configHash(validateConfig(input.waveform.otherSide.resolved)) !== input.waveform.otherSide.resolvedHash)) throw new ReferenceInputError('INVALID_CONFIG', 'Other-side configuration identity mismatch') + const dataset = tx.select({ contentHash: schema.biometricDatasets.contentHash, releasedAtMs: schema.biometricDatasets.releasedAtMs, manifest: schema.biometricDatasets.manifest }) + .from(schema.biometricDatasets).where(eq(schema.biometricDatasets.id, experiment.datasetId)).get() + if (!dataset) throw new ReferenceInputError('NOT_FOUND', 'Dataset unavailable') + if (dataset.releasedAtMs !== null) throw new ReferenceInputError('DATASET_RELEASED', 'Retained dataset inputs were explicitly released') + const manifest = manifestSchema.parse(dataset.manifest) + const totalNights = manifest.splitProtocol.assignments.filter(group => group.split === input.split).length + if (!totalNights) throw new ReferenceInputError('INSUFFICIENT_NIGHTS', 'The frozen dataset has no nights assigned to this split') + let holdoutUse = this.holdoutUseCount(experiment.datasetId) + if (input.split === 'holdout') { + holdoutUse++ + tx.update(experiments).set({ holdoutAccesses: experiment.holdoutAccesses + 1 }).where(eq(experiments.id, experiment.id)).run() + } + const now = this.now() + const job = tx.insert(jobs).values({ id: randomUUID(), experimentId: experiment.id, idempotencyKey: input.idempotencyKey, requestHash, + input: { version: 'backtest-job-v1', ...input, datasetId: experiment.datasetId, datasetHash: dataset.contentHash, + protocol: experiment.protocol, protocolHash: experiment.protocolHash, holdoutUse, + evidence: input.split === 'holdout' && holdoutUse > 1 ? 'reused_holdout_is_validation' : 'descriptive_only' }, + state: 'queued', progress: { phase: 'queued', completedNights: 0, totalNights }, createdAtMs: now, updatedAtMs: now, + }).returning().get() + if (input.split === 'holdout') { + for (const window of manifest.splitProtocol.assignments.filter(row => row.split === 'holdout')) { + tx.insert(exposure).values({ id: randomUUID(), jobId: job.id, side: experiment.side, startMs: window.startMs, endMs: window.endMs, accessedAtMs: now }).run() + } + } + return job + }, { behavior: 'immediate' }) + } + + page(side: 'left' | 'right', cursor?: { createdAtMs: number, id: string }) { + const rows = this.db.select({ job: jobs }).from(jobs).innerJoin(experiments, eq(jobs.experimentId, experiments.id)) + .where(and(eq(experiments.side, side), cursor + ? or(lt(jobs.createdAtMs, cursor.createdAtMs), and(eq(jobs.createdAtMs, cursor.createdAtMs), lt(jobs.id, cursor.id))) + : undefined)) + .orderBy(desc(jobs.createdAtMs), desc(jobs.id)).limit(51).all().map(row => row.job) + const items = rows.slice(0, 50) + const last = items.at(-1) + return { items, nextCursor: rows.length > 50 && last ? { createdAtMs: last.createdAtMs, id: last.id } : null } + } + + get(id: string) { + const job = this.db.select().from(jobs).where(eq(jobs.id, id)).get() + if (!job) throw new ReferenceInputError('NOT_FOUND', 'Backtest job not found') + return job + } + + result(id: string) { + const storage = this.storage() + const read = () => { + const job = this.get(id) + if (job.state !== 'succeeded') return null + const result = this.db.select().from(results).where(eq(results.jobId, id)).get() + if (!result || result.fencingToken !== job.fencingToken) throw new ReferenceInputError('RESULT_CORRUPT', 'Backtest result integrity check failed') + const input = z.object({ datasetId: z.string() }).parse(job.input) + const currentHoldoutUseCount = this.holdoutUseCount(input.datasetId) + if (result.releasedAtMs !== null) return { ...result, report: null, available: false, currentHoldoutUseCount } + const artifact = reportArtifactSchema.safeParse(result.report) + const report = artifact.success ? this.artifacts().read(artifact.data) : result.report + if (dataHash(report) !== result.contentHash) throw new ReferenceInputError('RESULT_CORRUPT', 'Backtest result integrity check failed') + return { ...result, report, available: true, currentHoldoutUseCount } + } + // A release cannot unlink an artifact between reading its descriptor and bytes. + return storage ? storage.withPublication(read) : read() + } + + /** Release payload bytes, retaining winning-attempt and holdout audit records. + * The caller must identify the exact report hash it intends to release. */ + releaseResult(id: string, expectedContentHash: string) { + z.string().regex(/^[a-f0-9]{64}$/).parse(expectedContentHash) + const storage = this.storage() + const release = () => { + const previous = this.db.transaction((tx) => { + const job = this.get(id) + if (job.state !== 'succeeded') throw new ReferenceInputError('INVALID_STATE', 'Only completed reports can be released') + const result = tx.select().from(results).where(eq(results.jobId, id)).get() + if (!result || result.fencingToken !== job.fencingToken) throw new ReferenceInputError('RESULT_CORRUPT', 'Backtest result integrity check failed') + if (result.contentHash !== expectedContentHash) throw new ReferenceInputError('REVISION_CONFLICT', 'Report hash changed') + if (result.releasedAtMs === null) { + tx.update(results).set({ report: {}, releasedAtMs: this.now() }).where(eq(results.jobId, id)).run() + } + return result.report + }, { behavior: 'immediate' }) + const artifact = reportArtifactSchema.safeParse(previous) + if (artifact.success && storage) { + const stillReferenced = this.db.select({ report: results.report }).from(results).all().some((row) => { + const named = z.object({ name: z.string() }).safeParse(row.report) + return named.success && named.data.name === artifact.data.name + }) + if (!stillReferenced) storage.removeReleased(artifact.data) + } + } + if (storage) storage.withPublication(release) + else release() + return this.result(id) + } + + /** Recover expired attempts without granting stale owners publication rights. */ + recover() { + const now = this.now() + return this.db.transaction((tx) => { + const expired = tx.select().from(jobs).where(and(inArray(jobs.state, ['running', 'cancel_requested']), lte(jobs.leaseExpiresAtMs, now))).all() + for (const job of expired) { + tx.update(jobs).set({ state: job.state === 'cancel_requested' ? 'cancelled' : job.attempt >= 3 ? 'failed' : 'interrupted', + leaseOwner: null, leaseExpiresAtMs: null, error: job.state === 'cancel_requested' ? null : 'Worker lease expired', updatedAtMs: now, + }).where(eq(jobs.id, job.id)).run() + } + return expired.length + }, { behavior: 'immediate' }) + } + + claim(owner: string, leaseMs = 30000) { + key.parse(owner) + z.number().int().min(1000).max(120000).parse(leaseMs) + this.recover() + return this.db.transaction((tx) => { + const now = this.now() + const active = tx.select({ id: jobs.id }).from(jobs).where(and(inArray(jobs.state, ['running', 'cancel_requested']), gt(jobs.leaseExpiresAtMs, now))).get() + if (active) return null // one background replay at a time across processes + const job = tx.select().from(jobs).where(inArray(jobs.state, ['queued', 'interrupted'])).orderBy(asc(jobs.createdAtMs), asc(jobs.id)).get() + if (!job) return null + return tx.update(jobs).set({ state: 'running', leaseOwner: owner, leaseExpiresAtMs: now + leaseMs, + heartbeatAtMs: now, fencingToken: job.fencingToken + 1, attempt: job.attempt + 1, + progress: { ...progressSchema.parse(job.progress), phase: 'loading', completedNights: 0 }, error: null, updatedAtMs: now, + }).where(eq(jobs.id, job.id)).returning().get() + }, { behavior: 'immediate' }) + } + + private owned(lease: JobLease) { + leaseSchema.parse(lease) + const job = this.get(lease.id) + if (job.leaseOwner !== lease.owner || job.fencingToken !== lease.token || (job.leaseExpiresAtMs ?? 0) <= this.now() + || !['running', 'cancel_requested'].includes(job.state)) throw new ReferenceInputError('STALE_LEASE', 'Worker lease is no longer authoritative') + return job + } + + heartbeat(lease: JobLease, rawProgress: z.input, leaseMs = 30000) { + const progress = progressSchema.parse(rawProgress) + z.number().int().min(1000).max(120000).parse(leaseMs) + return this.db.transaction((tx) => { + const job = this.owned(lease) + const previous = progressSchema.parse(job.progress) + if (progress.totalNights !== previous.totalNights || progress.completedNights < previous.completedNights || progress.completedNights > progress.totalNights) throw new ReferenceInputError('INVALID_PROGRESS', 'Progress must be monotonic within the frozen night count') + const now = this.now() + tx.update(jobs).set({ progress, heartbeatAtMs: now, leaseExpiresAtMs: now + leaseMs, updatedAtMs: now }).where(eq(jobs.id, job.id)).run() + return { cancelRequested: job.state === 'cancel_requested' } + }, { behavior: 'immediate' }) + } + + cancel(id: string) { + return this.db.transaction((tx) => { + const job = this.get(id) + if (['succeeded', 'failed', 'cancelled'].includes(job.state)) return job + const now = this.now() + return tx.update(jobs).set({ state: job.state === 'queued' || job.state === 'interrupted' ? 'cancelled' : 'cancel_requested', + cancellationRequestedAtMs: job.cancellationRequestedAtMs ?? now, updatedAtMs: now }).where(eq(jobs.id, id)).returning().get() + }, { behavior: 'immediate' }) + } + + acknowledgeCancellation(lease: JobLease) { + return this.db.transaction((tx) => { + const job = this.owned(lease) + if (job.state !== 'cancel_requested') throw new ReferenceInputError('INVALID_STATE', 'Cancellation has not been requested') + return tx.update(jobs).set({ state: 'cancelled', leaseOwner: null, leaseExpiresAtMs: null, updatedAtMs: this.now() }).where(eq(jobs.id, job.id)).returning().get() + }, { behavior: 'immediate' }) + } + + interrupt(lease: JobLease, reason: 'Worker host stopped' | 'Resource admission withdrawn' = 'Worker host stopped') { + return this.db.transaction((tx) => { + const job = this.owned(lease) + return tx.update(jobs).set({ state: job.state === 'cancel_requested' ? 'cancelled' : job.attempt >= 3 ? 'failed' : 'interrupted', + error: reason, leaseOwner: null, leaseExpiresAtMs: null, updatedAtMs: this.now() }).where(eq(jobs.id, job.id)).returning().get() + }, { behavior: 'immediate' }) + } + + fail(lease: JobLease, message: string) { + return this.db.transaction((tx) => { + const job = this.owned(lease) + return tx.update(jobs).set({ state: job.state === 'cancel_requested' ? 'cancelled' : 'failed', + error: message.slice(0, 1000), leaseOwner: null, leaseExpiresAtMs: null, updatedAtMs: this.now() }).where(eq(jobs.id, job.id)).returning().get() + }, { behavior: 'immediate' }) + } + + complete(lease: JobLease, report: unknown) { + // Concrete evaluator schemas validate reports before calling this sink. + const serialized = JSON.stringify(report) + if (!serialized || Buffer.byteLength(serialized) > MAX_REPORT_BYTES) throw new ReferenceInputError('RESOURCE_LIMIT', 'Backtest report exceeds 16 MiB') + const contentHash = dataHash(report) + const artifacts = this.storage() + const publish = () => { + const owned = this.owned(lease) + if (owned.state !== 'running') throw new ReferenceInputError('INVALID_STATE', 'Cancelled work cannot publish a result') + let stored: unknown = report + const inlineBytes = this.db.select({ bytes: sql`coalesce(sum(case when json_extract(${results.report}, '$.kind') = 'biometric-report-artifact-v1' then 0 else length(cast(${results.report} as blob)) end), 0)` }).from(results).get()?.bytes ?? 0 + if (Buffer.byteLength(serialized) > 2 * 1024 * 1024) { + if (!artifacts) throw new ReferenceInputError('RESOURCE_LIMIT', 'Large reports require a persistent artifact directory') + stored = artifacts.write(lease.id, lease.token, serialized, inlineBytes) + } + else if (inlineBytes + (artifacts?.usage() ?? 0) + Buffer.byteLength(serialized) > REPORT_BUDGET_BYTES) { + throw new ReferenceInputError('RESOURCE_LIMIT', 'Reports and staged artifacts reached the 64 MiB storage budget') + } + return this.db.transaction((tx) => { + const job = this.owned(lease) + if (job.state !== 'running') throw new ReferenceInputError('INVALID_STATE', 'Cancelled work cannot publish a result') + const retained = tx.select({ bytes: sql`coalesce(sum(case when json_extract(${results.report}, '$.kind') = 'biometric-report-artifact-v1' then json_extract(${results.report}, '$.byteLength') else length(cast(${results.report} as blob)) end), 0)` }).from(results).get()?.bytes ?? 0 + if (retained + Buffer.byteLength(serialized) > REPORT_BUDGET_BYTES) throw new ReferenceInputError('RESOURCE_LIMIT', 'Backtest summaries have reached their 64 MiB storage budget') + const now = this.now() + tx.insert(results).values({ jobId: job.id, fencingToken: lease.token, report: stored, contentHash, completedAtMs: now }).run() + const progress = progressSchema.parse(job.progress) + tx.update(jobs).set({ state: 'succeeded', progress: { ...progress, phase: 'complete', completedNights: progress.totalNights }, + leaseOwner: null, leaseExpiresAtMs: null, updatedAtMs: now }).where(eq(jobs.id, job.id)).run() + return { jobId: job.id, contentHash } + }, { behavior: 'immediate' }) + } + return artifacts ? artifacts.withPublication(publish) : publish() + } +} diff --git a/src/biometrics/backtest/location.ts b/src/biometrics/backtest/location.ts new file mode 100644 index 000000000..ed8fdd254 --- /dev/null +++ b/src/biometrics/backtest/location.ts @@ -0,0 +1,30 @@ +import { existsSync } from 'node:fs' +import { resolve } from 'node:path' +import { ReferenceInputError } from '../references/time' + +/** Server-owned fixed package locations; no caller-supplied executable paths. + * These bundles are emitted by scripts/build/biometrics-worker.mjs AFTER Next + * tracing. Tracing their runtime probes would recursively include the build tree. */ +export function packagedWorkerPath() { + for (const relative of ['biometrics-worker/worker.cjs', '.next/standalone/biometrics-worker/worker.cjs']) { + const path = resolve(/* turbopackIgnore: true */ relative) + if (existsSync(/* turbopackIgnore: true */ path)) return path + } + throw new ReferenceInputError('WORKER_UNAVAILABLE', 'Build the packaged biometrics worker before submitting a replay') +} + +export function packagedCoordinatorPath() { + for (const relative of ['biometrics-worker/coordinator.cjs', '.next/standalone/biometrics-worker/coordinator.cjs']) { + const path = resolve(/* turbopackIgnore: true */ relative) + if (existsSync(/* turbopackIgnore: true */ path)) return path + } + throw new ReferenceInputError('WORKER_UNAVAILABLE', 'Build the packaged biometrics coordinator before accessing retained evaluation data') +} + +export function packagedShadowHostPath() { + for (const relative of ['biometrics-worker/shadow-host.cjs', '.next/standalone/biometrics-worker/shadow-host.cjs']) { + const path = resolve(/* turbopackIgnore: true */ relative) + if (existsSync(/* turbopackIgnore: true */ path)) return path + } + throw new ReferenceInputError('WORKER_UNAVAILABLE', 'Build the packaged shadow host before scheduling prospective runs') +} diff --git a/src/biometrics/backtest/night-summary-core.ts b/src/biometrics/backtest/night-summary-core.ts new file mode 100644 index 000000000..36adfa5ee --- /dev/null +++ b/src/biometrics/backtest/night-summary-core.ts @@ -0,0 +1,99 @@ +import type { NightBootstrapPolicy } from './uncertainty-policy' +import type { createNightUncertainty } from './uncertainty-core' +import type { compareStageOutputs } from './stage-comparison' + +type WindowComparison = ReturnType & { startMs: number, endMs: number } +interface NightComparison { + baseline: string + candidate: string + baselineHash: string + candidateHash: string + windows: WindowComparison[] +} +interface Night { cohortId: string, comparisons: NightComparison[] } +const stages = ['wake', 'light', 'deep', 'rem'] as const +const difference = (a: number | null, b: number | null) => a === null || b === null ? null : a - b +const mean = (values: number[]) => values.length ? values.reduce((sum, value) => sum + value, 0) / values.length : null + +function commonScore(windows: WindowComparison[], side: 'baseline' | 'candidate') { + const confusionMs = { wake: { wake: 0, light: 0, deep: 0, rem: 0 }, light: { wake: 0, light: 0, deep: 0, rem: 0 }, + deep: { wake: 0, light: 0, deep: 0, rem: 0 }, rem: { wake: 0, light: 0, deep: 0, rem: 0 } } + for (const window of windows) { + const matrix = side === 'baseline' ? window.commonSupport.baselineConfusionMs : window.commonSupport.candidateConfusionMs + for (const reference of stages) for (const predicted of stages) confusionMs[reference][predicted] += matrix[reference][predicted] + } + const perStage = stages.map((stage) => { + const referenceMs = stages.reduce((sum, predicted) => sum + confusionMs[stage][predicted], 0) + const predictedMs = stages.reduce((sum, reference) => sum + confusionMs[reference][stage], 0) + const correctMs = confusionMs[stage][stage] + return { stage, referenceMs, predictedMs, precision: predictedMs ? correctMs / predictedMs : null, + recall: referenceMs ? correctMs / referenceMs : null, f1: referenceMs + predictedMs ? 2 * correctMs / (referenceMs + predictedMs) : null } + }) + const pairedDurationMs = perStage.reduce((sum, row) => sum + row.referenceMs, 0) + const correctMs = stages.reduce((sum, stage) => sum + confusionMs[stage][stage], 0) + return { confusionMs, perStage, pairedDurationMs, agreement: pairedDurationMs ? correctMs / pairedDurationMs : null, + macroF1: mean(perStage.flatMap(row => row.f1 === null ? [] : [row.f1])) } +} + +function summarizeNight(cohortId: string, comparison: NightComparison) { + const baseline = commonScore(comparison.windows, 'baseline'), candidate = commonScore(comparison.windows, 'candidate') + const durationMs = comparison.windows.reduce((sum, window) => sum + window.endMs - window.startMs, 0) + const ownSupport = (side: 'baseline' | 'candidate') => { + const totals = comparison.windows.reduce((sum, window) => { + const score = window.ownSupport[side] + return { outputMs: sum.outputMs + score.outputDurationMs, pairedMs: sum.pairedMs + score.pairedDurationMs, + referenceMs: sum.referenceMs + score.eligibleReferenceMs, correctMs: sum.correctMs + (score.agreement ?? 0) * score.pairedDurationMs } + }, { outputMs: 0, pairedMs: 0, referenceMs: 0, correctMs: 0 }) + return { ...totals, agreement: totals.pairedMs ? totals.correctMs / totals.pairedMs : null, + outputCoverage: durationMs ? totals.outputMs / durationMs : null, + referenceCoverage: totals.referenceMs ? totals.pairedMs / totals.referenceMs : null } + } + return { cohortId, evaluationDurationMs: durationMs, commonOutputDurationMs: comparison.windows.reduce((sum, window) => sum + window.commonSupport.baseline.outputDurationMs, 0), commonSupport: { baseline, candidate }, + ownSupport: { baseline: ownSupport('baseline'), candidate: ownSupport('candidate') }, + commonAgreementDifference: difference(candidate.agreement, baseline.agreement), + commonMacroF1Difference: difference(candidate.macroF1, baseline.macroF1), + coverageReduced: comparison.windows.some(window => window.coverageReduced) } +} + +/** Aggregate disconnected windows within each whole-night cohort first. Equal + * night means and pooled duration scores are both explicit; neither silently + * substitutes epochs for independent nights. No promotion status is inferred. */ +export function createNightSummary(nightUncertainty: ReturnType) { + return function summarizeStageNights(nights: Night[], uncertaintyPolicy?: NightBootstrapPolicy) { + if (nights.length > 60 || new Set(nights.map(night => night.cohortId)).size !== nights.length) throw new Error('Invalid whole-night summary input') + const groups = new Map, nights: ReturnType[], windows: WindowComparison[] }>() + for (const night of nights) { + if (night.comparisons.length > 3) throw new Error('Too many comparisons within night') + const seen = new Set() + for (const comparison of night.comparisons) { + const { windows, ...identity } = comparison + const ordered = [...windows].sort((a, b) => a.startMs - b.startMs) + if (ordered.length > 60 || ordered.some((window, index) => !Number.isSafeInteger(window.startMs) || !Number.isSafeInteger(window.endMs) + || window.endMs <= window.startMs || window.endMs - window.startMs > 86400000 || (index > 0 && ordered[index - 1].endMs > window.startMs))) { + throw new Error('Nightly comparison windows must be bounded and nonoverlapping') + } + const key = JSON.stringify(identity) + if (seen.has(key)) throw new Error('Duplicate comparison within night') + seen.add(key) + const group = groups.get(key) ?? { identity, nights: [], windows: [] } + group.nights.push(summarizeNight(night.cohortId, comparison)) + group.windows.push(...windows) + groups.set(key, group) + } + } + return [...groups.values()].map((group) => { + const deltas = group.nights.flatMap(night => night.commonAgreementDifference === null ? [] : [night.commonAgreementDifference]) + return { ...group.identity, nights: group.nights, + totalNights: group.nights.length, scoredNights: deltas.length, unscoredNights: group.nights.length - deltas.length, + improvedAgreementNights: deltas.filter(delta => delta > 0).length, + regressedAgreementNights: deltas.filter(delta => delta < 0).length, + unchangedAgreementNights: deltas.filter(delta => delta === 0).length, + coverageReducedNights: group.nights.filter(night => night.coverageReduced).length, + equalNightMeanAgreementDifference: mean(deltas), + uncertainty: nightUncertainty(group.nights.map(night => ({ cohortId: night.cohortId, difference: night.commonAgreementDifference })), uncertaintyPolicy), + observedNightRange: deltas.length ? { minimum: Math.min(...deltas), maximum: Math.max(...deltas) } : null, + pooledCommonSupport: { baseline: commonScore(group.windows, 'baseline'), candidate: commonScore(group.windows, 'candidate') }, + interpretation: 'descriptive_night_aggregation_not_confidence_interval' as const } + }) + } +} diff --git a/src/biometrics/backtest/night-summary.ts b/src/biometrics/backtest/night-summary.ts new file mode 100644 index 000000000..108d2f579 --- /dev/null +++ b/src/biometrics/backtest/night-summary.ts @@ -0,0 +1,3 @@ +import { nightUncertainty } from './uncertainty' +import { createNightSummary } from './night-summary-core' +export const summarizeStageNights = createNightSummary(nightUncertainty) diff --git a/src/biometrics/backtest/numeric-comparison.ts b/src/biometrics/backtest/numeric-comparison.ts new file mode 100644 index 000000000..87e33ede7 --- /dev/null +++ b/src/biometrics/backtest/numeric-comparison.ts @@ -0,0 +1,59 @@ +import { alignReferences, type AlignmentPolicy, type Observation } from '../references/alignment' +import type { NormalizedMeasurement, ReferenceMetric } from '../references/contracts' + +type Label = 'default' | 'current' | 'candidate' +interface Output { label: Label, configHash: string, observations: Observation[] } +const summary = (score: ReturnType) => ({ status: score.status, metrics: score.metrics, + pairedCount: score.pairedCount, referenceCoverage: score.referenceCoverage, validObservationCount: score.validObservationCount, + unmatchedReferences: score.unmatchedReferences, unmatchedObservations: score.unmatchedObservations }) +const difference = (candidate: number | null, baseline: number | null) => candidate === null || baseline === null ? null : candidate - baseline + +/** Equal timestamps, not nearest output-to-output matching. All configurations + * contribute to the intersection before reference alignment, so dropping a hard + * sample cannot change an interval's aggregation support in only one result. + * Duplicate timestamps are ambiguous and excluded from common support. Known + * durations are deliberately not inferred from point timestamps. + */ +export function compareNumericConfigurations(metric: ReferenceMetric, windowMs: number | null, outputs: Output[], references: NormalizedMeasurement[], policy: AlignmentPolicy) { + if (outputs.length < 2 || outputs.length > 3 || new Set(outputs.map(output => output.label)).size !== outputs.length) throw new Error('Invalid numeric comparison configurations') + const own = outputs.map(output => alignReferences(metric, windowMs, output.observations, references, policy)) + const indexes = outputs.map((output) => { + const values = new Map(), seen = new Set(), ambiguous = new Set() + for (const row of output.observations) { + if (seen.has(row.timestampMs)) { + ambiguous.add(row.timestampMs) + values.delete(row.timestampMs) + } + seen.add(row.timestampMs) + if (!ambiguous.has(row.timestampMs) && row.value !== null) values.set(row.timestampMs, row.value) + } + return { values, ambiguous } + }) + const timestamps = [...indexes[0].values.keys()].filter(timestamp => indexes.every(index => index.values.has(timestamp))).sort((a, b) => a - b) + const common = indexes.map(index => alignReferences(metric, windowMs, + timestamps.map(timestampMs => ({ timestampMs, value: index.values.get(timestampMs) ?? null })), references, policy)) + return outputs.flatMap((candidate, c) => outputs.flatMap((baseline, b) => { + if (!((candidate.label === 'candidate' && baseline.label !== 'candidate') || (candidate.label === 'current' && baseline.label === 'default'))) return [] + const lost = [...indexes[b].values.keys()].filter(timestamp => !indexes[c].values.has(timestamp)).length + const added = [...indexes[c].values.keys()].filter(timestamp => !indexes[b].values.has(timestamp)).length + return [{ version: 'paired-numeric-support-v1' as const, metric, + baseline: baseline.label, candidate: candidate.label, baselineHash: baseline.configHash, candidateHash: candidate.configHash, + commonSupportScope: 'all_configurations' as const, supportDefinition: 'identical_unambiguous_observation_timestamps' as const, + commonObservationCount: timestamps.length, + ownSupport: { baseline: summary(own[b]), candidate: summary(own[c]) }, + commonSupport: { baseline: summary(common[b]), candidate: summary(common[c]) }, + differences: { + commonMae: difference(common[c].metrics?.mae ?? null, common[b].metrics?.mae ?? null), + commonMedianAbsoluteError: difference(common[c].metrics?.medianAbsoluteError ?? null, common[b].metrics?.medianAbsoluteError ?? null), + commonP95AbsoluteError: difference(common[c].metrics?.p95AbsoluteError ?? null, common[b].metrics?.p95AbsoluteError ?? null), + commonSignedBias: difference(common[c].metrics?.signedBias ?? null, common[b].metrics?.signedBias ?? null), + ownMae: difference(own[c].metrics?.mae ?? null, own[b].metrics?.mae ?? null), + referenceCoverage: difference(own[c].referenceCoverage, own[b].referenceCoverage), + }, + lostObservationTimestamps: lost, addedObservationTimestamps: added, + ambiguousObservationTimestamps: { baseline: indexes[b].ambiguous.size, candidate: indexes[c].ambiguous.size }, + coverageReduced: lost > 0 || own[c].pairedCount < own[b].pairedCount, + interpretation: 'descriptive_numeric_agreement_not_validated_improvement' as const, + }] + })) +} diff --git a/src/biometrics/backtest/numeric-night-summary.ts b/src/biometrics/backtest/numeric-night-summary.ts new file mode 100644 index 000000000..09eee3787 --- /dev/null +++ b/src/biometrics/backtest/numeric-night-summary.ts @@ -0,0 +1,83 @@ +import type { compareNumericConfigurations } from './numeric-comparison' +import type { createNightUncertainty } from './uncertainty-core' +import type { NightBootstrapPolicy } from './uncertainty-policy' + +type Comparison = ReturnType[number] +interface Window { startMs: number, endMs: number, metrics: Comparison[] } +interface Night { cohortId: string, numericComparisons?: Window[] } +const mean = (values: number[]) => values.length ? values.reduce((sum, value) => sum + value, 0) / values.length : null +function pooled(rows: Comparison[], side: 'baseline' | 'candidate') { + let pairedCount = 0, absoluteError = 0, signedError = 0 + for (const row of rows) { + const score = row.commonSupport[side] + if (score.metrics && score.pairedCount) { + pairedCount += score.pairedCount + absoluteError += score.metrics.mae * score.pairedCount + signedError += score.metrics.signedBias * score.pairedCount + } + } + // Quantiles cannot be pooled from window quantiles; deliberately omit them. + return { pairedCount, mae: pairedCount ? absoluteError / pairedCount : null, signedBias: pairedCount ? signedError / pairedCount : null } +} + +/** Weight references within each whole night, then weight nights equally. */ +export function createNumericNightSummary(uncertainty: ReturnType) { + return function summarize(nights: Night[], policy?: NightBootstrapPolicy) { + if (nights.length > 60 || new Set(nights.map(night => night.cohortId)).size !== nights.length) throw new Error('Invalid numeric night cohorts') + const groups = new Map + nights: Array<{ + cohortId: string + commonSupport: { baseline: ReturnType, candidate: ReturnType } + commonMaeDifference: number | null + coverageReduced: boolean + lostObservationTimestamps: number + addedObservationTimestamps: number + }> + rows: Comparison[] + }>() + for (const night of nights) { + const windows = [...(night.numericComparisons ?? [])].sort((a, b) => a.startMs - b.startMs) + if (windows.length > 60 || windows.some((window, i) => !Number.isSafeInteger(window.startMs) || !Number.isSafeInteger(window.endMs) + || window.endMs <= window.startMs || window.endMs - window.startMs > 86400000 || (i > 0 && windows[i - 1].endMs > window.startMs))) throw new Error('Invalid numeric night windows') + const local = new Map() + for (const window of windows) { + if (window.metrics.length > 9) throw new Error('Too many numeric comparisons') + const seen = new Set() + for (const row of window.metrics) { + const identity = { metric: row.metric, baseline: row.baseline, candidate: row.candidate, baselineHash: row.baselineHash, candidateHash: row.candidateHash } + const key = JSON.stringify(identity) + if (seen.has(key)) throw new Error('Duplicate numeric comparison') + if (row.commonSupport.baseline.pairedCount !== row.commonSupport.candidate.pairedCount) throw new Error('Unequal common reference support') + seen.add(key) + const rows = local.get(key) ?? [] + rows.push(row) + local.set(key, rows) + if (!groups.has(key)) groups.set(key, { identity, nights: [], rows: [] }) + } + } + for (const [key, rows] of local) { + const group = groups.get(key) + if (!group) throw new Error('Missing numeric comparison group') + const baseline = pooled(rows, 'baseline'), candidate = pooled(rows, 'candidate') + group.rows.push(...rows) + group.nights.push({ cohortId: night.cohortId, commonSupport: { baseline, candidate }, + commonMaeDifference: baseline.mae === null || candidate.mae === null ? null : candidate.mae - baseline.mae, + coverageReduced: rows.some(row => row.coverageReduced), + lostObservationTimestamps: rows.reduce((sum, row) => sum + row.lostObservationTimestamps, 0), + addedObservationTimestamps: rows.reduce((sum, row) => sum + row.addedObservationTimestamps, 0) }) + } + } + return [...groups.values()].map((group) => { + const values = group.nights.flatMap(night => night.commonMaeDifference === null ? [] : [night.commonMaeDifference]) + return { ...group.identity, version: 'numeric-night-summary-v1' as const, nights: group.nights, + totalNights: group.nights.length, scoredNights: values.length, unscoredNights: group.nights.length - values.length, + improvedErrorNights: values.filter(value => value < 0).length, regressedErrorNights: values.filter(value => value > 0).length, + unchangedErrorNights: values.filter(value => value === 0).length, coverageReducedNights: group.nights.filter(night => night.coverageReduced).length, + equalNightMeanMaeDifference: mean(values), + pooledCommonSupport: { baseline: pooled(group.rows, 'baseline'), candidate: pooled(group.rows, 'candidate') }, + uncertainty: uncertainty(group.nights.map(night => ({ cohortId: night.cohortId, difference: night.commonMaeDifference })), policy), + interpretation: 'descriptive_numeric_agreement_not_validated_improvement' as const } + }) + } +} diff --git a/src/biometrics/backtest/numeric-subgroups.ts b/src/biometrics/backtest/numeric-subgroups.ts new file mode 100644 index 000000000..a2637b770 --- /dev/null +++ b/src/biometrics/backtest/numeric-subgroups.ts @@ -0,0 +1,44 @@ +import type { NormalizedMeasurement } from '../references/contracts' + +/** Fixed reference-defined groups; never select membership using candidate + * output, error, quality, or coverage. Interval records are never split. */ +export const numericSubgroupNames = ['reference_points', 'reference_intervals', 'window_first_half', 'window_second_half'] as const +export type NumericSubgroupName = typeof numericSubgroupNames[number] +export function subgroupReferences(name: NumericSubgroupName, rows: NormalizedMeasurement[], window: { startMs: number, endMs: number }) { + const middle = window.startMs + (window.endMs - window.startMs) / 2 + return rows.filter(row => name === 'reference_points' + ? row.temporalKind === 'point' + : name === 'reference_intervals' + ? row.temporalKind === 'interval' + : name === 'window_first_half' + ? (row.temporalKind === 'point' ? row.startMs < middle : row.endMs <= middle) + : row.startMs >= middle) +} + +export function subgroupBoundaryExclusions(name: NumericSubgroupName, rows: NormalizedMeasurement[], window: { startMs: number, endMs: number }) { + if (name !== 'window_first_half' && name !== 'window_second_half') return 0 + const middle = window.startMs + (window.endMs - window.startMs) / 2 + return rows.filter(row => row.temporalKind === 'interval' && row.startMs < middle && row.endMs > middle).length +} + +interface Identity { + baseline: string + candidate: string + metric: string + baselineHash: string + candidateHash: string +} +interface SubgroupSummary extends Identity { + name: string + nights: Array<{ commonMaeDifference: number | null, commonSupport: { candidate: { pairedCount: number } } }> +} +export function subgroupObjectiveEvidence(summaries: SubgroupSummary[], identity: Identity, minimumPairs: number) { + return summaries.filter(row => row.baseline === identity.baseline && row.candidate === identity.candidate && row.metric === identity.metric + && row.baselineHash === identity.baselineHash && row.candidateHash === identity.candidateHash).map((row) => { + const supported = row.nights.filter(night => night.commonMaeDifference !== null && night.commonSupport.candidate.pairedCount >= minimumPairs) + return { name: row.name, pairedNights: supported.length, + maeDifference: supported.length > 0 && supported.length === row.nights.length + ? supported.reduce((sum, night) => sum + (night.commonMaeDifference ?? 0), 0) / supported.length + : null } + }) +} diff --git a/src/biometrics/backtest/numeric-trace-page.ts b/src/biometrics/backtest/numeric-trace-page.ts new file mode 100644 index 000000000..7530a7a2c --- /dev/null +++ b/src/biometrics/backtest/numeric-trace-page.ts @@ -0,0 +1,57 @@ +import { z } from 'zod' +import { ReferenceInputError } from '../references/time' +import { traceRequestSchema } from './trace-contracts' +import { waveformReadingSchema } from './waveform-output' + +const reading = waveformReadingSchema.pick({ timestampMs: true, heartRate: true, breathingRate: true, hrvIndex: true, + hrvMetric: true, qualityScore: true, flags: true, sourceSide: true, configHash: true, side: true, limitations: true }).strip() +export const numericTracePageSchema = z.object({ jobId: z.string().uuid(), contentHash: z.string(), + configuration: z.enum(['default', 'current', 'candidate']), side: z.enum(['left', 'right']), + provenance: z.literal('retained_waveform_recomputation'), timestampBasis: z.literal('original_trace_before_reference_clock_offset'), clockOffsetMs: z.number(), + totalItems: z.number().int(), offset: z.number().int(), limit: z.number().int(), downsampled: z.literal(false), + items: z.array(reading.extend({ cohortId: z.string(), revisionId: z.string().uuid().nullable() })).max(500), + nextCursor: z.object({ offset: z.number().int(), contentHash: z.string() }).nullable(), +}) +const reportSchema = z.object({ version: z.literal('waveform-backtest-report-v1'), side: z.enum(['left', 'right']), + protocol: z.object({ mode: z.literal('signal_replay'), clockOffsetMs: z.number().int() }), + waveform: z.object({ outputs: z.array(z.object({ cohortId: z.string(), configHash: z.string(), readings: z.array(reading).max(20000) })).max(180) }), + nights: z.array(z.object({ cohortId: z.string(), evaluations: z.array(z.object({ label: z.enum(['default', 'current', 'candidate']), + configHash: z.string(), revisionId: z.string().uuid().nullable(), + windows: z.array(z.object({ startMs: z.number().int(), endMs: z.number().int() })).max(60), + })).max(3) })).max(60), +}).refine(report => report.waveform.outputs.reduce((sum, output) => sum + output.readings.length, 0) <= 20000 + && new Set(report.nights.map(night => night.cohortId)).size === report.nights.length) + +/** Exact retained numeric rows for the dataset side only. Cross-side routing + * provenance is retained, but opposite-side observations are never returned. */ +export function pageNumericTraces(report: unknown, raw: z.input) { + const input = traceRequestSchema.parse(raw) + const parsed = reportSchema.safeParse(report) + if (!parsed.success) throw new ReferenceInputError('RESULT_CORRUPT', 'Numeric trace format or side provenance is unavailable') + const data = parsed.data + const nights = data.nights.filter(night => input.cohortId === undefined || night.cohortId === input.cohortId) + const items: z.infer['items'] = [] + let totalItems = 0, selections = 0 + for (const night of nights) { + const evaluations = night.evaluations.filter(row => row.label === input.configuration) + if (evaluations.length > 1) throw new ReferenceInputError('RESULT_CORRUPT', 'Duplicate numeric configuration') + for (const evaluation of evaluations) { + selections++ + const outputs = data.waveform.outputs.filter(output => output.cohortId === night.cohortId && output.configHash === evaluation.configHash) + if (outputs.length !== 1) throw new ReferenceInputError('RESULT_CORRUPT', 'Numeric trace output identity is ambiguous or missing') + for (const row of outputs[0].readings) { + if (row.configHash !== evaluation.configHash && row.side === data.side) throw new ReferenceInputError('RESULT_CORRUPT', 'Numeric reading configuration mismatch') + if (row.side !== data.side || !evaluation.windows.some(window => row.timestampMs >= window.startMs && row.timestampMs < window.endMs)) continue + if (totalItems >= input.offset && items.length < input.limit) items.push({ ...row, cohortId: night.cohortId, revisionId: evaluation.revisionId }) + totalItems++ + } + } + } + if (!selections) throw new ReferenceInputError('NOT_FOUND', 'Requested cohort or configuration was not evaluated') + if (input.offset > totalItems) throw new ReferenceInputError('INVALID_CURSOR', 'Trace cursor exceeds the selected report') + const next = input.offset + items.length + return numericTracePageSchema.parse({ jobId: input.id, contentHash: input.expectedContentHash, configuration: input.configuration, side: data.side, + provenance: 'retained_waveform_recomputation', timestampBasis: 'original_trace_before_reference_clock_offset', clockOffsetMs: data.protocol.clockOffsetMs, + totalItems, offset: input.offset, limit: input.limit, items, downsampled: false, + nextCursor: next < totalItems ? { offset: next, contentHash: input.expectedContentHash } : null }) +} diff --git a/src/biometrics/backtest/objective-finalization.ts b/src/biometrics/backtest/objective-finalization.ts new file mode 100644 index 000000000..0ec983035 --- /dev/null +++ b/src/biometrics/backtest/objective-finalization.ts @@ -0,0 +1,62 @@ +import { assessStageObjective, stageObjectivePolicySchema, stageObjectiveEvidenceSchema, type StageObjectivePolicy } from './stage-objective' +import { z } from 'zod' +import { jobOperationsSchema } from './job-operations' +import { assessNumericObjective, objectiveEvidenceSchema, objectivePolicySchema, type ObjectivePolicy, type ObjectiveEvidence } from './objective-policy' + +/** Convert supervisor measurements only when this frozen policy explicitly + * permits sampled RSS. Legacy peak policies do not silently change meaning. */ +export function objectiveOperations(raw: unknown, policy: Pick): ObjectiveEvidence['operational'] { + if (!policy.operational.observations) return null + const parsed = jobOperationsSchema.safeParse(raw) + if (!parsed.success) return null + const { wallMs, ingestion, memory } = parsed.data + if (!memory || !memory.allSamplesAvailable || memory.samples < 2 || memory.samples !== memory.attempts || memory.peakObservedRssBytes === null + || !ingestion.allChecksMeasured || ingestion.samples < 2 || ingestion.samples !== ingestion.checks + || ingestion.maximumObservedAgeMs === null || ingestion.qualificationId === null) return null + return { wallMs, peakRssBytes: memory.peakObservedRssBytes, maximumIngestionAgeMs: ingestion.maximumObservedAgeMs, + observations: { version: 'sampled-process-group-v1', qualificationId: ingestion.qualificationId, + maximumMemoryGapMs: memory.maximumObservationGapMs, maximumIngestionGapMs: ingestion.maximumObservationGapMs } } +} +const assessment = z.object({ version: z.literal('numeric-objective-assessment-v1'), policy: objectivePolicySchema, + evidence: objectiveEvidenceSchema, baseline: z.enum(['default', 'current']), candidate: z.literal('candidate'), + baselineHash: z.string(), candidateHash: z.string() }) + +/** Called after final admission, before immutable report publication. Operational + * evidence from the scoring child is replaced, never trusted as host telemetry. */ +export function finalizeObjectiveReport(raw: unknown, policy: ObjectivePolicy | readonly ObjectivePolicy[] | undefined, operations: unknown, designatedPolicy: ObjectivePolicy | readonly ObjectivePolicy[] | null) { + const report = z.record(z.string(), z.unknown()).parse(raw) + const policies = policy === undefined ? [] : Array.isArray(policy) ? policy : [policy] + if (!policies.length) return report + const parsedPolicies = policies.map(item => objectivePolicySchema.parse(item)) + if (new Set(parsedPolicies.map(item => item.metric)).size !== parsedPolicies.length) throw new Error('Duplicate numeric objective metric') + const designations = designatedPolicy === null ? [] : Array.isArray(designatedPolicy) ? designatedPolicy : [designatedPolicy] + const parsedDesignations = designations.map(item => objectivePolicySchema.parse(item)) + if (new Set(parsedDesignations.map(item => item.metric)).size !== parsedDesignations.length) throw new Error('Duplicate numeric policy designation') + const assessments = z.array(assessment).length(parsedPolicies.length * 2).parse(report.objectiveAssessments) + const identities = new Set() + const finalized = assessments.map((row) => { + const selected = parsedPolicies.find(item => item.metric === row.policy.metric) + const identity = `${row.policy.metric}:${row.baseline}` + if (!selected || identities.has(identity) || JSON.stringify(row.policy) !== JSON.stringify(selected)) throw new Error('Objective report identity mismatch') + identities.add(identity) + return { baseline: row.baseline, candidate: row.candidate, baselineHash: row.baselineHash, candidateHash: row.candidateHash, + ...assessNumericObjective(selected, { ...row.evidence, operational: objectiveOperations(operations, selected) }, + parsedDesignations.find(item => item.metric === selected.metric) ?? null) } + }) + return { ...report, objectiveAssessments: finalized } +} + +const stageAssessment = z.object({ version: z.literal('stage-objective-assessment-v1'), policy: stageObjectivePolicySchema, + evidence: stageObjectiveEvidenceSchema, baseline: z.enum(['default', 'current']), candidate: z.literal('candidate'), baselineHash: z.string(), candidateHash: z.string() }) +export function finalizeStageObjectiveReport(raw: unknown, policy: StageObjectivePolicy | undefined, operations: unknown, designation: StageObjectivePolicy | null) { + const report = z.record(z.string(), z.unknown()).parse(raw) + if (!policy) return report + const assessments = z.array(stageAssessment).length(2).parse(report.stageObjectiveAssessments) + if (new Set(assessments.map(row => row.baseline)).size !== 2 + || assessments.some(row => JSON.stringify(row.policy) !== JSON.stringify(stageObjectivePolicySchema.parse(policy)))) throw new Error('Stage objective report identity mismatch') + const operational = objectiveOperations(operations, policy) + return { ...report, stageObjectiveAssessments: assessments.map(row => ({ baseline: row.baseline, candidate: row.candidate, + baselineHash: row.baselineHash, candidateHash: row.candidateHash, + ...assessStageObjective(policy, { ...row.evidence, operational }, designation), + })) } +} diff --git a/src/biometrics/backtest/objective-policy.ts b/src/biometrics/backtest/objective-policy.ts new file mode 100644 index 000000000..db7b26746 --- /dev/null +++ b/src/biometrics/backtest/objective-policy.ts @@ -0,0 +1,106 @@ +import { z } from 'zod' + +const nonnegative = z.number().finite().nonnegative() +export const operationalPolicySchema = z.object({ maximumWallMs: z.number().int().positive(), maximumPeakRssBytes: z.number().int().positive(), + maximumIngestionAgeMs: z.number().int().positive(), + observations: z.object({ version: z.literal('sampled-process-group-v1'), + maximumMemoryGapMs: z.number().int().positive(), maximumIngestionGapMs: z.number().int().positive(), + }).strict().optional(), +}).strict() + +/** Frozen with the experiment. Supplying thresholds never designates this as a + * release promotion policy; that authority belongs to the qualified build. */ +export const objectivePolicySchema = z.object({ + version: z.literal('numeric-objective-v1'), + qualificationId: z.string().min(1).max(200), + metric: z.enum(['heart_rate', 'breathing_rate', 'window_ibi_variability_v1']), + minimumNights: z.number().int().min(3).max(60), + minimumPairedReferencesPerNight: z.number().int().min(1).max(100000), + maximumLostObservationFraction: z.number().min(0).max(1), + minimumMaeImprovement: z.number().finite().positive(), + maximumNightMaeRegression: nonnegative, + maximumRegressedNightFraction: z.number().min(0).max(1), + maximumSubgroupMaeRegression: nonnegative, + requiredSubgroups: z.array(z.string().min(1).max(100)).max(20), + operational: operationalPolicySchema, +}).strict().refine(policy => new Set(policy.requiredSubgroups).size === policy.requiredSubgroups.length, 'Duplicate objective subgroups') +export type ObjectivePolicy = z.infer + +export const operationalEvidenceSchema = z.object({ wallMs: nonnegative, peakRssBytes: nonnegative, maximumIngestionAgeMs: nonnegative, + observations: z.object({ version: z.literal('sampled-process-group-v1'), qualificationId: z.string(), + maximumMemoryGapMs: nonnegative, maximumIngestionGapMs: nonnegative, + }).strict().optional(), +}).strict() + +const nightSchema = z.object({ cohortId: z.string(), pairedReferences: z.number().int().nonnegative(), + maeDifference: z.number().finite().nullable(), baselineObservationCount: z.number().int().nonnegative(), lostObservations: z.number().int().nonnegative() }).strict() +export const objectiveEvidenceSchema = z.object({ + metricCompatible: z.boolean(), split: z.enum(['tuning', 'validation', 'holdout']), holdoutUse: z.number().int().nonnegative(), + nights: z.array(nightSchema).max(60), + intervalUpper: z.number().finite().nullable(), + subgroups: z.array(z.object({ name: z.string(), pairedNights: z.number().int().nonnegative(), maeDifference: z.number().finite().nullable() }).strict()).max(20), + operational: operationalEvidenceSchema.nullable(), +}).strict() +export type ObjectiveEvidence = z.infer + +/** Pure assessment. Designation is supplied by a trusted build dependency, not + * experiment JSON. No clinical accuracy or activation is inferred. */ +export function assessNumericObjective(rawPolicy: ObjectivePolicy, rawEvidence: ObjectiveEvidence, designatedPolicy: ObjectivePolicy | null) { + const policy = objectivePolicySchema.parse(rawPolicy), evidence = objectiveEvidenceSchema.parse(rawEvidence) + if (new Set(evidence.nights.map(night => night.cohortId)).size !== evidence.nights.length + || new Set(evidence.subgroups.map(group => group.name)).size !== evidence.subgroups.length + || evidence.nights.some(night => night.lostObservations > night.baselineObservationCount)) throw new Error('Invalid objective evidence identity or coverage') + const designated = designatedPolicy !== null && JSON.stringify(objectivePolicySchema.parse(designatedPolicy)) === JSON.stringify(policy) + const missing: string[] = [], failures: string[] = [] + if (!designated) missing.push('policy_not_designated') + if (evidence.split !== 'holdout' || evidence.holdoutUse !== 1) missing.push('independent_holdout_required') + const scored = evidence.nights.filter(night => night.maeDifference !== null && night.pairedReferences >= policy.minimumPairedReferencesPerNight) + if (scored.length < policy.minimumNights) missing.push('insufficient_supported_nights') + if (scored.length !== evidence.nights.length) missing.push('unsupported_nights') + if (evidence.intervalUpper === null) missing.push('uncertainty_unavailable') + const operational = assessOperationalEvidence(policy.qualificationId, policy.operational, evidence.operational) + missing.push(...operational.missing) + failures.push(...operational.failures) + if (evidence.nights.some(night => night.baselineObservationCount === 0)) missing.push('coverage_unavailable') + if (evidence.nights.some(night => night.baselineObservationCount > 0 && night.lostObservations / night.baselineObservationCount > policy.maximumLostObservationFraction)) failures.push('coverage_loss') + const deltas = scored.map(night => night.maeDifference ?? 0) + if (deltas.some(delta => delta > policy.maximumNightMaeRegression)) failures.push('night_regression') + if (deltas.length && deltas.filter(delta => delta > 0).length / deltas.length > policy.maximumRegressedNightFraction) failures.push('regressed_night_fraction') + for (const name of policy.requiredSubgroups) { + const group = evidence.subgroups.find(row => row.name === name) + if (!group || group.maeDifference === null || group.pairedNights < policy.minimumNights) missing.push(`subgroup_support:${name}`) + else if (group.maeDifference > policy.maximumSubgroupMaeRegression) failures.push(`subgroup_regression:${name}`) + } + const estimate = deltas.length ? deltas.reduce((sum, delta) => sum + delta, 0) / deltas.length : null + const status = !evidence.metricCompatible + ? 'incompatible' as const + : missing.length + ? 'insufficient_evidence' as const + : failures.length + ? 'regressed' as const + : estimate !== null && estimate <= -policy.minimumMaeImprovement && evidence.intervalUpper !== null && evidence.intervalUpper <= -policy.minimumMaeImprovement + ? 'improved' as const + : 'no_material_change' as const + return { version: 'numeric-objective-assessment-v1' as const, policy, evidence, designated, status, missingEvidence: missing, failedThresholds: failures, + equalNightMeanMaeDifference: estimate, intervalUpper: evidence.intervalUpper, + interpretation: 'agreement_with_named_reference_not_clinical_validation' as const } +} + +export function assessOperationalEvidence(qualificationId: string, policy: z.infer, evidence: z.infer | null) { + const missing: string[] = [], failures: string[] = [] + if (!evidence) missing.push('operational_evidence_unavailable') + else { + const required = policy.observations, observed = evidence.observations + if (required && !observed) missing.push('operational_sampling_evidence_unavailable') + if (!required && observed) missing.push('sampled_memory_not_authorized_by_policy') + if (required && observed) { + if (observed.qualificationId !== qualificationId) missing.push('operational_qualification_mismatch') + if (observed.maximumMemoryGapMs > required.maximumMemoryGapMs) missing.push('memory_sampling_gap') + if (observed.maximumIngestionGapMs > required.maximumIngestionGapMs) missing.push('ingestion_sampling_gap') + } + if (evidence.wallMs > policy.maximumWallMs) failures.push('wall_time_limit') + if (evidence.peakRssBytes > policy.maximumPeakRssBytes) failures.push('memory_limit') + if (evidence.maximumIngestionAgeMs > policy.maximumIngestionAgeMs) failures.push('ingestion_limit') + } + return { missing, failures } +} diff --git a/src/biometrics/backtest/process-group-memory-contract.ts b/src/biometrics/backtest/process-group-memory-contract.ts new file mode 100644 index 000000000..bf4ead7a9 --- /dev/null +++ b/src/biometrics/backtest/process-group-memory-contract.ts @@ -0,0 +1,12 @@ +import { z } from 'zod' + +export const processGroupMemorySchema = z.object({ + version: z.literal('process-group-rss-observations-v1'), + scope: z.literal('sum_rss_of_replay_process_group'), + attempts: z.number().int().nonnegative(), samples: z.number().int().nonnegative(), + allSamplesAvailable: z.boolean(), peakObservedRssBytes: z.number().int().positive().nullable(), + maximumObservationGapMs: z.number().finite().nonnegative(), intervalMs: z.literal(1000), + exactKernelPeak: z.literal(false), +}).strict() +export type ProcessGroupMemory = z.infer + diff --git a/src/biometrics/backtest/process-group-memory.ts b/src/biometrics/backtest/process-group-memory.ts new file mode 100644 index 000000000..71de1b61e --- /dev/null +++ b/src/biometrics/backtest/process-group-memory.ts @@ -0,0 +1,82 @@ +import { readdir, readFile } from 'node:fs/promises' +import { join } from 'node:path' +import type { ProcessGroupMemory } from './process-group-memory-contract' + +/** Fixed procfs reads only; never inspect argv, environment, or other exports. + * Shared pages may be counted in more than one process's RSS. */ +export async function sampleProcessGroupRss(group: number, root = '/proc'): Promise { + if (!Number.isSafeInteger(group) || group < 1) return null + try { + const pids = (await readdir(/* turbopackIgnore: true */ root)).filter(name => /^[1-9]\d*$/.test(name)) + if (pids.length > 4096) return null + let total = 0, members = 0 + for (let i = 0; i < pids.length; i += 16) { + const values = await Promise.all(pids.slice(i, i + 16).map(async (pid) => { + try { + const stat = await readFile(join(root, pid, 'stat'), 'utf8') + if (stat.length > 16384) throw new Error('Invalid proc stat') + const end = stat.lastIndexOf(')') + if (end < 0) throw new Error('Invalid proc stat') + const fields = stat.slice(end + 1).trim().split(/\s+/) + if (!/^\d+$/.test(fields[2] ?? '')) throw new Error('Missing process group') + if (Number(fields[2]) !== group) return 0 + const status = await readFile(join(root, pid, 'status'), 'utf8') + if (status.length > 16384) throw new Error('Invalid proc status') + const rss = /^VmRSS:\s+(\d+)\s+kB\s*$/m.exec(status) + // Zombies have no address space and no VmRSS entry. + if (!rss && fields[0] === 'Z') return 0 + if (!rss) throw new Error('Missing process RSS') + members++ + const bytes = Number(rss[1]) * 1024 + if (!Number.isSafeInteger(bytes) || bytes < 0) throw new Error('Invalid process RSS') + return bytes + } + catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT' || (error as NodeJS.ErrnoException).code === 'ESRCH') return 0 + throw error + } + })) + total += values.reduce((sum, value) => sum + value, 0) + if (members > 64 || !Number.isSafeInteger(total)) return null + } + return members > 0 && total > 0 ? total : null + } + catch { return null } +} + +export function monitorProcessGroupMemory(group: number, options: { sample?: () => Promise, now?: () => number } = {}) { + const now = options.now ?? (() => performance.now()) + const read = options.sample ?? (() => process.platform === 'linux' ? sampleProcessGroupRss(group) : Promise.resolve(null)) + const started = now() + let last = started, gap = 0, attempts = 0, samples = 0, peak = 0, stopped = false + let active: Promise | null = null + const sample = () => { + if (stopped) return Promise.resolve() + active ??= Promise.resolve().then(read).catch(() => null).then((rss) => { + const at = now() + gap = Math.max(gap, at - last) + last = at + attempts++ + if (rss !== null && Number.isSafeInteger(rss) && rss > 0) { + samples++ + peak = Math.max(peak, rss) + } + }).finally(() => { active = null }) + return active + } + const timer = setInterval(() => { + void sample() + }, 1000) + timer.unref() + void sample() + return { sample, + snapshot: (): ProcessGroupMemory => ({ version: 'process-group-rss-observations-v1', scope: 'sum_rss_of_replay_process_group', + attempts, samples, allSamplesAvailable: samples > 0 && samples === attempts, peakObservedRssBytes: samples ? peak : null, + maximumObservationGapMs: Math.max(gap, now() - last), intervalMs: 1000, exactKernelPeak: false }), + async stop() { + stopped = true + clearInterval(timer) + await active + }, + } +} diff --git a/src/biometrics/backtest/raw-catalog.ts b/src/biometrics/backtest/raw-catalog.ts new file mode 100644 index 000000000..fb8051755 --- /dev/null +++ b/src/biometrics/backtest/raw-catalog.ts @@ -0,0 +1,106 @@ +import { and, asc, eq, gt, gte, isNull, lt, ne } from 'drizzle-orm' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import { z } from 'zod' +import * as schema from '@/src/db/biometrics-schema' +import { configHash } from '../algorithms/canonical' +import { ReferenceInputError } from '../references/time' + +const count = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) +const hash = z.string().regex(/^[a-f0-9]{64}$/) +export { rawArtifactDescriptorSchema } from './waveform-contracts' +import { rawArtifactDescriptorSchema } from './waveform-contracts' +const channel = z.object({ samples: count, lost_markers: count, truncated_tail_bytes: count }).strict() +export const rawInventorySchema = z.object({ + version: z.literal('retained-raw-inventory-v1'), artifact_sha256: hash, + decoded_records: count, piezo_records: count, pump_records: count, + usable_dual_frames: count, empty_channel_frames: count, oversize_channel_frames: count, + invalid_timestamps: count, timestamp_reversals: count, duplicate_timestamps: count, + first_event_ms: count.nullable(), last_event_ms: count.nullable(), + channels: z.object({ left1: channel, right1: channel }).strict(), + ordered_waveform_available: z.boolean(), limitations: z.array(z.string().max(500)).max(10), +}).strict().superRefine((value, context) => { + const invalid = (message: string) => context.addIssue({ code: 'custom', message }) + if ((value.first_event_ms === null) !== (value.last_event_ms === null) + || (value.first_event_ms !== null && value.last_event_ms !== null && value.first_event_ms > value.last_event_ms)) invalid('Invalid event bounds') + if (value.piezo_records + value.pump_records > value.decoded_records + || value.usable_dual_frames + value.empty_channel_frames + value.oversize_channel_frames !== value.piezo_records) invalid('Inconsistent record counts') + for (const totals of Object.values(value.channels)) { + if (totals.lost_markers > totals.samples) invalid('Lost markers exceed sample count') + } + const ordered = value.usable_dual_frames > 0 && value.invalid_timestamps === 0 + && value.timestamp_reversals === 0 && value.oversize_channel_frames === 0 + if (value.ordered_waveform_available !== ordered || (ordered && value.first_event_ms === null)) invalid('Inconsistent waveform availability') +}) +export const rawInspectionSchema = z.object({ + source_name: z.string().max(255).regex(/^[A-Za-z0-9_.-]+\.RAW\.gz$/), + retained: z.literal(false), artifact: rawArtifactDescriptorSchema, inventory: rawInventorySchema, +}).strict().refine(value => value.artifact.sha256 === value.inventory.artifact_sha256, 'Inventory must identify the inspected bytes') + +export const rawSourceDescriptorSchema = z.object({ + id: hash, inspection: rawInspectionSchema, inspectedAtMs: count, unavailableAtMs: count.nullable(), + availability: z.enum(['requires_reverification', 'unavailable']), +}).strict() +export const rawSourcePageResultSchema = z.object({ items: z.array(rawSourceDescriptorSchema).max(50), nextCursor: hash.nullable() }).strict() + +export const rawSourcePageSchema = z.object({ startMs: count, endMs: count, afterId: hash.optional() }).strict() + .refine(value => value.endMs > value.startMs, 'Expected a nonempty time window') + +/** Internal scan evidence only. This catalog neither retains bytes nor grants replay capability. */ +export class RawSourceCatalog { + constructor(private readonly db: BetterSQLite3Database, private readonly now = Date.now) {} + + record(raw: unknown) { + const input = rawInspectionSchema.parse(raw) + const id = configHash({ sourceName: input.source_name, artifactHash: input.artifact.sha256, inventoryVersion: input.inventory.version }) + const inspectedAtMs = count.parse(this.now()) + return this.db.transaction((tx) => { + const existing = tx.select().from(schema.biometricRawSources).where(eq(schema.biometricRawSources.id, id)).get() + if (existing && (configHash(existing.descriptor) !== configHash(input.artifact) + || configHash(existing.inventory) !== configHash(input.inventory))) { + throw new ReferenceInputError('SOURCE_CONFLICT', 'Identical source bytes and inventory version produced different inspection evidence') + } + // Replacement invalidates only previous identities of this filename. It + // never changes their frozen descriptor or any dataset's retained bytes. + tx.update(schema.biometricRawSources).set({ unavailableAtMs: inspectedAtMs }) + .where(and(eq(schema.biometricRawSources.sourceName, input.source_name), ne(schema.biometricRawSources.id, id), isNull(schema.biometricRawSources.unavailableAtMs))).run() + tx.insert(schema.biometricRawSources).values({ id, sourceName: input.source_name, + artifactHash: input.artifact.sha256, descriptor: input.artifact, inventoryVersion: input.inventory.version, + inventory: input.inventory, firstEventMs: input.inventory.first_event_ms, lastEventMs: input.inventory.last_event_ms, + inspectedAtMs, unavailableAtMs: null, + }).onConflictDoUpdate({ target: schema.biometricRawSources.id, set: { inspectedAtMs, unavailableAtMs: null } }).run() + return id + }, { behavior: 'immediate' }) + } + + markUnavailable(id: string) { + // Mark the failed inspected identity, not a filename that may now point at + // a newer successful scan. Existing tombstone timestamps remain stable. + this.db.update(schema.biometricRawSources).set({ unavailableAtMs: count.parse(this.now()) }) + .where(and(eq(schema.biometricRawSources.id, hash.parse(id)), isNull(schema.biometricRawSources.unavailableAtMs))).run() + } + + read(id: string) { + const row = this.db.select().from(schema.biometricRawSources).where(eq(schema.biometricRawSources.id, hash.parse(id))).get() + if (!row) throw new ReferenceInputError('NOT_FOUND', 'Inspected RAW source not found') + const inspection = rawInspectionSchema.parse({ source_name: row.sourceName, retained: false, artifact: row.descriptor, inventory: row.inventory }) + if (row.artifactHash !== inspection.artifact.sha256 || row.inventoryVersion !== inspection.inventory.version + || row.firstEventMs !== inspection.inventory.first_event_ms || row.lastEventMs !== inspection.inventory.last_event_ms + || row.id !== configHash({ sourceName: row.sourceName, artifactHash: row.artifactHash, inventoryVersion: row.inventoryVersion })) { + throw new ReferenceInputError('SOURCE_CORRUPT', 'Inspected RAW source integrity check failed') + } + return { id: row.id, inspection, inspectedAtMs: row.inspectedAtMs, unavailableAtMs: row.unavailableAtMs, + availability: row.unavailableAtMs === null ? 'requires_reverification' as const : 'unavailable' as const } + } + + page(raw: { startMs: number, endMs: number, afterId?: string }) { + const input = rawSourcePageSchema.parse(raw) + // Event bounds are inclusive; requested windows are half-open. Inclusion + // only indicates potential overlap, never continuous or complete coverage. + const rows = this.db.select({ id: schema.biometricRawSources.id }).from(schema.biometricRawSources) + .where(and(isNull(schema.biometricRawSources.unavailableAtMs), lt(schema.biometricRawSources.firstEventMs, input.endMs), + gte(schema.biometricRawSources.lastEventMs, input.startMs), input.afterId ? gt(schema.biometricRawSources.id, input.afterId) : undefined)) + .orderBy(asc(schema.biometricRawSources.id)).limit(51).all() + const items = rows.slice(0, 50).map(row => this.read(row.id)) + return { items, nextCursor: rows.length > 50 ? items[49].id : null } + } +} diff --git a/src/biometrics/backtest/raw-collection-child.ts b/src/biometrics/backtest/raw-collection-child.ts new file mode 100644 index 000000000..4941a17fa --- /dev/null +++ b/src/biometrics/backtest/raw-collection-child.ts @@ -0,0 +1,61 @@ +import { spawn } from 'node:child_process' +import { tmpdir } from 'node:os' +import { z } from 'zod' + +const resultSchema = z.object({ version: z.literal('raw-collection-result-v1'), released: z.number().int().min(0).max(10000), orphans: z.number().int().min(0).max(10000) }).strict() +interface RawCollectionLocation { pythonPath: string, modulesPath: string, databasePath: string, retainedDirectory: string } + +/** Trusted deployment paths only; no RPC arguments select files or alter retention grace. */ +export function runRawCollectionChild(location: RawCollectionLocation, options: { + signal?: AbortSignal + timeoutMs?: number +} = {}): Promise> { + const timeoutMs = options.timeoutMs ?? 120000 + if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 10 || timeoutMs > 120000) throw new Error('Invalid collection time budget') + if (options.signal?.aborted) return Promise.reject(new Error('RAW collection cancelled')) + return new Promise((resolve, reject) => { + const child = spawn(location.pythonPath, ['-B', '-m', 'common.biometrics.collection_worker', '--database-path', location.databasePath, '--retained-directory', location.retainedDirectory], { + cwd: tmpdir(), env: { NODE_ENV: 'production', PYTHONPATH: location.modulesPath, LANG: 'C.UTF-8', TZ: 'UTC', + OPENBLAS_NUM_THREADS: '1', OMP_NUM_THREADS: '1', MKL_NUM_THREADS: '1', NUMEXPR_NUM_THREADS: '1' }, + stdio: ['pipe', 'pipe', 'pipe'], + }) + const chunks: Buffer[] = [] + let outputBytes = 0, diagnosticBytes = 0 + let failure: Error | undefined, killTimer: ReturnType | undefined + const stop = (error: Error) => { + failure ??= error + child.kill('SIGTERM') + killTimer ??= setTimeout(() => child.kill('SIGKILL'), 500) + } + const timeout = setTimeout(() => stop(new Error('RAW collection exceeded its time budget')), timeoutMs) + const abort = () => stop(new Error('RAW collection cancelled')) + options.signal?.addEventListener('abort', abort, { once: true }) + if (options.signal?.aborted) abort() + child.on('error', () => stop(new Error('RAW collection worker could not start'))) + child.stdin.on('error', () => stop(new Error('RAW collection input pipe closed'))) + child.stderr.on('data', (chunk: Buffer) => { + diagnosticBytes += chunk.length + if (diagnosticBytes > 16384) stop(new Error('RAW collection diagnostics exceeded their bound')) + }) + child.stdout.on('data', (chunk: Buffer) => { + if (failure) return + outputBytes += chunk.length + if (outputBytes > 1024) return stop(new Error('RAW collection output exceeded its bound')) + chunks.push(chunk) + }) + child.on('close', (code) => { + clearTimeout(timeout) + if (killTimer) clearTimeout(killTimer) + options.signal?.removeEventListener('abort', abort) + if (failure) return reject(failure) + try { + if (code !== 0) throw new Error('Worker failed') + const text = new TextDecoder('utf-8', { fatal: true }).decode(Buffer.concat(chunks)) + const result = resultSchema.parse(JSON.parse(text)) + resolve(result) + } + catch { reject(new Error('RAW collection worker returned an invalid or incomplete result')) } + }) + child.stdin.end(JSON.stringify({ version: 'raw-collection-work-v1' })) + }) +} diff --git a/src/biometrics/backtest/raw-inspection-child.ts b/src/biometrics/backtest/raw-inspection-child.ts new file mode 100644 index 000000000..be58ba386 --- /dev/null +++ b/src/biometrics/backtest/raw-inspection-child.ts @@ -0,0 +1,81 @@ +import { spawn } from 'node:child_process' +import { tmpdir } from 'node:os' +import { z } from 'zod' +import { rawInspectionSchema } from './raw-catalog' + +export const rawSourceNameSchema = z.string().max(255).regex(/^[A-Za-z0-9_.-]+\.RAW\.gz$/) +export const rawScanInputSchema = z.object({ afterName: rawSourceNameSchema.nullable() }).strict() +const inspectionResultSchema = z.discriminatedUnion('status', [ + z.object({ status: z.literal('inspected'), nextCursor: rawSourceNameSchema, inspection: rawInspectionSchema }).strict(), + z.object({ status: z.literal('source_unavailable'), nextCursor: rawSourceNameSchema }).strict(), + z.object({ status: z.literal('archive_unavailable'), nextCursor: z.null() }).strict(), + z.object({ status: z.literal('complete'), nextCursor: z.null() }).strict(), +]) +export interface RawInspectionLocation { + pythonPath: string + modulesPath: string + archiveDirectory: string +} +export const podRawInspectionLocation: RawInspectionLocation = { + pythonPath: '/opt/sleepypod/modules/piezo-processor/.venv/bin/python', + modulesPath: '/opt/sleepypod/modules', + archiveDirectory: '/persistent/biometrics-archive', +} + +/** Trusted deployment paths only; callers supply a filename cursor, never code or paths. */ +export function runRawInspectionChild(location: RawInspectionLocation, raw: z.input, options: { + signal?: AbortSignal + timeoutMs?: number +} = {}): Promise> { + const input = rawScanInputSchema.parse(raw) + const timeoutMs = options.timeoutMs ?? 120000 + if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 10 || timeoutMs > 120000) throw new Error('Invalid inspection time budget') + if (options.signal?.aborted) return Promise.reject(new Error('RAW inspection cancelled')) + return new Promise((resolve, reject) => { + const child = spawn(location.pythonPath, ['-B', '-m', 'common.biometrics.inspection_worker', '--archive-directory', location.archiveDirectory], { + cwd: tmpdir(), env: { NODE_ENV: 'production', PYTHONPATH: location.modulesPath, LANG: 'C.UTF-8', TZ: 'UTC', + OPENBLAS_NUM_THREADS: '1', OMP_NUM_THREADS: '1', MKL_NUM_THREADS: '1', NUMEXPR_NUM_THREADS: '1' }, + stdio: ['pipe', 'pipe', 'pipe'], + }) + const chunks: Buffer[] = [] + let outputBytes = 0, diagnosticBytes = 0 + let failure: Error | undefined, killTimer: ReturnType | undefined + const stop = (error: Error) => { + failure ??= error + child.kill('SIGTERM') + killTimer ??= setTimeout(() => child.kill('SIGKILL'), 500) + } + const timeout = setTimeout(() => stop(new Error('RAW inspection exceeded its time budget')), timeoutMs) + const abort = () => stop(new Error('RAW inspection cancelled')) + options.signal?.addEventListener('abort', abort, { once: true }) + if (options.signal?.aborted) abort() + child.on('error', () => stop(new Error('RAW inspection worker could not start'))) + child.stdin.on('error', () => stop(new Error('RAW inspection input pipe closed'))) + child.stderr.on('data', (chunk: Buffer) => { + diagnosticBytes += chunk.length + if (diagnosticBytes > 16384) stop(new Error('RAW inspection diagnostics exceeded their bound')) + }) + child.stdout.on('data', (chunk: Buffer) => { + if (failure) return + outputBytes += chunk.length + if (outputBytes > 65536) return stop(new Error('RAW inspection output exceeded its bound')) + chunks.push(chunk) + }) + child.on('close', (code) => { + clearTimeout(timeout) + if (killTimer) clearTimeout(killTimer) + options.signal?.removeEventListener('abort', abort) + if (failure) return reject(failure) + try { + if (code !== 0) throw new Error('Worker failed') + const text = new TextDecoder('utf-8', { fatal: true }).decode(Buffer.concat(chunks)) + const result = inspectionResultSchema.parse(JSON.parse(text)) + if ((result.nextCursor !== null && input.afterName !== null && result.nextCursor <= input.afterName) + || (result.status === 'inspected' && result.nextCursor !== result.inspection.source_name)) throw new Error('Invalid cursor') + resolve(result) + } + catch { reject(new Error('RAW inspection worker returned an invalid or incomplete result')) } + }) + child.stdin.end(JSON.stringify({ version: 'raw-inspection-work-v1', after_name: input.afterName })) + }) +} diff --git a/src/biometrics/backtest/raw-publication-child.ts b/src/biometrics/backtest/raw-publication-child.ts new file mode 100644 index 000000000..0f6b6f649 --- /dev/null +++ b/src/biometrics/backtest/raw-publication-child.ts @@ -0,0 +1,70 @@ +import { spawn } from 'node:child_process' +import { tmpdir } from 'node:os' +import { z } from 'zod' +import type { RawInspectionLocation } from './raw-inspection-child' +import type { prepareSelectedWaveformDataset } from './waveform-selection' + +const publicationResultSchema = z.object({ type: z.literal('result'), id: z.string().uuid(), + contentHash: z.string().regex(/^[a-f0-9]{64}$/), available: z.boolean() }).strict() +export interface RawPublicationLocation extends RawInspectionLocation { + databasePath: string + retainedDirectory: string +} + +/** Only coordinator-prepared rows enter this fixed shipped module. No RPC paths or code. */ +export function runRawPublicationChild(location: RawPublicationLocation, work: ReturnType, options: { + signal?: AbortSignal + timeoutMs?: number +} = {}): Promise> { + const serialized = JSON.stringify({ version: 'waveform-publication-work-v1', ...work }) + if (Buffer.byteLength(serialized) > 10 * 1024 * 1024) return Promise.reject(new Error('Waveform publication input exceeds 10 MiB')) + const timeoutMs = options.timeoutMs ?? 120000 + if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 10 || timeoutMs > 120000) throw new Error('Invalid publication time budget') + if (options.signal?.aborted) return Promise.reject(new Error('Waveform publication cancelled')) + return new Promise((resolve, reject) => { + const child = spawn(location.pythonPath, ['-B', '-m', 'common.biometrics.publication_worker', '--database', location.databasePath, '--archive-directory', location.archiveDirectory, '--retained-directory', location.retainedDirectory], { + cwd: tmpdir(), env: { NODE_ENV: 'production', PYTHONPATH: location.modulesPath, LANG: 'C.UTF-8', TZ: 'UTC', + OPENBLAS_NUM_THREADS: '1', OMP_NUM_THREADS: '1', MKL_NUM_THREADS: '1', NUMEXPR_NUM_THREADS: '1' }, + stdio: ['pipe', 'pipe', 'pipe'], + }) + const chunks: Buffer[] = [] + let outputBytes = 0, diagnosticBytes = 0 + let failure: Error | undefined, killTimer: ReturnType | undefined + const stop = (error: Error) => { + failure ??= error + child.kill('SIGTERM') + killTimer ??= setTimeout(() => child.kill('SIGKILL'), 500) + } + const timeout = setTimeout(() => stop(new Error('Waveform publication exceeded its time budget')), timeoutMs) + const abort = () => stop(new Error('Waveform publication cancelled')) + options.signal?.addEventListener('abort', abort, { once: true }) + if (options.signal?.aborted) abort() + child.on('error', () => stop(new Error('Waveform publication worker could not start'))) + child.stdin.on('error', () => stop(new Error('Waveform publication input pipe closed'))) + child.stderr.on('data', (chunk: Buffer) => { + diagnosticBytes += chunk.length + if (diagnosticBytes > 16384) stop(new Error('Waveform publication diagnostics exceeded their bound')) + }) + child.stdout.on('data', (chunk: Buffer) => { + if (failure) return + outputBytes += chunk.length + if (outputBytes > 65536) return stop(new Error('Waveform publication output exceeded its bound')) + chunks.push(chunk) + }) + child.on('close', (code) => { + clearTimeout(timeout) + if (killTimer) clearTimeout(killTimer) + options.signal?.removeEventListener('abort', abort) + if (failure) return reject(failure) + try { + if (code !== 0) throw new Error('Worker failed') + const text = new TextDecoder('utf-8', { fatal: true }).decode(Buffer.concat(chunks)) + const result = publicationResultSchema.parse(JSON.parse(text)) + if (result.contentHash !== work.prepared.contentHash) throw new Error('Mismatched published dataset') + resolve(result) + } + catch { reject(new Error('Waveform publication worker returned an invalid or incomplete result')) } + }) + child.stdin.end(serialized) + }) +} diff --git a/src/biometrics/backtest/reference-trace-page.ts b/src/biometrics/backtest/reference-trace-page.ts new file mode 100644 index 000000000..9987a848f --- /dev/null +++ b/src/biometrics/backtest/reference-trace-page.ts @@ -0,0 +1,66 @@ +import { z } from 'zod' +import { traceRequestSchema } from './trace-contracts' +import { normalizedMeasurementSchema } from '../references/contracts' +import { referenceCompatibility } from '../references/alignment' +import { ReferenceInputError } from '../references/time' +import type { DatasetSnapshot } from './contracts' + +export const referenceTraceRequestSchema = traceRequestSchema.omit({ configuration: true }).extend({ + cohortId: z.string().min(1).max(200), metric: z.enum(['heart_rate', 'breathing_rate', 'window_ibi_variability_v1', 'sleep_stages']), +}) +const projected = normalizedMeasurementSchema.omit({ original: true }).strip().extend({ + importId: z.string(), importRevision: z.number().int(), importContentHash: z.string(), source: z.string(), + compatibility: z.string().nullable(), +}) +export const referenceTracePageSchema = z.object({ jobId: z.string().uuid(), contentHash: z.string(), datasetHash: z.string(), + cohortId: z.string(), metric: referenceTraceRequestSchema.shape.metric, side: z.enum(['left', 'right']), + provenance: z.literal('frozen_dataset_reference'), timestampBasis: z.literal('reference_source_time'), + clockOffsetMs: z.number().int(), pointToleranceMs: z.number().int(), + totalItems: z.number().int(), offset: z.number().int(), limit: z.number().int(), downsampled: z.literal(false), + items: z.array(projected).max(500), nextCursor: z.object({ offset: z.number().int(), contentHash: z.string() }).nullable(), +}) +const reportSchema = z.object({ version: z.enum(['processed-backtest-report-v1', 'waveform-backtest-report-v1']), + datasetId: z.string().uuid(), datasetHash: z.string(), + protocol: z.object({ clockOffsetMs: z.number().int(), pointToleranceMs: z.number().int() }), + nights: z.array(z.object({ cohortId: z.string(), evaluations: z.array(z.object({ + windows: z.array(z.object({ startMs: z.number().int(), endMs: z.number().int() })).max(60), + })).max(3) })).max(60), +}) +export function referenceTraceIdentity(report: unknown) { + const parsed = reportSchema.safeParse(report) + if (!parsed.success) throw new ReferenceInputError('RESULT_CORRUPT', 'Reference trace report identity is unavailable') + return parsed.data +} + +/** Read only frozen imports, never the current reference library. Projection + * excludes submitted payloads, annotations and subject identity. No score is + * recomputed, and display records are not claims of independently paired data. */ +export function pageReferenceTraces(report: unknown, dataset: { contentHash: string, snapshot: DatasetSnapshot }, raw: z.input) { + const input = referenceTraceRequestSchema.parse(raw), identity = referenceTraceIdentity(report) + if (identity.datasetHash !== dataset.contentHash) throw new ReferenceInputError('REVISION_CONFLICT', 'Reference dataset hash changed') + const nights = identity.nights.filter(night => night.cohortId === input.cohortId) + if (nights.length !== 1) throw new ReferenceInputError('NOT_FOUND', 'Requested reference cohort was not evaluated') + const windows = nights[0].evaluations[0]?.windows + if (!windows) throw new ReferenceInputError('RESULT_CORRUPT', 'Reference scoring windows unavailable') + const items: z.infer['items'] = [] + let totalItems = 0 + for (const reference of dataset.snapshot.references) { + for (const row of reference.measurements) { + const sameFamily = row.metric === input.metric || (input.metric === 'window_ibi_variability_v1' && row.metric.startsWith('hrv_')) + if (!sameFamily || !windows.some(window => row.startMs >= window.startMs && (row.temporalKind === 'point' ? row.startMs < window.endMs : row.endMs <= window.endMs))) continue + if (totalItems >= input.offset && items.length < input.limit) { + items.push(projected.parse({ ...row, importId: reference.importId, importRevision: reference.revision, importContentHash: reference.contentHash, source: reference.source, + compatibility: input.metric === 'sleep_stages' + ? (row.scorable ? null : row.exclusions[0] ?? 'unscorable') + : referenceCompatibility(input.metric, input.metric === 'window_ibi_variability_v1' ? 300000 : null, row) })) + } + totalItems++ + } + } + if (input.offset > totalItems) throw new ReferenceInputError('INVALID_CURSOR', 'Reference cursor exceeds the selected records') + const next = input.offset + items.length + return referenceTracePageSchema.parse({ jobId: input.id, contentHash: input.expectedContentHash, datasetHash: identity.datasetHash, + cohortId: input.cohortId, metric: input.metric, side: dataset.snapshot.side, provenance: 'frozen_dataset_reference', timestampBasis: 'reference_source_time', + ...identity.protocol, totalItems, offset: input.offset, limit: input.limit, items, downsampled: false, + nextCursor: next < totalItems ? { offset: next, contentHash: input.expectedContentHash } : null }) +} diff --git a/src/biometrics/backtest/report-summary.ts b/src/biometrics/backtest/report-summary.ts new file mode 100644 index 000000000..55f78d1c0 --- /dev/null +++ b/src/biometrics/backtest/report-summary.ts @@ -0,0 +1,40 @@ +import { z } from 'zod' +import { ReferenceInputError } from '../references/time' + +const report = z.looseObject({ waveform: z.looseObject({ outputs: z.array(z.looseObject({ readings: z.unknown() })).max(180) }).optional(), version: z.enum(['processed-backtest-report-v1', 'waveform-backtest-report-v1']), + nights: z.array(z.looseObject({ evaluations: z.array(z.looseObject({ + windows: z.array(z.looseObject({ blocks: z.unknown() })).max(60), + })).max(3) })).max(60), +}) + +/** Project only after verifying the complete retained report. The content hash + * still identifies that complete artifact; exact traces remain separately paged. */ +export function summarizeBacktestResult(result: T | null) { + if (result === null) return null + let summary: unknown = null + let waveformReport = false + if (result.available) { + const parsed = report.safeParse(result.report) + if (!parsed.success) throw new ReferenceInputError('RESULT_CORRUPT', 'Report summary format is unavailable') + waveformReport = parsed.data.version === 'waveform-backtest-report-v1' + summary = { ...parsed.data, ...(parsed.data.waveform + ? { waveform: { ...parsed.data.waveform, outputs: parsed.data.waveform.outputs.map((output) => { + const metadata = { ...output } + delete metadata.readings + return metadata + }) } } + : {}), nights: parsed.data.nights.map(night => ({ ...night, + evaluations: night.evaluations.map(evaluation => ({ ...evaluation, windows: evaluation.windows.map((window) => { + const scores = { ...window } + delete scores.blocks + return scores + }) })), + })) } + // Bound the response without truncating scores or silently dropping nights. + if (new TextEncoder().encode(JSON.stringify(summary)).byteLength > 2 * 1024 * 1024) { + throw new ReferenceInputError('RESOURCE_LIMIT', 'Report summary exceeds its 2 MiB transport limit') + } + } + return { ...result, report: summary, projection: waveformReport ? 'scores_without_replay_trace_rows_v1' as const : 'scores_without_stage_trace_rows_v1' as const, + contentHashScope: 'complete_retained_report' as const } +} diff --git a/src/biometrics/backtest/runner.ts b/src/biometrics/backtest/runner.ts new file mode 100644 index 000000000..de48c293e --- /dev/null +++ b/src/biometrics/backtest/runner.ts @@ -0,0 +1,157 @@ +import { assessEvaluation } from '../profiles/evaluation-assessment' +import { dataHash } from '../algorithms/canonical' +import { verifyReplayContext } from '../profiles/replay-context' +import { datasetManifestSchema } from './contracts' +import { installedPromotionPolicies, unqualifiedPolicies } from '../profiles/qualification-registry' +import { numericObjectivePolicies } from './contracts' +import { finalizeObjectiveReport, finalizeStageObjectiveReport } from './objective-finalization' +import { createJobOperations, type AdmissionReceipt } from './job-operations' +import type { ReplayBudget } from './execution-budget' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import { z } from 'zod' +import type * as schema from '@/src/db/biometrics-schema' +import { DatasetStore } from './datasets' +import { BacktestJobStore } from './jobs' +import { backtestWorkSchema } from './evaluator' +import { executeWaveformInputs, verifyWaveformJobBuild } from './waveform-execution' +import type { RawInspectionLocation } from './raw-inspection-child' +import { workerBuildHash, runBacktestChild } from './child' + +/** One leased attempt. Call only after the host's resource/ingest gate admits work. */ +export async function executeNextBacktest(db: BetterSQLite3Database, options: { + owner: string + workerPath: string + rawLocation?: RawInspectionLocation + replayBudget?: ReplayBudget + signal?: AbortSignal + timeoutMs?: number + admit: () => Promise + admissionReceipt?: () => AdmissionReceipt | null +}) { + const operations = createJobOperations() + const admit = async () => { + const allowed = await options.admit() + operations.record(options.admissionReceipt?.() ?? null) + return allowed + } + if (options.signal?.aborted || !await admit() || options.signal?.aborted) return { state: 'deferred' as const } + const jobs = new BacktestJobStore(db) + const claimed = jobs.claim(options.owner) + if (!claimed) return { state: 'idle' as const } + const lease = { id: claimed.id, owner: options.owner, token: claimed.fencingToken } + const controller = new AbortController() + const abort = () => controller.abort() + options.signal?.addEventListener('abort', abort, { once: true }) + if (options.signal?.aborted) controller.abort() + let progress = z.object({ completedNights: z.number(), totalNights: z.number() }).parse(claimed.progress) + let heartbeatFailure: unknown + let resourceFailure = false + let resourceCheck: Promise | null = null + const heartbeat = () => { + try { + const result = jobs.heartbeat(lease, { phase: 'evaluating', ...progress }) + if (result.cancelRequested) controller.abort() + } + catch (error) { + heartbeatFailure = error + controller.abort() + } + } + const checkResources = async () => { + try { + if (await admit()) return + } + catch { /* Missing health evidence must stop background work. */ } + resourceFailure = true + controller.abort() + } + const timer = setInterval(() => { + heartbeat() + if (!resourceCheck && !controller.signal.aborted) { + resourceCheck = checkResources().finally(() => { + resourceCheck = null + }) + } + }, 5000) + try { + const input = z.object({ datasetId: z.string() }).parse(claimed.input) + const retained = new DatasetStore(db).read(input.datasetId) + const work = backtestWorkSchema.parse({ job: claimed.input, dataset: { manifest: retained.manifest, snapshot: retained.snapshot } }) + if (work.job.protocol.mode === 'signal_replay') { + work.waveform = await executeWaveformInputs(db, work, { ...options, signal: controller.signal }) + } + else if (await workerBuildHash(options.workerPath) !== work.job.evaluatorBuildHash) throw new Error('Evaluator bundle changed since enqueue') + heartbeat() + const report = await runBacktestChild(options.workerPath, work, { signal: controller.signal, timeoutMs: options.timeoutMs, + onProgress: (completedNights, totalNights) => { + if (totalNights !== progress.totalNights || completedNights < progress.completedNights || completedNights > totalNights) throw new Error('Invalid worker progress') + progress = { completedNights, totalNights } + heartbeat() + }, + }) + if (work.job.protocol.mode === 'signal_replay') await verifyWaveformJobBuild(work, { ...options, signal: controller.signal }) + let policies = await installedPromotionPolicies(options.workerPath, { evaluatorBuildHash: work.job.evaluatorBuildHash, + pythonBuildHash: work.job.waveform?.pythonBuildHash, protocolHash: work.job.protocolHash }) + // Quiesce the periodic check before the final fresh check and publication. + clearInterval(timer) + await resourceCheck + await checkResources() + if (heartbeatFailure) throw heartbeatFailure + const identity = z.object({ datasetHash: z.string(), protocolHash: z.string(), evaluatorBuildHash: z.string(), split: z.string(), nights: z.array(z.unknown()) }).parse(report) + if (identity.evaluatorBuildHash !== work.job.evaluatorBuildHash || identity.datasetHash !== work.job.datasetHash || identity.protocolHash !== work.job.protocolHash || identity.split !== work.job.split + || identity.nights.length !== progress.totalNights || progress.completedNights !== progress.totalNights) throw new Error('Worker report identity/progress mismatch') + if (controller.signal.aborted) throw new Error('Replay cancelled') + const jobOperations = operations.finish() + let qualificationWarning: string | null = null + if (policies.protocolHash && work.job.waveform) { + try { + verifyReplayContext(z.object({ waveform: z.unknown() }).parse(report).waveform, { + cohortIds: datasetManifestSchema.parse(retained.manifest).splitProtocol.assignments.filter(row => row.split === work.job.split).map(row => row.cohortId), + configHashes: work.job.configs.map(config => config.resolvedHash), side: retained.side, + pythonBuildHash: work.job.waveform.pythonBuildHash, otherSideConfigHash: work.job.waveform.otherSide.resolvedHash, + }, policies.waveformContext) + } + catch { + policies = unqualifiedPolicies() + qualificationWarning = 'Replay context is outside the installed qualification; this report is descriptive.' + } + } + const finalized = finalizeObjectiveReport(report, numericObjectivePolicies(work.job.protocol), jobOperations, policies.numeric) + const measured = { ...finalizeStageObjectiveReport(finalized, work.job.protocol.stageObjectivePolicy, jobOperations, policies.stages), jobOperations, qualificationWarning } + let status: ReturnType['status'] = 'insufficient_evidence' + const candidate = work.job.configs.find(config => config.label === 'candidate') + const current = work.job.configs.find(config => config.label === 'current') + if (candidate?.revisionId && current) { + try { + status = assessEvaluation(work.job, { report: measured, contentHash: dataHash(measured), available: true, + currentHoldoutUseCount: jobs.holdoutUseCount(work.job.datasetId) }, { + side: retained.side, revisionId: candidate.revisionId, resolvedHash: candidate.resolvedHash, currentResolvedHash: current.resolvedHash, + evaluatorBuildHash: work.job.evaluatorBuildHash, dataset: { id: retained.id, contentHash: retained.contentHash, side: retained.side, + available: retained.available, holdoutCohortIds: datasetManifestSchema.parse(retained.manifest).splitProtocol.assignments.filter(row => row.split === 'holdout').map(row => row.cohortId) }, + ...(work.job.waveform ? { waveform: { pythonBuildHash: work.job.waveform.pythonBuildHash, otherSideConfigHash: work.job.waveform.otherSide.resolvedHash } } : {}), + }, policies).status + } + catch { /* Exploratory, incomplete or incompatible evidence cannot claim improvement. */ } + } + if (candidate && current && candidate.resolvedHash === current.resolvedHash) status = 'no_material_change' + jobs.complete(lease, { ...measured, status, evidence: status === 'improved' ? 'qualified_reference_agreement' : work.job.evidence }) + return { state: 'succeeded' as const, jobId: claimed.id } + } + catch { + // Only the authoritative lease may update state. Expired work is recovered + // by the next claimant; it must not overwrite a replacement's result. + try { + const latest = jobs.get(claimed.id) + if (latest.state === 'cancel_requested') jobs.acknowledgeCancellation(lease) + else if (options.signal?.aborted || resourceFailure) jobs.interrupt(lease, resourceFailure ? 'Resource admission withdrawn' : 'Worker host stopped') + else jobs.fail(lease, 'Replay failed validation, resources, or execution') + } + catch { /* Lost lease: recovery owns the state transition. */ } + return { state: 'stopped' as const, jobId: claimed.id } + } + finally { + clearInterval(timer) + await resourceCheck + options.signal?.removeEventListener('abort', abort) + } +} diff --git a/src/biometrics/backtest/splits.ts b/src/biometrics/backtest/splits.ts new file mode 100644 index 000000000..986b035a0 --- /dev/null +++ b/src/biometrics/backtest/splits.ts @@ -0,0 +1,74 @@ +import { z } from 'zod' + +export const sessionWindowSchema = z.object({ + id: z.number().int().positive(), + startMs: z.number().int().min(1e12).max(4102444800000), + endMs: z.number().int().min(1e12).max(4102444800000), +}).strict().refine(row => row.endMs > row.startMs && row.endMs - row.startMs <= 36 * 3600000, + { message: 'Sessions must be positive and at most 36 hours long' }) +export type SessionWindow = z.infer +export type EvaluationSplit = 'tuning' | 'validation' | 'holdout' + +/** + * Local-noon night grouping. Sessions touching the same night, or overlapping + * sessions crossing noon, are one indivisible cohort. Snapshot the returned + * assignment; never recalculate an existing dataset after timezone changes. + */ +export function assignWholeNightSplits(raw: SessionWindow[], timezone: string) { + const sessions = z.array(sessionWindowSchema).min(1).max(60).parse(raw).sort((a, b) => a.startMs - b.startMs || a.id - b.id) + if (new Set(sessions.map(row => row.id)).size !== sessions.length) throw new Error('Duplicate session selection') + const formatter = new Intl.DateTimeFormat('en-CA', { timeZone: timezone, year: 'numeric', month: '2-digit', day: '2-digit', hour: '2-digit', hourCycle: 'h23' }) + const nightAt = (timestamp: number) => { + const parts = Object.fromEntries(formatter.formatToParts(timestamp).map(part => [part.type, part.value])) + const day = new Date(Date.UTC(Number(parts.year), Number(parts.month) - 1, Number(parts.day))) + if (Number(parts.hour) < 12) day.setUTCDate(day.getUTCDate() - 1) + return day.toISOString().slice(0, 10) + } + const labels = sessions.map((session) => { + const keys = new Set([nightAt(session.startMs), nightAt(session.endMs - 1)]) + // Less than any local night length, including DST transitions. Endpoints + // preserve short naps; intermediate samples cover sessions spanning noon. + for (let time = session.startMs + 3600000; time < session.endMs; time += 3600000) keys.add(nightAt(time)) + return [...keys].sort() + }) + const parent = sessions.map((_, i) => i) + const root = (index: number): number => { + while (parent[index] !== index) index = parent[index] + return index + } + const owner = new Map() + for (let index = 0; index < sessions.length; index++) { + for (const label of labels[index]) { + const previous = owner.get(label) + if (previous !== undefined) parent[root(index)] = root(previous) + else owner.set(label, index) + } + } + const grouped = new Map() + for (let index = 0; index < sessions.length; index++) { + const key = root(index) + if (!grouped.has(key)) grouped.set(key, []) + grouped.get(key)?.push(index) + } + const groups = [...grouped.values()].sort((a, b) => sessions[a[0]].startMs - sessions[b[0]].startMs) + // This partition establishes independence, not statistical sufficiency. + // With fewer than three cohorts, all work is descriptive tuning only. + const hasThreeSplits = groups.length >= 3 + const tuningCount = hasThreeSplits ? Math.max(1, Math.min(groups.length - 2, Math.floor(groups.length * 0.6))) : groups.length + const validationCount = hasThreeSplits ? Math.max(1, Math.floor(groups.length * 0.2)) : 0 + const assignments = groups.map((indices, index) => ({ + cohortId: labels[indices[0]][0], + nightLabels: [...new Set(indices.flatMap(i => labels[i]))].sort(), + sessionIds: indices.map(i => sessions[i].id), + startMs: Math.min(...indices.map(i => sessions[i].startMs)), + endMs: Math.max(...indices.map(i => sessions[i].endMs)), + split: (index < tuningCount ? 'tuning' : index < tuningCount + validationCount ? 'validation' : 'holdout') as EvaluationSplit, + })) + return { version: 'chronological-whole-night-v1' as const, timezone, + timezoneRuntime: { icu: typeof process !== 'undefined' ? process.versions?.icu ?? 'unknown' : 'unknown', tz: typeof process !== 'undefined' ? process.versions?.tz ?? 'unknown' : 'unknown' }, + grouping: 'local_noon_connected_sessions' as const, assignments, + evidence: hasThreeSplits ? 'partitioned_not_validated' as const : 'descriptive_only' as const, + limitations: ['Whole-night separation does not by itself establish sufficient reference support or improvement.', + ...(hasThreeSplits ? [] : ['Fewer than three independent night cohorts; no validation or holdout claim is available.'])], + } +} diff --git a/src/biometrics/backtest/stage-comparison.ts b/src/biometrics/backtest/stage-comparison.ts new file mode 100644 index 000000000..608facbbd --- /dev/null +++ b/src/biometrics/backtest/stage-comparison.ts @@ -0,0 +1,43 @@ +import { intersectOutputSupport, restrictToSupport, type SupportInterval } from './common-support' +import type { SleepStageBlock } from '@/src/lib/sleep-stages' +import type { NormalizedMeasurement } from '../references/contracts' +import { scoreStageDurations } from '../references/stage-scoring' + +const summarize = (score: ReturnType) => ({ + agreement: score.agreement, macroF1: score.macroF1, pairedDurationMs: score.pairedDurationMs, + eligibleReferenceMs: score.eligibleReferenceMs, outputDurationMs: score.outputDurationMs, + referenceCoverage: score.referenceCoverage, outputCoverage: score.outputCoverage, +}) +const difference = (candidate: number | null, baseline: number | null) => candidate === null || baseline === null ? null : candidate - baseline + +/** Compare accuracy on exactly the same intervals, while preserving each + * output's own support. Dropping hard intervals cannot manufacture an accuracy + * gain on common support. This is descriptive evidence, not a promotion gate. */ +export function compareStageOutputs(baseline: SleepStageBlock[], candidate: SleepStageBlock[], references: NormalizedMeasurement[], startMs: number, endMs: number, sharedSupport?: SupportInterval[]) { + // The shared scorer validates ordering/overlap and normalizes conflicts across + // reference imports identically for both outputs. + const baselineOwn = scoreStageDurations(baseline, references, startMs, endMs) + const candidateOwn = scoreStageDurations(candidate, references, startMs, endMs) + const pairSupport = intersectOutputSupport([baseline, candidate], startMs, endMs) + const support = sharedSupport === undefined ? pairSupport : restrictToSupport(pairSupport, sharedSupport) + const commonBaseline = restrictToSupport(baseline, support) + const commonCandidate = restrictToSupport(candidate, support) + const baselineCommon = scoreStageDurations(commonBaseline, references, startMs, endMs) + const candidateCommon = scoreStageDurations(commonCandidate, references, startMs, endMs) + return { version: 'paired-stage-support-v1' as const, + commonSupportScope: sharedSupport === undefined ? 'pair' as const : 'all_configurations' as const, + ownSupport: { baseline: summarize(baselineOwn), candidate: summarize(candidateOwn) }, + commonSupport: { baseline: summarize(baselineCommon), candidate: summarize(candidateCommon), + baselineConfusionMs: baselineCommon.confusionMs, candidateConfusionMs: candidateCommon.confusionMs }, + differences: { + commonAgreement: difference(candidateCommon.agreement, baselineCommon.agreement), + commonMacroF1: difference(candidateCommon.macroF1, baselineCommon.macroF1), + ownAgreement: difference(candidateOwn.agreement, baselineOwn.agreement), + referenceCoverage: difference(candidateOwn.referenceCoverage, baselineOwn.referenceCoverage), + outputCoverage: candidateOwn.outputCoverage - baselineOwn.outputCoverage, + pairedDurationMs: candidateOwn.pairedDurationMs - baselineOwn.pairedDurationMs, + }, + coverageReduced: candidateOwn.pairedDurationMs < baselineOwn.pairedDurationMs || candidateOwn.outputDurationMs < baselineOwn.outputDurationMs, + interpretation: 'descriptive_agreement_not_validated_improvement' as const, + } +} diff --git a/src/biometrics/backtest/stage-objective.ts b/src/biometrics/backtest/stage-objective.ts new file mode 100644 index 000000000..476c7c723 --- /dev/null +++ b/src/biometrics/backtest/stage-objective.ts @@ -0,0 +1,70 @@ +import { z } from 'zod' +import { operationalPolicySchema, operationalEvidenceSchema, assessOperationalEvidence } from './objective-policy' +const fraction = z.number().min(0).max(1) +const delta = z.number().min(-1).max(1) +const stage = z.enum(['wake', 'light', 'deep', 'rem']) +export const stageObjectivePolicySchema = z.object({ + version: z.literal('stage-objective-v1'), qualificationId: z.string().min(1).max(200), + minimumNights: z.number().int().min(3).max(60), + minimumPairedDurationMsPerNight: z.number().int().positive().max(86400000), + minimumOutputCoverage: fraction, maximumLostOutputFraction: fraction, + minimumAgreementImprovement: z.number().positive().max(1), + maximumNightAgreementRegression: fraction, maximumRegressedNightFraction: fraction, + maximumMacroF1Regression: fraction, maximumStageF1Regression: fraction, + minimumStageReferenceMsPerNight: z.number().int().positive().max(86400000), + requiredStages: z.array(stage).min(1).max(4).refine(rows => new Set(rows).size === rows.length, 'Duplicate required stages'), + operational: operationalPolicySchema, +}).strict() +export type StageObjectivePolicy = z.infer +export const stageObjectiveEvidenceSchema = z.object({ + split: z.enum(['tuning', 'validation', 'holdout']), holdoutUse: z.number().int().nonnegative(), + intervalLower: delta.nullable(), + nights: z.array(z.object({ cohortId: z.string(), pairedDurationMs: z.number().finite().nonnegative(), + agreementDifference: delta.nullable(), macroF1Difference: delta.nullable(), candidateOutputCoverage: fraction.nullable(), + baselineOutputMs: z.number().finite().nonnegative(), commonOutputMs: z.number().finite().nonnegative(), + perStage: z.array(z.object({ stage, referenceMs: z.number().finite().nonnegative(), f1Difference: delta.nullable() }).strict()).max(4), + }).strict()).max(60), + operational: operationalEvidenceSchema.nullable(), +}).strict() +export type StageObjectiveEvidence = z.infer +export function assessStageObjective(rawPolicy: StageObjectivePolicy, rawEvidence: StageObjectiveEvidence, designation: StageObjectivePolicy | null) { + const policy = stageObjectivePolicySchema.parse(rawPolicy), evidence = stageObjectiveEvidenceSchema.parse(rawEvidence) + if (new Set(evidence.nights.map(row => row.cohortId)).size !== evidence.nights.length + || evidence.nights.some(row => row.commonOutputMs > row.baselineOutputMs || row.pairedDurationMs > row.commonOutputMs + || row.perStage.reduce((sum, item) => sum + item.referenceMs, 0) !== row.pairedDurationMs || new Set(row.perStage.map(item => item.stage)).size !== row.perStage.length)) throw new Error('Invalid stage objective evidence') + const designated = designation !== null && JSON.stringify(stageObjectivePolicySchema.parse(designation)) === JSON.stringify(policy) + const missing: string[] = [], failures: string[] = [] + if (!designated) missing.push('policy_not_designated') + if (evidence.split !== 'holdout' || evidence.holdoutUse !== 1) missing.push('independent_holdout_required') + const supported = evidence.nights.filter(row => row.agreementDifference !== null && row.pairedDurationMs >= policy.minimumPairedDurationMsPerNight) + if (supported.length < policy.minimumNights) missing.push('insufficient_supported_nights') + if (supported.length !== evidence.nights.length) missing.push('unsupported_nights') + if (evidence.intervalLower === null) missing.push('uncertainty_unavailable') + if (evidence.nights.some(row => row.candidateOutputCoverage === null || row.baselineOutputMs === 0)) missing.push('coverage_unavailable') + if (evidence.nights.some(row => row.candidateOutputCoverage !== null && row.candidateOutputCoverage < policy.minimumOutputCoverage)) failures.push('output_coverage') + if (evidence.nights.some(row => row.baselineOutputMs > 0 && (row.baselineOutputMs - row.commonOutputMs) / row.baselineOutputMs > policy.maximumLostOutputFraction)) failures.push('coverage_loss') + const differences = supported.map(row => row.agreementDifference ?? 0) + if (differences.some(value => value < -policy.maximumNightAgreementRegression)) failures.push('night_regression') + if (differences.length && differences.filter(value => value < 0).length / differences.length > policy.maximumRegressedNightFraction) failures.push('regressed_night_fraction') + if (evidence.nights.some(row => row.macroF1Difference === null)) missing.push('macro_f1_unavailable') + if (evidence.nights.some(row => row.macroF1Difference !== null && row.macroF1Difference < -policy.maximumMacroF1Regression)) failures.push('macro_f1_regression') + for (const required of policy.requiredStages) { + const rows = evidence.nights.map(row => row.perStage.find(item => item.stage === required)) + if (rows.some(row => !row || row.referenceMs < policy.minimumStageReferenceMsPerNight || row.f1Difference === null)) missing.push(`stage_support:${required}`) + if (rows.some(row => row?.f1Difference !== null && row?.f1Difference !== undefined && row.f1Difference < -policy.maximumStageF1Regression)) failures.push(`stage_regression:${required}`) + } + const operations = assessOperationalEvidence(policy.qualificationId, policy.operational, evidence.operational) + missing.push(...operations.missing) + failures.push(...operations.failures) + const estimate = differences.length ? differences.reduce((sum, value) => sum + value, 0) / differences.length : null + const status = missing.length + ? 'insufficient_evidence' as const + : failures.length + ? 'regressed' as const + : estimate !== null && estimate >= policy.minimumAgreementImprovement && evidence.intervalLower !== null && evidence.intervalLower >= policy.minimumAgreementImprovement + ? 'improved' as const + : 'no_material_change' as const + return { version: 'stage-objective-assessment-v1' as const, policy, evidence, designated, status, missingEvidence: missing, failedThresholds: failures, + equalNightMeanAgreementDifference: estimate, intervalLower: evidence.intervalLower, + interpretation: 'agreement_with_named_stage_reference_not_clinical_validation' as const } +} diff --git a/src/biometrics/backtest/submission.ts b/src/biometrics/backtest/submission.ts new file mode 100644 index 000000000..ef9d3cfd3 --- /dev/null +++ b/src/biometrics/backtest/submission.ts @@ -0,0 +1,105 @@ +import { eq } from 'drizzle-orm' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import { z } from 'zod' +import type * as configSchema from '@/src/db/schema' +import * as bioSchema from '@/src/db/biometrics-schema' +import { resolveReplayBaseline } from '../profiles/replay-baseline' +import { experimentProtocolSchema } from './contracts' +import { workerBuildHash } from './child' +import { packagedCoordinatorPath, packagedWorkerPath } from './location' +import { waveformEvaluatorBuildIdentity } from './waveform-build' +import { podRawInspectionLocation } from './raw-inspection-child' +import { configHash } from '../algorithms/canonical' +import { defaultConfig, manifestHash } from '../algorithms/registry' +import { ProfileStore } from '../profiles/store' +import { ReferenceInputError } from '../references/time' +import { BacktestJobStore } from './jobs' + +export const submitBacktestSchema = z.object({ experimentId: z.string().uuid(), candidateRevisionId: z.string().uuid(), + expectedBindingGeneration: z.number().int().nonnegative(), split: z.enum(['tuning', 'validation', 'holdout']), + idempotencyKey: z.string().min(1).max(200) }).strict() + +export function retryBacktestSubmission(bioDb: BetterSQLite3Database, raw: z.input) { + const input = submitBacktestSchema.parse(raw) + const submissionHash = configHash(input) + const existing = bioDb.select().from(bioSchema.biometricBacktestJobs).where(eq(bioSchema.biometricBacktestJobs.idempotencyKey, input.idempotencyKey)).get() + if (existing) { + const saved = z.object({ submissionHash: z.string().optional() }).parse(existing.input) + if (saved.submissionHash !== submissionHash) throw new ReferenceInputError('REVISION_CONFLICT', 'Backtest idempotency key belongs to a different submission') + return existing + } + return null +} + +const buildProbes = globalThis as typeof globalThis & { + __waveformSubmissionBuild?: { key: string, promise: ReturnType } +} +function sharedWaveformBuild(workerPath: string) { + const coordinatorPath = packagedCoordinatorPath() + const key = JSON.stringify([coordinatorPath, workerPath, podRawInspectionLocation.pythonPath, podRawInspectionLocation.modulesPath]) + if (buildProbes.__waveformSubmissionBuild?.key === key) return buildProbes.__waveformSubmissionBuild.promise + const entry = { key, promise: waveformEvaluatorBuildIdentity(coordinatorPath, workerPath, podRawInspectionLocation) } + buildProbes.__waveformSubmissionBuild = entry + // Coalesce concurrent submissions, but never cache an identity after its probe. + entry.promise = entry.promise.finally(() => { + if (buildProbes.__waveformSubmissionBuild === entry) delete buildProbes.__waveformSubmissionBuild + }) + return entry.promise +} + +/** Resolve immutable build pins only after checking for an existing submission. */ +export async function resolveSubmissionBuild(bioDb: BetterSQLite3Database, raw: z.input) { + const input = submitBacktestSchema.parse(raw) + const existing = retryBacktestSubmission(bioDb, input) + if (existing) return { existing } + const experiment = new BacktestJobStore(bioDb).getExperiment(input.experimentId) + return resolveInstalledReplayBuild(experiment.protocol.mode) +} + +/** Read installed artifacts; never accept evaluator identity from an RPC caller. */ +export async function resolveInstalledReplayBuild(mode: z.infer['mode']) { + try { + const workerPath = packagedWorkerPath() + if (mode === 'signal_replay') { + const build = await sharedWaveformBuild(workerPath) + return { evaluatorBuildHash: build.hash, pythonBuildHash: build.python.hash } + } + return { evaluatorBuildHash: await workerBuildHash(workerPath) } + } + catch { throw new ReferenceInputError('WORKER_UNAVAILABLE', 'Packaged replay build identity cannot be read') } +} + +/** Public callers select revisions, never supply trusted baseline snapshots. */ +export function submitBacktest(configDb: BetterSQLite3Database, bioDb: BetterSQLite3Database, evaluatorBuildHash: string, + raw: z.input, pythonBuildHash?: string) { + const input = submitBacktestSchema.parse(raw) + const submissionHash = configHash(input) + const existing = retryBacktestSubmission(bioDb, input) + if (existing) return existing + const experiment = bioDb.select().from(bioSchema.biometricExperiments).where(eq(bioSchema.biometricExperiments.id, input.experimentId)).get() + if (!experiment) throw new ReferenceInputError('NOT_FOUND', 'Experiment not found') + const signalReplay = experimentProtocolSchema.parse(experiment.protocol).mode === 'signal_replay' + if (signalReplay && !pythonBuildHash) throw new ReferenceInputError('WORKER_UNAVAILABLE', 'Waveform submission requires the Python build identity') + const snapshots = configDb.transaction(() => bioDb.transaction(() => { + const profiles = new ProfileStore(configDb) + const binding = profiles.binding(experiment.side) + if (binding.generation !== input.expectedBindingGeneration) throw new ReferenceInputError('REVISION_CONFLICT', 'Profile binding changed; inspect the current side before evaluating') + const candidate = profiles.validateRevision(input.candidateRevisionId, experiment.side) + // This is the actual retrospective view used by existing stage endpoints. + // A desired-but-unacknowledged vitals profile is not a current baseline. + const currentBaseline = resolveReplayBaseline(configDb, bioDb, experiment.side) + const otherBaseline = signalReplay ? resolveReplayBaseline(configDb, bioDb, experiment.side === 'left' ? 'right' : 'left') : null + const current = currentBaseline.resolved + return { waveform: otherBaseline && pythonBuildHash + ? { pythonBuildHash, + otherSide: { manifestHash, resolvedHash: otherBaseline.hash, resolved: otherBaseline.resolved }, + baselineProvenance: { current: currentBaseline.provenance, otherSide: otherBaseline.provenance } } + : undefined, configs: [ + { label: 'default' as const, revisionId: null, manifestHash, resolvedHash: configHash(defaultConfig), resolved: defaultConfig }, + { label: 'current' as const, revisionId: currentBaseline.revisionId, manifestHash, resolvedHash: configHash(current), resolved: current }, + { label: 'candidate' as const, revisionId: candidate.revision.id, manifestHash, resolvedHash: candidate.hash, resolved: candidate.resolved }, + ] } + })) + return new BacktestJobStore(bioDb).enqueue({ experimentId: input.experimentId, idempotencyKey: input.idempotencyKey, + evaluatorBuildHash, submissionHash, split: input.split, configs: snapshots.configs, ...(snapshots.waveform ? { waveform: snapshots.waveform } : {}) }) +} diff --git a/src/biometrics/backtest/tests/admission-build.test.ts b/src/biometrics/backtest/tests/admission-build.test.ts new file mode 100644 index 000000000..0331ecfa1 --- /dev/null +++ b/src/biometrics/backtest/tests/admission-build.test.ts @@ -0,0 +1,32 @@ +// @vitest-environment node +import { build } from 'esbuild' +import { createRequire } from 'node:module' +import { runInNewContext } from 'node:vm' +import { expect, it } from 'vitest' + +it('retains Linux health collection when a bundler substitutes a macOS build platform', async () => { + const compiled = await build({ entryPoints: ['src/biometrics/backtest/admission.ts'], bundle: true, + platform: 'node', format: 'cjs', write: false, minify: true, + define: { 'process.platform': JSON.stringify('darwin') } }) + const nativeRequire = createRequire(import.meta.url) + const exported: { exports: { collectAdmissionSample?: (path: string, piezo: () => number) => Promise } } = { exports: {} } + const accesses: string[] = [] + const at = Date.now() + runInNewContext(compiled.outputFiles[0].text, { module: exported, exports: exported.exports, + process, setTimeout, clearTimeout, Buffer, + require: (name: string) => { + if (name === 'node:os') return { platform: () => 'darwin', availableParallelism: () => 4, loadavg: () => [1, 1, 1] } + if (name === 'node:fs/promises') return { + readFile: async (path: string) => { + accesses.push(path) + return path === '/proc/meminfo' ? 'MemAvailable: 1024 kB\n' : '35000\n' + }, + statfs: async () => ({ bavail: 100, bsize: 4096 }), + } + return nativeRequire(name) + } }) + expect(await exported.exports.collectAdmissionSample?.('/fixture', () => at)).toMatchObject({ + availableMemoryBytes: 1048576, freeDiskBytes: 409600, temperatureC: 35, loadPerCpu: 0.25, piezoReceivedAtMs: at, + }) + expect(accesses).toEqual(expect.arrayContaining(['/proc/meminfo', '/sys/class/thermal/thermal_zone0/temp'])) +}) diff --git a/src/biometrics/backtest/tests/admission-collector.test.ts b/src/biometrics/backtest/tests/admission-collector.test.ts new file mode 100644 index 000000000..ba8c99082 --- /dev/null +++ b/src/biometrics/backtest/tests/admission-collector.test.ts @@ -0,0 +1,39 @@ +// @vitest-environment node +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +const mocks = vi.hoisted(() => ({ readFile: vi.fn(), statfs: vi.fn(), loadavg: vi.fn(), availableParallelism: vi.fn() })) +vi.mock('node:fs/promises', () => ({ readFile: mocks.readFile, statfs: mocks.statfs })) +vi.mock('node:os', () => ({ loadavg: mocks.loadavg, availableParallelism: mocks.availableParallelism })) +import { collectAdmissionSample } from '../admission' +beforeEach(() => { + mocks.readFile.mockImplementation(async (path: string) => path === '/proc/meminfo' ? 'MemAvailable: 1024 kB\n' : '35000\n') + mocks.statfs.mockResolvedValue({ bavail: 100, bsize: 4096 }) + mocks.loadavg.mockReturnValue([2, 1, 1]) + mocks.availableParallelism.mockReturnValue(4) +}) +afterEach(() => { + vi.resetAllMocks() +}) +it('collects available disk/memory and a current scheduling probe with a piezo timestamp', async () => { + const before = Date.now() + const sample = await collectAdmissionSample('/fixture/data', () => before - 100) + expect(sample).toMatchObject({ availableMemoryBytes: 1048576, freeDiskBytes: 409600, loadPerCpu: 0.5, + temperatureC: 35, piezoReceivedAtMs: before - 100 }) + expect(sample?.measuredAtMs).toBeGreaterThanOrEqual(before) + expect(sample?.measuredAtMs).toBeLessThanOrEqual(Date.now()) + expect(sample?.eventLoopDelayMs).toBeGreaterThanOrEqual(0) + expect(mocks.statfs).toHaveBeenCalledWith('/fixture/data') +}) +it('returns unavailable when piezo has never been observed', async () => { + expect(await collectAdmissionSample('/fixture/data', () => undefined)).toBeNull() +}) +it('does not convert empty thermal files into a zero-degree reading', async () => { + mocks.readFile.mockImplementation(async (path: string) => path === '/proc/meminfo' ? 'MemAvailable: 1024 kB\n' : '\n') + expect(await collectAdmissionSample('/fixture/data', Date.now)).toBeNull() +}) +it('returns unavailable on collector failures and missing Linux health files', async () => { + mocks.statfs.mockRejectedValue(new Error('I/O error')) + expect(await collectAdmissionSample('/fixture/data', Date.now)).toBeNull() + mocks.statfs.mockResolvedValue({ bavail: 100, bsize: 4096 }) + mocks.readFile.mockRejectedValue(Object.assign(new Error('Missing Linux health file'), { code: 'ENOENT' })) + expect(await collectAdmissionSample('/fixture/data', Date.now)).toBeNull() +}) diff --git a/src/biometrics/backtest/tests/admission-service.test.ts b/src/biometrics/backtest/tests/admission-service.test.ts new file mode 100644 index 000000000..393bcc422 --- /dev/null +++ b/src/biometrics/backtest/tests/admission-service.test.ts @@ -0,0 +1,55 @@ +// @vitest-environment node +import { afterEach, expect, it, vi } from 'vitest' +import type * as Admission from '../admission' +import { collectAdmissionSample } from '../admission' +import { replayAdmission, replayAdmitter } from '../admission-service' + +const frames = vi.hoisted(() => ({ 'piezo-dual': 1000 })) +vi.mock('@/src/streaming/piezoStream', () => ({ getSensorFrameTimes: () => frames })) +vi.mock('../admission', async original => ({ ...await original(), collectAdmissionSample: vi.fn() })) +const db = { all: vi.fn(() => [{ name: 'main', file: '/private/biometrics.db' }]) } as unknown as Parameters[0] +const policy = { qualificationId: 'test-evidence', minimumAvailableMemoryBytes: 100, minimumFreeDiskBytes: 100, + maximumLoadPerCpu: 1, maximumEventLoopDelayMs: 100, maximumPiezoAgeMs: 10000, maximumTemperatureC: 65, maximumSampleAgeMs: 1000 } +afterEach(() => { + vi.unstubAllEnvs() + vi.clearAllMocks() +}) + +it.each([undefined, '', '{', '{}', JSON.stringify({ ...policy, maximumPiezoAgeMs: 0 })])('denies missing or invalid server policy without collecting health: %s', async (value) => { + vi.stubEnv('SLEEPYPOD_BIOMETRICS_ADMISSION_POLICY', value) + expect(await replayAdmission(db)).toEqual({ configured: false, qualificationId: null, admitted: false, reasons: ['unqualified'] }) + expect(collectAdmissionSample).not.toHaveBeenCalled() +}) + +it('uses fresh piezo receipt times and reevaluates health for every admission', async () => { + vi.stubEnv('SLEEPYPOD_BIOMETRICS_ADMISSION_POLICY', JSON.stringify(policy)) + vi.mocked(collectAdmissionSample).mockImplementation(async (directory, piezo) => { + expect(directory).toBe('/private') + expect(piezo()).toBe(frames['piezo-dual']) + return { measuredAtMs: Date.now(), piezoReceivedAtMs: Date.now(), availableMemoryBytes: 200, freeDiskBytes: 200, + loadPerCpu: 0.2, eventLoopDelayMs: 2, temperatureC: 30 } + }) + expect(await replayAdmission(db)).toEqual({ configured: true, qualificationId: 'test-evidence', admitted: true, reasons: [] }) + vi.mocked(collectAdmissionSample).mockResolvedValueOnce(null) + expect(await replayAdmitter(db)()).toBe(false) + expect(collectAdmissionSample).toHaveBeenCalledTimes(2) +}) + +it('fails closed when collecting health throws', async () => { + vi.stubEnv('SLEEPYPOD_BIOMETRICS_ADMISSION_POLICY', JSON.stringify(policy)) + vi.mocked(collectAdmissionSample).mockRejectedValueOnce(new Error('unavailable')) + expect(await replayAdmission(db)).toEqual({ configured: true, qualificationId: 'test-evidence', admitted: false, reasons: ['health_unavailable'] }) +}) + +it('returns the admitted sample only through the internal health callback', async () => { + const { replayHealthAdmission } = await import('../admission-service') + vi.stubEnv('SLEEPYPOD_BIOMETRICS_ADMISSION_POLICY', JSON.stringify(policy)) + vi.mocked(collectAdmissionSample).mockImplementation(async () => ({ measuredAtMs: Date.now(), piezoReceivedAtMs: Date.now() - 250, + availableMemoryBytes: 200, freeDiskBytes: 200, loadPerCpu: 0.2, eventLoopDelayMs: 2, temperatureC: 30 })) + const measured = await replayHealthAdmission(db)() + expect(measured).toMatchObject({ allowed: true, sample: { qualificationId: 'test-evidence', ingestionAgeMs: expect.any(Number) } }) + expect(measured.sample?.ingestionAgeMs).toBeGreaterThanOrEqual(250) + expect(await replayAdmission(db)).not.toHaveProperty('receipt') + vi.mocked(collectAdmissionSample).mockResolvedValueOnce(null) + expect(await replayHealthAdmission(db)()).toEqual({ allowed: false, sample: null }) +}) diff --git a/src/biometrics/backtest/tests/admission.test.ts b/src/biometrics/backtest/tests/admission.test.ts new file mode 100644 index 000000000..a15e14650 --- /dev/null +++ b/src/biometrics/backtest/tests/admission.test.ts @@ -0,0 +1,39 @@ +// @vitest-environment node +import { describe, expect, it } from 'vitest' +import { evaluateAdmission, parseAvailableMemory, type AdmissionPolicy, type AdmissionSample } from '../admission' +// Test values deliberately are not deployment defaults. +const policy: AdmissionPolicy = { qualificationId: 'fixture', minimumAvailableMemoryBytes: 1000, minimumFreeDiskBytes: 2000, + maximumLoadPerCpu: 0.8, maximumEventLoopDelayMs: 50, maximumPiezoAgeMs: 1000, maximumTemperatureC: 60, maximumSampleAgeMs: 500 } +const sample: AdmissionSample = { measuredAtMs: 10000, availableMemoryBytes: 1000, freeDiskBytes: 2000, + loadPerCpu: 0.8, eventLoopDelayMs: 50, piezoReceivedAtMs: 9500, temperatureC: 60 } +describe('replay resource admission', () => { + it('admits fresh evidence at exact qualified boundaries', () => { + expect(evaluateAdmission(sample, policy, 10500)).toEqual({ admitted: true, reasons: [] }) + }) + it.each([ + ['availableMemoryBytes', 999, 'memory_pressure'], ['freeDiskBytes', 1999, 'disk_pressure'], + ['loadPerCpu', 0.81, 'cpu_pressure'], ['eventLoopDelayMs', 51, 'event_loop_delay'], + ['piezoReceivedAtMs', 8999, 'piezo_stale'], ['temperatureC', 61, 'temperature'], + ['measuredAtMs', 9499, 'stale_sample'], ['piezoReceivedAtMs', 10001, 'clock_skew'], + ['measuredAtMs', 10001, 'clock_skew'], + ])('defers for %s outside the qualified range', (key, value, reason) => { + expect(evaluateAdmission({ ...sample, [key]: value }, policy, 10000)).toEqual({ admitted: false, reasons: [reason] }) + }) + it('does not turn absent, malformed, or nonfinite health evidence into zeros', () => { + for (const invalid of [null, {}, { ...sample, availableMemoryBytes: undefined }, { ...sample, temperatureC: NaN }, { ...sample, freeDiskBytes: Infinity }]) { + expect(evaluateAdmission(invalid, policy, 10000)).toEqual({ admitted: false, reasons: ['health_unavailable'] }) + } + expect(evaluateAdmission(sample, null, 10000)).toEqual({ admitted: false, reasons: ['unqualified'] }) + expect(evaluateAdmission(sample, { ...policy, maximumLoadPerCpu: 0 }, 10000).admitted).toBe(false) + }) + it('reports simultaneous pressure without hiding subsequent reasons', () => { + expect(evaluateAdmission({ ...sample, availableMemoryBytes: 0, freeDiskBytes: 0, temperatureC: 70 }, policy, 10000).reasons) + .toEqual(['memory_pressure', 'disk_pressure', 'temperature']) + }) + it('reads reclaimable available Linux memory, not only unused pages', () => { + expect(parseAvailableMemory('MemTotal: 2000 kB\nMemFree: 100 kB\nMemAvailable: 1200 kB\n')).toBe(1228800) + for (const invalid of ['MemFree: 100 kB', 'MemAvailable: -1 kB', 'MemAvailable: 2 MB', 'MemAvailable: 99999999999999999 kB']) { + expect(() => parseAvailableMemory(invalid)).toThrow() + } + }) +}) diff --git a/src/biometrics/backtest/tests/artifacts.test.ts b/src/biometrics/backtest/tests/artifacts.test.ts new file mode 100644 index 000000000..5cf0da3e7 --- /dev/null +++ b/src/biometrics/backtest/tests/artifacts.test.ts @@ -0,0 +1,139 @@ +// @vitest-environment node +import { mkdtempSync, readFileSync, readdirSync, rmSync, utimesSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { spawn } from 'node:child_process' +import { once } from 'node:events' +import { createRequire } from 'node:module' +import { join } from 'node:path' +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { afterEach, beforeEach, expect, it } from 'vitest' +import * as schema from '@/src/db/biometrics-schema' +import { ReportArtifactStore, REPORT_BUDGET_BYTES } from '../artifacts' +import { BacktestJobStore } from '../jobs' +import { DatasetStore } from '../datasets' +import { backtestFixture } from './fixtures/backtest' +let directory: string +beforeEach(() => { + directory = mkdtempSync(join(tmpdir(), 'biometric-artifacts-')) +}) +afterEach(() => rmSync(directory, { recursive: true, force: true })) +const id = '11111111-1111-4111-8111-111111111111' +it('atomically retains complete data, verifies bytes and rejects traversal and corrupt files', () => { + const store = new ReportArtifactStore(directory) + const report = { traces: Array.from({ length: 1100 }, () => 'x'.repeat(2048)) } + const descriptor = store.write(id, 1, JSON.stringify(report), 0) + expect(store.read(descriptor)).toEqual(report) + expect(readdirSync(directory).filter(name => !name.startsWith('.publication.sqlite'))).toEqual([descriptor.name]) + expect(() => store.read({ ...descriptor, name: '../private.json' })).toThrow('descriptor') + writeFileSync(join(directory, descriptor.name), 'corrupt') + expect(() => store.read(descriptor)).toThrow('corrupt') +}) +it('counts staged storage against quota and only collects old unreferenced owned files', () => { + const store = new ReportArtifactStore(directory) + expect(() => store.write(id, 1, '{}', REPORT_BUDGET_BYTES)).toThrow('budget') + const committed = store.write(id, 1, '{"first":1}', 0) + const orphan = store.write(id, 2, '{"second":2}', 0) + const fresh = store.write(id, 3, '{"third":3}', 0) + const old = new Date(Date.now() - 7200000) + for (const name of [committed.name, orphan.name]) utimesSync(join(directory, name), old, old) + writeFileSync(join(directory, 'unrelated.txt'), 'keep') + expect(store.collect(new Set([committed.name]))).toBe(1) + expect(readdirSync(directory).filter(name => !name.startsWith('.publication.sqlite')).sort()).toEqual([committed.name, fresh.name, 'unrelated.txt'].sort()) +}) +it('publishes a descriptor under the lease, resolves it after reopening, and retains an orphan after failed DB commit', () => { + const databasePath = join(directory, 'biometrics.db') + const raw = new Database(databasePath) + try { + const db = drizzle(raw, { schema }) + migrate(db, { migrationsFolder: 'src/db/biometrics-migrations' }) + const now = Date.now() + db.insert(schema.sleepRecords).values({ side: 'left', enteredBedAt: new Date(now - 3600000), leftBedAt: new Date(now), sleepDurationSeconds: 3600, timesExitedBed: 0 }).run() + const dataset = new DatasetStore(db).create({ side: 'left', subjectId: 'fixture', sessionIds: [1], referenceImportIds: [], timezone: 'UTC', idempotencyKey: 'dataset' }) + const store = new BacktestJobStore(db) + const fixture = backtestFixture() + const experiment = store.createExperiment(dataset.id, fixture.job.protocol) + const enqueue = (key: string) => store.enqueue({ experimentId: experiment.id, configs: fixture.job.configs, evaluatorBuildHash: 'f'.repeat(64), idempotencyKey: key, split: 'tuning' }) + const job = enqueue('first') + const claimed = store.claim('fixture') + if (!claimed) throw new Error('Missing lease') + const report = { version: 'fixture', traces: Array.from({ length: 1100 }, () => 'x'.repeat(2048)) } + store.complete({ id: job.id, owner: 'fixture', token: claimed.fencingToken }, report) + const persisted = db.select().from(schema.biometricBacktestResults).get() + expect(persisted?.report).toMatchObject({ kind: 'biometric-report-artifact-v1' }) + expect(Buffer.byteLength(JSON.stringify(persisted?.report))).toBeLessThan(512) + const reopened = new Database(databasePath) + try { + expect(new BacktestJobStore(drizzle(reopened, { schema })).result(job.id)?.report).toEqual(report) + } + finally { reopened.close() } + const second = enqueue('second') + const secondLease = store.claim('fixture') + if (!secondLease) throw new Error('Missing second lease') + raw.exec('CREATE TRIGGER reject_result BEFORE INSERT ON biometric_backtest_results BEGIN SELECT RAISE(ABORT, \'full\'); END;') + expect(() => store.complete({ id: second.id, owner: 'fixture', token: secondLease.fencingToken }, report)).toThrow('full') + expect(store.get(second.id).state).toBe('running') + expect(store.result(second.id)).toBeNull() + const root = join(directory, 'biometric-report-artifacts') + expect(readdirSync(root).filter(name => !name.startsWith('.publication.sqlite'))).toHaveLength(2) + for (const name of readdirSync(root).filter(name => !name.startsWith('.publication.sqlite'))) { + expect(JSON.parse(readFileSync(join(root, name), 'utf8'))).toEqual(report) + utimesSync(join(root, name), new Date(0), new Date(0)) + } + expect(store.collectOrphanedArtifacts()).toBe(1) + expect(store.result(job.id)?.report).toEqual(report) + const hash = persisted?.contentHash + if (!hash) throw new Error('Missing report hash') + raw.exec('CREATE TRIGGER reject_release BEFORE UPDATE ON biometric_backtest_results BEGIN SELECT RAISE(ABORT, \'full\'); END;') + expect(() => store.releaseResult(job.id, hash)).toThrow('full') + expect(store.result(job.id)?.report).toEqual(report) + raw.exec('DROP TRIGGER reject_release') + const released = store.releaseResult(job.id, hash) + expect(released).toMatchObject({ available: false, report: null, contentHash: hash }) + expect(readdirSync(root).filter(name => !name.startsWith('.publication.sqlite'))).toEqual([]) + expect(store.releaseResult(job.id, hash)).toEqual(released) + expect(store.result(job.id)).toEqual(released) + } + finally { raw.close() } +}) + +it('reuses an identical staged artifact near quota and serializes other publishers', () => { + const store = new ReportArtifactStore(directory) + const text = JSON.stringify({ value: 'data' }) + const descriptor = store.write(id, 1, text, 0) + expect(store.write(id, 1, text, REPORT_BUDGET_BYTES - descriptor.byteLength)).toEqual(descriptor) + const contender = new ReportArtifactStore(directory) + store.withPublication(() => { + expect(() => contender.write(id, 2, '{}', 0)).toThrow(/locked/) + }) + expect(contender.write(id, 2, '{}', 0).byteLength).toBe(2) +}) + +it('releases the publication lock after a process dies without committing', async () => { + const sqliteModule = createRequire(import.meta.url).resolve('better-sqlite3') + const child = spawn(process.execPath, ['-e', ` + const Database = require(process.argv[1]); + const db = new Database(process.argv[2]); + db.exec('BEGIN IMMEDIATE'); + process.send('locked'); + setInterval(() => {}, 1000); + `, sqliteModule, join(directory, '.publication.sqlite')], { stdio: ['ignore', 'ignore', 'pipe', 'ipc'] }) + const exited = once(child, 'exit') + try { + await Promise.race([ + once(child, 'message'), + exited.then(() => { throw new Error('Lock holder exited before readiness') }), + ]) + const store = new ReportArtifactStore(directory) + expect(() => store.write(id, 1, '{}', 0)).toThrow(/locked/) + child.kill('SIGKILL') + await exited + const descriptor = store.write(id, 1, '{}', 0) + expect(store.read(descriptor)).toEqual({}) + } + finally { + child.kill('SIGKILL') + await exited + } +}) diff --git a/src/biometrics/backtest/tests/child.test.ts b/src/biometrics/backtest/tests/child.test.ts new file mode 100644 index 000000000..69e070c58 --- /dev/null +++ b/src/biometrics/backtest/tests/child.test.ts @@ -0,0 +1,57 @@ +// @vitest-environment node +import { execFile } from 'node:child_process' +import { mkdtemp, rm, writeFile, readFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { promisify } from 'node:util' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { dataHash } from '../../algorithms/canonical' +import { runBacktestChild } from '../child' +import { backtestFixture } from './fixtures/backtest' +let directory: string +let worker: string +beforeAll(async () => { + directory = await mkdtemp(join(tmpdir(), 'biometrics-worker-test-')) + worker = join(directory, 'worker.cjs') + await promisify(execFile)(process.execPath, ['scripts/build/biometrics-worker.mjs', directory]) +}) +afterAll(async () => rm(directory, { recursive: true, force: true })) + +describe('shipped worker process', () => { + it('executes a full frozen replay in a child without live dependencies', async () => { + const progress: number[] = [] + const report = await runBacktestChild(worker, backtestFixture(), { onProgress: completed => progress.push(completed) }) + expect(progress).toEqual([1]) + expect(report).toMatchObject({ version: 'processed-backtest-report-v1', status: 'insufficient_evidence', nights: [{ uniqueObservationCount: 10 }] }) + const dependencies = await readFile(join(directory, 'dependencies.json'), 'utf8') + expect(dependencies).not.toMatch(/src\/db\/|src\/hardware\/|drizzle-orm|better-sqlite3/) + }) + it('processes a retained twelve-hour window beyond the configuration hash size limit', async () => { + const input = backtestFixture() + const start = input.dataset.snapshot.sessions[0].startMs + for (const session of input.dataset.snapshot.sessions) { + session.endMs = start + 43200000 + session.vitals = Array.from({ length: 720 }, (_, id) => ({ id, timestampMs: start + id * 60000, heartRate: 60.1, hrv: 30, breathingRate: 14, processingRunId: null })) + } + input.dataset.manifest.splitProtocol.assignments[0].endMs = start + 43200000 + input.job.datasetHash = dataHash(input.dataset) + expect(await runBacktestChild(worker, input)).toMatchObject({ nights: [{ uniqueObservationCount: 720 }] }) + }) + + it('terminates stalled work on time budget and cancellation, including ignored SIGTERM', async () => { + const stalled = join(directory, 'stalled.cjs') + await writeFile(stalled, 'process.on(\'SIGTERM\', () => {}); process.stdin.resume(); setInterval(() => {}, 1000)') + await expect(runBacktestChild(stalled, {}, { timeoutMs: 100 })).rejects.toThrow('time budget') + const controller = new AbortController() + const running = runBacktestChild(stalled, {}, { signal: controller.signal }) + setTimeout(() => controller.abort(), 100) + await expect(running).rejects.toThrow('cancelled') + }) + it('rejects invalid inputs, malformed output and nonzero worker exits', async () => { + await expect(runBacktestChild(worker, {})).rejects.toThrow('failed') + const malformed = join(directory, 'malformed.cjs') + await writeFile(malformed, 'process.stdin.resume(); process.stdout.write(\'not JSON\\n\')') + await expect(runBacktestChild(malformed, {})).rejects.toThrow('invalid event') + await expect(runBacktestChild(worker, {}, { timeoutMs: 0 })).rejects.toThrow('time budget') + }) +}) diff --git a/src/biometrics/backtest/tests/common-support.test.ts b/src/biometrics/backtest/tests/common-support.test.ts new file mode 100644 index 000000000..88a34b91b --- /dev/null +++ b/src/biometrics/backtest/tests/common-support.test.ts @@ -0,0 +1,38 @@ +// @vitest-environment node +import { expect, it } from 'vitest' +import { intersectOutputSupport, restrictToSupport } from '../common-support' +import { compareStageOutputs } from '../stage-comparison' +import { previewReferenceImport } from '../../references/normalize' +it('intersects all three outputs and clips boundaries without filling gaps', () => { + const outputs = [[{ start: 0, end: 100 }], [{ start: 10, end: 40 }, { start: 50, end: 90 }], [{ start: 20, end: 60 }]] + const before = JSON.stringify(outputs) + expect(intersectOutputSupport(outputs, 25, 80)).toEqual([{ start: 25, end: 40 }, { start: 50, end: 60 }]) + expect(JSON.stringify(outputs)).toBe(before) + expect(intersectOutputSupport([outputs[0], [], outputs[2]], 0, 100)).toEqual([]) +}) +it('preserves stage labels while rejecting ambiguous support', () => { + expect(restrictToSupport([{ start: 0, end: 10, stage: 'deep' }], [{ start: 2, end: 4 }, { start: 6, end: 8 }])) + .toEqual([{ start: 2, end: 4, stage: 'deep' }, { start: 6, end: 8, stage: 'deep' }]) + expect(() => intersectOutputSupport([[{ start: 0, end: 10 }, { start: 5, end: 15 }]], 0, 20)).toThrow('nonoverlapping') + expect(() => intersectOutputSupport([], 0, 20)).toThrow('bounded') +}) +it('scores every pair on the same three-way denominator while keeping own coverage', () => { + const start = 1791673200000, end = start + 20000 + const full = [{ start, end, stage: 'light' as const }] + const current = [{ start, end, stage: 'deep' as const }] + const candidate = [{ start, end: start + 10000, stage: 'light' as const }] + const refs = previewReferenceImport({ side: 'left', subjectId: 'fixture', source: 'test', rows: [ + { recordKey: 'stage', metric: 'sleep_stages', value: 'light', unit: 'stage', startMs: start, endMs: end }, + ] }).measurements + const support = intersectOutputSupport([full, current, candidate], start, end) + for (const [baseline, evaluated] of [[full, current], [full, candidate], [current, candidate]]) { + const comparison = compareStageOutputs(baseline, evaluated, refs, start, end, support) + expect(comparison.commonSupportScope).toBe('all_configurations') + expect(comparison.commonSupport.baseline.pairedDurationMs).toBe(10000) + expect(comparison.commonSupport.candidate.pairedDurationMs).toBe(10000) + } + const comparison = compareStageOutputs(full, current, refs, start, end, support) + expect(comparison.ownSupport.baseline.pairedDurationMs).toBe(20000) + expect(comparison.ownSupport.candidate.pairedDurationMs).toBe(20000) + expect(compareStageOutputs(full, current, refs, start, end, []).differences.commonAgreement).toBeNull() +}) diff --git a/src/biometrics/backtest/tests/coordinator-entry.test.ts b/src/biometrics/backtest/tests/coordinator-entry.test.ts new file mode 100644 index 000000000..3c01bc07b --- /dev/null +++ b/src/biometrics/backtest/tests/coordinator-entry.test.ts @@ -0,0 +1,139 @@ +// @vitest-environment node +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import type { performCoordinatorTask } from '../coordinator-tasks' + +const mocks = vi.hoisted(() => ({ task: vi.fn(), open: vi.fn(), close: vi.fn(), pragma: vi.fn(), db: {} })) +vi.mock('../coordinator-tasks', () => ({ performCoordinatorTask: mocks.task })) +vi.mock('better-sqlite3', () => ({ default: class { + constructor(...args: unknown[]) { mocks.open(...args) } + close = mocks.close + pragma = mocks.pragma +} })) +vi.mock('drizzle-orm/better-sqlite3', () => ({ drizzle: () => mocks.db })) +type Options = Parameters[2] +let listeners: Record unknown> +let sent: Array> +let disconnect: ReturnType +let kill: ReturnType +let exit: ReturnType +async function start(budget = 'null') { + const original = process + disconnect = vi.fn(() => listeners.disconnect?.()) + kill = vi.fn() + exit = vi.fn() + vi.stubGlobal('process', Object.assign(Object.create(original), { + argv: ['node', 'coordinator', '/owned/biometrics.db', '/owned/worker.cjs', 'null', budget], + on: (event: string, listener: (...args: unknown[]) => unknown) => { listeners[event] = listener }, + send: (message: Record, callback?: () => void) => { + sent.push(message) + callback?.() + }, + disconnect, kill, exit, + })) + await import('../coordinator-entry') +} +const message = (value: unknown) => listeners.message(value) +beforeEach(() => { + vi.resetModules() + vi.clearAllMocks() + mocks.task.mockReset().mockResolvedValue({ ok: true }) + mocks.open.mockReset() + listeners = {} + sent = [] +}) +afterEach(() => { + vi.unstubAllGlobals() + vi.useRealTimers() +}) + +it('runs once, closes its database, and disconnects without killing completed work', async () => { + await start() + await message(null) + await message({ type: 'unexpected' }) + await message({ type: 'run', task: { type: 'collectArtifacts' } }) + await message({ type: 'run', task: { type: 'execute' } }) + expect(mocks.open).toHaveBeenCalledExactlyOnceWith('/owned/biometrics.db', { fileMustExist: true, timeout: 1000 }) + expect(mocks.pragma).toHaveBeenCalledWith('foreign_keys = ON') + expect(mocks.task).toHaveBeenCalledOnce() + expect(mocks.task.mock.calls[0][2]).toMatchObject({ workerPath: '/owned/worker.cjs', rawLocation: undefined, replayBudget: undefined }) + expect(sent).toEqual([{ type: 'result', result: { ok: true } }]) + expect(mocks.close).toHaveBeenCalledOnce() + expect(disconnect).toHaveBeenCalledOnce() + expect(kill).not.toHaveBeenCalled() + expect(exit).not.toHaveBeenCalled() +}) + +it.each([['valid', true], ['malformed', false], ['timeout', false]] as const)('correlates admission and fails closed for %s receipts', async (kind, allowed) => { + vi.useFakeTimers() + let options!: Options + mocks.task.mockImplementation(async (_db, _task, received: Options) => { + options = received + expect(options.admissionReceipt?.()).toBeNull() + return { admitted: await options.admit() } + }) + await start() + const running = message({ type: 'run', task: {} }) + expect(sent).toEqual([{ type: 'admission', sequence: 1 }]) + await message({ type: 'admission', sequence: 99, allowed: true }) + expect(options.admissionReceipt?.()).toBeNull() + if (kind === 'timeout') await vi.advanceTimersByTimeAsync(2000) + else await message({ type: 'admission', sequence: 1, allowed: kind === 'valid' ? true : 'yes', sample: null }) + await running + expect(sent.at(-1)).toEqual({ type: 'result', result: { admitted: allowed } }) + expect(options.admissionReceipt?.()).toEqual({ allowed, sample: null }) + expect(vi.getTimerCount()).toBe(0) +}) + +it('propagates cancellation and kills its owned group on premature supervisor disconnect', async () => { + await start() + await message({ type: 'cancel' }) + mocks.task.mockImplementation(async (_db, _task, options: Options) => { + expect(options.signal.aborted).toBe(true) + listeners.disconnect() + return null + }) + await message({ type: 'run', task: {} }) + if (process.platform !== 'win32') expect(kill).toHaveBeenCalledWith(-process.pid, 'SIGKILL') + expect(exit).toHaveBeenCalledExactlyOnceWith(1) +}) + +it.each([ + ['known', 'INVALID_INPUT', 'x'.repeat(1000)], + ['unexpected', 'COORDINATOR_FAILED', 'Biometrics coordinator failed'], +])('bounds known error text and suppresses unexpected exports', async (error, code, text) => { + await start() + const { ReferenceInputError } = await import('../../references/time') + mocks.task.mockRejectedValue(error === 'known' ? new ReferenceInputError('INVALID_INPUT', 'x'.repeat(1200)) : new Error('private source export')) + await message({ type: 'run', task: {} }) + expect(sent).toEqual([{ type: 'error', code, message: text }]) + expect(mocks.close).toHaveBeenCalledOnce() + expect(disconnect).toHaveBeenCalledOnce() + expect(exit).not.toHaveBeenCalled() +}) + +it('rejects oversized results without sending their contents', async () => { + mocks.task.mockResolvedValue({ private: 'x'.repeat(17 * 1024 * 1024) }) + await start() + await message({ type: 'run', task: {} }) + expect(sent).toEqual([{ type: 'error', code: 'COORDINATOR_FAILED', message: 'Biometrics coordinator failed' }]) + expect(mocks.close).toHaveBeenCalledOnce() +}) + +it('validates execution limits before opening storage', async () => { + await start('{"jobWallMs":-1}') + await message({ type: 'run', task: {} }) + expect(mocks.open).not.toHaveBeenCalled() + expect(mocks.task).not.toHaveBeenCalled() + expect(sent[0]).toMatchObject({ type: 'error', code: 'COORDINATOR_FAILED' }) +}) + +it('reports database-open failure without attempting work or leaking details', async () => { + mocks.open.mockImplementation(() => { + throw new Error('/private/path') + }) + await start() + await message({ type: 'run', task: {} }) + expect(mocks.task).not.toHaveBeenCalled() + expect(mocks.close).not.toHaveBeenCalled() + expect(sent).toEqual([{ type: 'error', code: 'COORDINATOR_FAILED', message: 'Biometrics coordinator failed' }]) +}) diff --git a/src/biometrics/backtest/tests/coordinator-queue.test.ts b/src/biometrics/backtest/tests/coordinator-queue.test.ts new file mode 100644 index 000000000..ca0fe3f0c --- /dev/null +++ b/src/biometrics/backtest/tests/coordinator-queue.test.ts @@ -0,0 +1,111 @@ +// @vitest-environment node +import { afterEach, expect, it, vi } from 'vitest' +import { CoordinatorQueue } from '../coordinator-queue' +const options = { databasePath: '/unused.db', coordinatorPath: '/unused.cjs', workerPath: '/unused-worker.cjs' } +afterEach(() => vi.useRealTimers()) +const report = { type: 'readSummary' as const, input: { id: '00000000-0000-4000-8000-000000000001' } } + +it('serializes report readers and frees the read slot after a failure without stopping replay', async () => { + const children: { task: string, resolve: (value: null) => void, reject: (error: Error) => void }[] = [] + const execute = vi.fn().mockImplementation((_options, task) => new Promise((resolve, reject) => { + children.push({ task: task.type, resolve, reject }) + })) + const queue = new CoordinatorQueue(options, execute) + const replay = queue.request({ type: 'execute', owner: 'test' }) + const failedRead = expect(queue.request(report)).rejects.toThrow('Unreadable report') + const nextRead = queue.request(report) + expect(children.map(child => child.task)).toEqual(['execute', 'readSummary']) + children[1].reject(new Error('Unreadable report')) + await failedRead + expect(children.map(child => child.task)).toEqual(['execute', 'readSummary', 'readSummary']) + children[2].resolve(null) + await nextRead + children[0].resolve(null) + await replay + await queue.stop() +}) + +it('serves sequential report reads while replay occupies the work slot', async () => { + let finishReplay: (() => void) | undefined + const execute = vi.fn().mockImplementation((_options, task) => task.type === 'execute' + ? new Promise((resolve) => { finishReplay = () => resolve(null) }) + : Promise.resolve(null)) + const queue = new CoordinatorQueue(options, execute) + const replay = queue.request({ type: 'execute', owner: 'test' }) + const scan = queue.request({ type: 'inspectRawSource', input: { afterName: null } }) + await queue.request(report) + await queue.request(report) + expect(execute.mock.calls.map(call => call[1].type)).toEqual(['execute', 'readSummary', 'readSummary']) + finishReplay?.() + await Promise.all([replay, scan]) + expect(execute.mock.calls.map(call => call[1].type)).toEqual(['execute', 'readSummary', 'readSummary', 'inspectRawSource']) + await queue.stop() +}) + +it('bounds both lanes and cancels both active children on shutdown without starting pending work', async () => { + const execute = vi.fn().mockImplementation((_options, _task, control) => new Promise((resolve) => { + control.signal.addEventListener('abort', () => resolve(null)) + })) + const queue = new CoordinatorQueue(options, execute) + const reads = Array.from({ length: 7 }, () => queue.request(report).catch(error => error)) + await expect(queue.request(report)).rejects.toMatchObject({ code: 'RESOURCE_LIMIT' }) + const replay = queue.request({ type: 'execute', owner: 'test' }) + await expect(queue.request({ type: 'collectArtifacts' })).rejects.toMatchObject({ code: 'RESOURCE_LIMIT' }) + expect(execute).toHaveBeenCalledTimes(2) + await queue.stop() + const outcomes = await Promise.all(reads) + await replay + expect(outcomes.slice(1).every(error => error instanceof Error)).toBe(true) + expect(execute.mock.calls.every(call => call[2].signal.aborted)).toBe(true) + expect(execute).toHaveBeenCalledTimes(2) +}) + +it('reserves read capacity while limiting and serializing active plus queued work', async () => { + let release: (() => void) | undefined + const execute = vi.fn().mockImplementationOnce(() => new Promise((resolve) => { + release = () => resolve(null) + })).mockResolvedValue(null) + const queue = new CoordinatorQueue(options, execute) + const requests = Array.from({ length: 7 }, () => queue.request({ type: 'collectArtifacts' })) + await expect(queue.request({ type: 'collectArtifacts' })).rejects.toMatchObject({ code: 'RESOURCE_LIMIT' }) + expect(execute).toHaveBeenCalledTimes(1) + release?.() + await Promise.all(requests) + expect(execute).toHaveBeenCalledTimes(7) + await queue.stop() +}) +it('expires queued requests without executing them and propagates cancellation to active work', async () => { + vi.useFakeTimers() + let release: (() => void) | undefined + const execute = vi.fn().mockImplementation((_options, _task, control) => new Promise((resolve) => { + release = () => resolve(null) + control.signal.addEventListener('abort', () => resolve(null)) + })) + const queue = new CoordinatorQueue(options, execute) + const running = queue.request({ type: 'collectArtifacts' }) + const pending = expect(queue.request({ type: 'collectArtifacts' })).rejects.toThrow('queue wait exceeded') + await vi.advanceTimersByTimeAsync(30000) + await pending + expect(execute).toHaveBeenCalledTimes(1) + await queue.stop() + await running + expect(execute.mock.calls[0][2].signal.aborted).toBe(true) + release?.() + await expect(queue.request({ type: 'collectArtifacts' })).rejects.toThrow('stopped') +}) +it('removes a cancelled queued request without waiting for the active operation', async () => { + let release: (() => void) | undefined + const execute = vi.fn().mockImplementation(() => new Promise((resolve) => { + release = () => resolve(null) + })) + const queue = new CoordinatorQueue(options, execute) + const first = queue.request({ type: 'collectArtifacts' }) + const controller = new AbortController() + const cancelled = queue.request({ type: 'collectArtifacts' }, { signal: controller.signal }) + controller.abort() + await expect(cancelled).rejects.toThrow('cancelled') + release?.() + await first + await queue.stop() + expect(execute).toHaveBeenCalledTimes(1) +}) diff --git a/src/biometrics/backtest/tests/coordinator-tasks.test.ts b/src/biometrics/backtest/tests/coordinator-tasks.test.ts new file mode 100644 index 000000000..f3a02fa8d --- /dev/null +++ b/src/biometrics/backtest/tests/coordinator-tasks.test.ts @@ -0,0 +1,204 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { build } from 'esbuild' +import { BacktestJobStore } from '../jobs' +import { DatasetStore } from '../datasets' +import { workerBuildHash } from '../child' +import { backtestFixture } from './fixtures/backtest' +import { datasetManifestSchema } from '../contracts' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { mkdtempSync, realpathSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { randomUUID } from 'node:crypto' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import * as schema from '@/src/db/biometrics-schema' +import { performCoordinatorTask, type CoordinatorTask } from '../coordinator-tasks' +import { runRawInspectionChild } from '../raw-inspection-child' +import type * as RawInspection from '../raw-inspection-child' +import { runRawCollectionChild } from '../raw-collection-child' + +vi.mock('../raw-inspection-child', async original => ({ ...await original(), runRawInspectionChild: vi.fn() })) +vi.mock('../raw-collection-child', () => ({ runRawCollectionChild: vi.fn() })) +let directory: string, sqlite: Database.Database, db: ReturnType> +let controller: AbortController +let admit: ReturnType Promise>> +const location = { pythonPath: '/trusted/python', modulesPath: '/trusted/modules', archiveDirectory: '/trusted/archive' } +const execute = (task: CoordinatorTask) => performCoordinatorTask(db, task, { workerPath: '/trusted/worker', rawLocation: location, signal: controller.signal, admit }) +const scan = () => execute({ type: 'inspectRawSource', input: { afterName: 'previous.RAW.gz' } }) +const deferred = { status: 'deferred', nextCursor: 'previous.RAW.gz' } +function inspection() { + const hash = 'a'.repeat(64) + return { source_name: 'capture.RAW.gz', retained: false as const, artifact: { name: `${hash}.RAW.gz`, sha256: hash, compressed_bytes: 100, expanded_bytes: 200 }, + inventory: { version: 'retained-raw-inventory-v1' as const, artifact_sha256: hash, + decoded_records: 2, piezo_records: 1, pump_records: 1, usable_dual_frames: 1, empty_channel_frames: 0, + oversize_channel_frames: 0, invalid_timestamps: 0, timestamp_reversals: 0, duplicate_timestamps: 0, + first_event_ms: 1000, last_event_ms: 2000, + channels: { left1: { samples: 10, lost_markers: 0, truncated_tail_bytes: 0 }, right1: { samples: 10, lost_markers: 0, truncated_tail_bytes: 0 } }, + ordered_waveform_available: true, limitations: [] } } +} +beforeEach(() => { + vi.clearAllMocks() + directory = realpathSync(mkdtempSync(join(tmpdir(), 'coordinator-tasks-'))) + sqlite = new Database(join(directory, 'biometrics.db')) + sqlite.pragma('foreign_keys = ON') + db = drizzle(sqlite, { schema }) + migrate(db, { migrationsFolder: 'src/db/biometrics-migrations' }) + controller = new AbortController() + admit = vi.fn().mockResolvedValue(true) + vi.mocked(runRawInspectionChild).mockReset().mockResolvedValue({ status: 'complete', nextCursor: null }) + vi.mocked(runRawCollectionChild).mockReset().mockResolvedValue({ version: 'raw-collection-result-v1', released: 0, orphans: 0 }) +}) +afterEach(() => { + vi.useRealTimers() + sqlite.close() + rmSync(directory, { recursive: true, force: true }) +}) + +it('retains an immutable dataset, releases it, and leaves canonical observations intact', async () => { + const start = 1791673200000 + db.insert(schema.sleepRecords).values({ id: 1, side: 'left', enteredBedAt: new Date(start), leftBedAt: new Date(start + 3600000), sleepDurationSeconds: 3600, timesExitedBed: 0 }).run() + db.insert(schema.vitals).values({ side: 'left', timestamp: new Date(start), heartRate: 65 }).run() + const input = { side: 'left' as const, subjectId: 'fixture', sessionIds: [1], referenceImportIds: [], timezone: 'UTC', idempotencyKey: 'one' } + const dataset = await execute({ type: 'createDataset', input }) as { id: string, contentHash: string } + expect(await execute({ type: 'createDataset', input })).toEqual(dataset) + expect(await execute({ type: 'datasetDescriptor', input: { id: dataset.id } })).toMatchObject({ contentHash: dataset.contentHash, available: true }) + await execute({ type: 'releaseDataset', input: { id: dataset.id } }) + expect(await execute({ type: 'datasetDescriptor', input: { id: dataset.id } })).toMatchObject({ available: false }) + expect(sqlite.prepare('SELECT heart_rate FROM vitals').all()).toEqual([{ heart_rate: 65 }]) +}) + +it.each(['readResult', 'readSummary', 'releaseResult'] as const)('returns no %s for an unknown experiment without creating rows', async (type) => { + const id = randomUUID() + await expect(execute(type === 'releaseResult' ? { type, input: { id, expectedContentHash: 'a'.repeat(64) } } : { type, input: { id } })).rejects.toMatchObject({ code: 'NOT_FOUND' }) + expect(sqlite.prepare('SELECT count(*) AS n FROM biometric_experiments').get()).toEqual({ n: 0 }) +}) + +it('refuses caller-supplied execution paths before inspecting or collecting', async () => { + await expect(performCoordinatorTask(db, { type: 'inspectRawSource', input: { afterName: null }, workerPath: '/untrusted' }, { workerPath: '/trusted', signal: controller.signal, admit })).rejects.toThrow() + expect(runRawInspectionChild).not.toHaveBeenCalled() + expect(admit).not.toHaveBeenCalled() +}) + +it.each(['cancelled', 'denied'] as const)('does not launch a %s inspection or collection', async (reason) => { + if (reason === 'cancelled') controller.abort() + else admit.mockResolvedValue(false) + expect(await scan()).toEqual(deferred) + expect(await execute({ type: 'collectArtifacts' })).toBeNull() + expect(runRawInspectionChild).not.toHaveBeenCalled() + expect(runRawCollectionChild).not.toHaveBeenCalled() +}) + +it('records inspected evidence only after rechecking admission and pages the retained identity', async () => { + vi.mocked(runRawInspectionChild).mockResolvedValue({ status: 'inspected', nextCursor: 'capture.RAW.gz', inspection: inspection() }) + const result = await scan() + expect(result).toMatchObject({ status: 'inspected', source: { inspection: inspection(), availability: 'requires_reverification' } }) + expect(admit).toHaveBeenCalledTimes(2) + expect(await execute({ type: 'pageRawSources', input: { startMs: 1000, endMs: 3000 } })).toMatchObject({ items: [expect.objectContaining({ inspection: inspection() })], nextCursor: null }) +}) + +it('discards inspected evidence when final admission is denied', async () => { + vi.mocked(runRawInspectionChild).mockResolvedValue({ status: 'inspected', nextCursor: 'capture.RAW.gz', inspection: inspection() }) + admit.mockResolvedValueOnce(true).mockResolvedValueOnce(false) + expect(await scan()).toEqual(deferred) + expect(await execute({ type: 'pageRawSources', input: { startMs: 1000, endMs: 3000 } })).toMatchObject({ items: [] }) +}) + +it('passes through archive unavailability and propagates actual inspection failures', async () => { + vi.mocked(runRawInspectionChild).mockResolvedValueOnce({ status: 'archive_unavailable', nextCursor: null }).mockRejectedValueOnce(new Error('decoder failed')) + expect(await scan()).toEqual({ status: 'archive_unavailable', nextCursor: null }) + await expect(scan()).rejects.toThrow('decoder failed') +}) + +it.each(['denied', 'admission_error'] as const)('cancels an in-flight inspection on %s without persisting evidence', async (reason) => { + vi.useFakeTimers() + vi.mocked(runRawInspectionChild).mockImplementation(async (_location, _input, options) => new Promise((_resolve, reject) => { + options?.signal?.addEventListener('abort', () => reject(new Error('cancelled')), { once: true }) + })) + if (reason === 'denied') admit.mockResolvedValueOnce(true).mockResolvedValue(false) + else admit.mockResolvedValueOnce(true).mockRejectedValue(new Error('health unavailable')) + const running = scan() + await vi.advanceTimersByTimeAsync(5000) + expect(await running).toEqual(deferred) + expect(vi.getTimerCount()).toBe(0) +}) + +it('forwards caller cancellation to inspection and removes its health timer', async () => { + vi.useFakeTimers() + vi.mocked(runRawInspectionChild).mockImplementation(async (_location, _input, options) => { + controller.abort() + expect(options?.signal?.aborted).toBe(true) + return { status: 'complete', nextCursor: null } + }) + expect(await scan()).toEqual(deferred) + expect(vi.getTimerCount()).toBe(0) +}) + +it('collects only beside the actual database with fixed deployment paths', async () => { + expect(await execute({ type: 'collectArtifacts' })).toBeNull() + expect(runRawCollectionChild).toHaveBeenCalledExactlyOnceWith({ ...location, databasePath: join(directory, 'biometrics.db'), retainedDirectory: join(directory, 'biometric-raw') }, { signal: expect.any(AbortSignal) }) +}) + +it('cancels collection after a failed health recheck and clears supervision', async () => { + vi.useFakeTimers() + let aborted = false + vi.mocked(runRawCollectionChild).mockImplementation(async (_location, options) => new Promise((resolve) => { + options?.signal?.addEventListener('abort', () => { + aborted = true + resolve({ version: 'raw-collection-result-v1', released: 0, orphans: 0 }) + }, { once: true }) + })) + admit.mockResolvedValueOnce(true).mockRejectedValue(new Error('health unavailable')) + const running = execute({ type: 'collectArtifacts' }) + await vi.advanceTimersByTimeAsync(5000) + expect(await running).toBeNull() + expect(aborted).toBe(true) + expect(vi.getTimerCount()).toBe(0) +}) + +it('dispatches a retained evaluation through publication, paged traces and hash-fenced release', async () => { + const start = 1791673200000 + db.insert(schema.sleepRecords).values({ id: 1, side: 'left', enteredBedAt: new Date(start), leftBedAt: new Date(start + 600000), sleepDurationSeconds: 600, timesExitedBed: 0 }).run() + db.insert(schema.vitals).values({ side: 'left', timestamp: new Date(start), heartRate: 65 }).run() + const dataset = new DatasetStore(db).create({ side: 'left', subjectId: 'fixture', sessionIds: [1], referenceImportIds: [], timezone: 'UTC', idempotencyKey: 'evaluation' }) + const fixture = backtestFixture().job + const experiment = await execute({ type: 'createExperiment', input: { datasetId: dataset.id, protocol: fixture.protocol, idempotencyKey: 'experiment' } }) as { id: string } + const workerPath = join(directory, 'worker.cjs') + await build({ entryPoints: ['src/biometrics/backtest/worker-entry.ts'], outfile: workerPath, bundle: true, platform: 'node', target: 'node22', format: 'cjs' }) + const jobs = new BacktestJobStore(db) + const job = jobs.enqueue({ experimentId: experiment.id, evaluatorBuildHash: await workerBuildHash(workerPath), configs: fixture.configs, split: 'tuning', idempotencyKey: 'job' }) + const before = sqlite.prepare('SELECT * FROM vitals').all() + expect(await performCoordinatorTask(db, { type: 'execute', owner: 'test' }, { workerPath, signal: controller.signal, admit })).toEqual({ state: 'succeeded', jobId: job.id }) + const result = await execute({ type: 'readResult', input: { id: job.id } }) as NonNullable> + expect(result.available).toBe(true) + expect(await execute({ type: 'readSummary', input: { id: job.id } })).toMatchObject({ contentHash: result.contentHash, contentHashScope: 'complete_retained_report' }) + const trace = { id: job.id, configuration: 'default' as const, expectedContentHash: result.contentHash } + expect(await execute({ type: 'readTrace', input: trace })).toMatchObject({ provenance: 'versioned_stage_replay', downsampled: false }) + await expect(execute({ type: 'readNumericTrace', input: { ...trace, expectedContentHash: 'f'.repeat(64) } })).rejects.toMatchObject({ code: 'REVISION_CONFLICT' }) + const cohortId = datasetManifestSchema.parse(dataset.manifest).splitProtocol.assignments[0].cohortId + expect(await execute({ type: 'readReferenceTrace', input: { id: job.id, expectedContentHash: result.contentHash, cohortId, metric: 'heart_rate' } })).toMatchObject({ totalItems: 0, items: [], provenance: 'frozen_dataset_reference' }) + await expect(execute({ type: 'releaseResult', input: { id: job.id, expectedContentHash: 'f'.repeat(64) } })).rejects.toMatchObject({ code: 'REVISION_CONFLICT' }) + expect(await execute({ type: 'releaseResult', input: { id: job.id, expectedContentHash: result.contentHash } })).toMatchObject({ available: false, report: null }) + await expect(execute({ type: 'readTrace', input: trace })).rejects.toMatchObject({ code: 'RESULT_RELEASED' }) + expect(sqlite.prepare('SELECT * FROM vitals').all()).toEqual(before) +}) + +it('does not overlap slow health rechecks and waits for a pending denial before committing inspection', async () => { + vi.useFakeTimers() + let resolveHealth!: (allowed: boolean) => void + admit.mockResolvedValueOnce(true).mockImplementation(() => new Promise((resolve) => { + resolveHealth = resolve + })) + let resolveScan!: (result: Awaited>) => void + vi.mocked(runRawInspectionChild).mockImplementation(() => new Promise((resolve) => { + resolveScan = resolve + })) + const running = scan() + await vi.advanceTimersByTimeAsync(15000) + expect(admit).toHaveBeenCalledTimes(2) + resolveScan({ status: 'complete', nextCursor: null }) + resolveHealth(false) + expect(await running).toEqual(deferred) + expect(vi.getTimerCount()).toBe(0) +}) diff --git a/src/biometrics/backtest/tests/coordinator.test.ts b/src/biometrics/backtest/tests/coordinator.test.ts new file mode 100644 index 000000000..58c2c5c96 --- /dev/null +++ b/src/biometrics/backtest/tests/coordinator.test.ts @@ -0,0 +1,288 @@ +import { withBiometricBackgroundSlot } from '../../background-admission' +import * as cleanup from '../../shadow/process-cleanup' +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { execFile } from 'node:child_process' +import { mkdtemp, mkdir, rm, writeFile, readFile, utimes } from 'node:fs/promises' +import { join, resolve } from 'node:path' +import { tmpdir } from 'node:os' +import { promisify } from 'node:util' +import { afterAll, beforeAll, expect, it, vi } from 'vitest' +import { runCoordinatorTask } from '../coordinator' +import { BacktestJobStore } from '../jobs' +import { ShadowStore } from '../../shadow/store' +import { dataHash } from '../../algorithms/canonical' +import { DatasetStore } from '../datasets' +import { workerBuildHash } from '../child' +import { backtestFixture } from './fixtures/backtest' +import * as schema from '@/src/db/biometrics-schema' +let directory: string +let databasePath: string +let coordinatorPath: string +let workerPath: string + +it('computes shadow health and reference agreement in the shipped coordinator without canonical writes', async () => { + const sqlite = new Database(databasePath) + const at = 1700000000000, hash = 'a'.repeat(64) + const store = new ShadowStore(sqlite, { maximumRuns: 10, maximumPoints: 10, maximumBytes: 100000 }, () => at) + const run = store.create({ schema: 'biometric-shadow-plan-v1', side: 'left', subjectId: 'shadow-fixture', startMs: at, endMs: at + 1000, + candidateConfigHash: hash, activeConfigHash: hash, engineBuildHash: hash, qualificationHash: hash, + maximumPoints: 10, maximumBytes: 10000, metrics: ['heart_rate'], + monitoring: { expectedIntervalMs: 1000, maximumGapMs: 2000, heartbeatStaleMs: 2000, minimumCandidateCoverage: 0.8, maximumCoverageLoss: 0.1, minimumQuality: 0.8, + alignment: { version: 'reference-alignment-v1', pointToleranceMs: 0, clockOffsetMs: 0, offsetSource: 'fixed' } } }, 'shadow-report') + const lease = store.claim(run.id, 10000, () => {}) + store.append(lease, { sequence: 0, side: 'left', timestampMs: at, inputHash: null, candidateConfigHash: hash, activeConfigHash: hash, + readings: [{ metric: 'heart_rate', candidate: { value: 62, quality: 1 }, active: { value: 60, quality: 1 } }] }) + store.finish(lease, 'completed', 'complete') + const before = sqlite.prepare('SELECT * FROM vitals').all() + try { + const reference = { side: 'left' as const, subjectId: 'shadow-fixture', source: 'test', rows: [{ recordKey: 'hr', metric: 'heart_rate', value: 63, unit: 'bpm', timestampMs: at }] } + const report = await runCoordinatorTask(options(), { type: 'readShadowReport', input: { id: run.id, reference } }) + expect(report.metrics[0].comparison.differences.commonMae).toBe(-2) + expect(report.metrics[0].comparison.commonSupport.candidate.pairedCount).toBe(1) + expect(report.interpretation).toBe('prospective_observation_and_reference_agreement_not_activation_authorization') + const { reportHash, ...body } = report + expect(reportHash).toBe(dataHash(body)) + expect(sqlite.prepare('SELECT * FROM vitals').all()).toEqual(before) + await expect(runCoordinatorTask(options(), { type: 'readShadowReport', input: { id: run.id, reference: { ...reference, side: 'right' } } })).rejects.toMatchObject({ code: 'SUBJECT_MISMATCH' }) + } + finally { sqlite.close() } +}) +beforeAll(async () => { + directory = await mkdtemp(join(tmpdir(), 'biometric-coordinator-test-')) + await promisify(execFile)(process.execPath, ['scripts/build/biometrics-worker.mjs', directory]) + databasePath = join(directory, 'synthetic.db') + coordinatorPath = join(directory, 'coordinator.cjs') + workerPath = join(directory, 'worker.cjs') + const sqlite = new Database(databasePath) + sqlite.pragma('journal_mode = WAL') + const db = drizzle(sqlite, { schema }) + migrate(db, { migrationsFolder: 'src/db/biometrics-migrations' }) + const start = Date.UTC(2026, 0, 1) + db.insert(schema.sleepRecords).values({ id: 1, side: 'left', enteredBedAt: new Date(start), leftBedAt: new Date(start + 600000), sleepDurationSeconds: 600, timesExitedBed: 0 }).run() + db.insert(schema.vitals).values({ side: 'left', timestamp: new Date(start), heartRate: 60 }).run() + sqlite.close() +}) +afterAll(async () => rm(directory, { recursive: true, force: true })) +const options = () => ({ databasePath, coordinatorPath, workerPath, modulePath: resolve('node_modules') }) +const id = '11111111-1111-4111-8111-111111111111' +it('creates retained inputs off-process and preserves idempotent retries and source observations', async () => { + const input = { side: 'left' as const, subjectId: 'fixture', sessionIds: [1], referenceImportIds: [], timezone: 'UTC', idempotencyKey: 'coordinator-test' } + const first = await runCoordinatorTask(options(), { type: 'createDataset', input }) + const retry = await runCoordinatorTask(options(), { type: 'createDataset', input }) + expect(retry).toEqual(first) + expect(first.available).toBe(true) + const db = new Database(databasePath, { readonly: true }) + expect(db.prepare('SELECT count(*) n FROM vitals').get()).toEqual({ n: 1 }) + expect(db.prepare('SELECT count(*) n FROM biometric_datasets').get()).toEqual({ n: 1 }) + db.close() +}) +it('preserves known error codes without leaking database diagnostics', async () => { + await expect(runCoordinatorTask(options(), { type: 'readTrace', input: { id, configuration: 'default', expectedContentHash: 'a'.repeat(64) } })).rejects.toMatchObject({ code: 'NOT_FOUND' }) + await expect(runCoordinatorTask({ ...options(), databasePath: join(directory, 'missing.db') }, { type: 'readResult', input: { id } })).rejects.toMatchObject({ code: 'COORDINATOR_FAILED', message: 'Biometrics coordinator failed' }) +}) +it('requires fresh parent admission and supports a successful idle execution', async () => { + let checks = 0 + expect(await runCoordinatorTask(options(), { type: 'execute', owner: 'test' })).toEqual({ state: 'deferred' }) + expect(await runCoordinatorTask(options(), { type: 'execute', owner: 'test' }, { admit: async () => { + checks++ + return true + } })).toEqual({ state: 'idle' }) + expect(checks).toBe(1) +}) +it('keeps the calling event loop responsive during synchronous coordinator work', async () => { + const busy = join(directory, 'busy.cjs') + await writeFile(busy, 'process.on(\'message\',()=>{const end=Date.now()+500;while(Date.now()process.disconnect())})') + let ticks = 0 + const timer = setInterval(() => ticks++, 25) + try { + expect(await runCoordinatorTask({ ...options(), coordinatorPath: busy }, { type: 'collectArtifacts' })).toBeNull() + expect(ticks).toBeGreaterThan(4) + } + finally { clearInterval(timer) } +}) +it('bounds a stalled coordinator and permits subsequent work', async () => { + const stalled = join(directory, 'stalled.cjs') + await writeFile(stalled, 'process.on("message",()=>{});setInterval(()=>{},1000)') + await expect(runCoordinatorTask({ ...options(), coordinatorPath: stalled, timeoutMs: 100 }, { type: 'collectArtifacts' })).rejects.toThrow('time budget') + expect(await runCoordinatorTask(options(), { type: 'collectArtifacts' })).toBeNull() +}) +it('allows an explicit longer execution budget without extending read deadlines', async () => { + const fixture = join(directory, 'budget.cjs') + const replayBudget = { jobWallMs: 300000, waveformWallMs: 200000, waveformCpuSeconds: 180 } + await writeFile(fixture, `process.on('message',()=>{process.send({type:'result',result:JSON.parse(process.argv[5])},()=>process.disconnect())})`) + const configured = { ...options(), coordinatorPath: fixture, replayBudget } + expect(await runCoordinatorTask({ ...configured, timeoutMs: 200000 }, { type: 'execute', owner: 'test' })).toEqual(replayBudget) + expect(await runCoordinatorTask(configured, { type: 'collectArtifacts' })).toBeNull() + expect(() => runCoordinatorTask({ ...configured, timeoutMs: 200000 }, { type: 'collectArtifacts' })).toThrow('Invalid coordinator timeout') + expect(() => runCoordinatorTask({ ...configured, timeoutMs: 300001 }, { type: 'execute', owner: 'test' })).toThrow('Invalid coordinator timeout') + for (const changed of [{ jobWallMs: 500 }, { waveformWallMs: 400000 }, { waveformCpuSeconds: 201 }, { jobWallMs: 172800001 }]) { + expect(() => runCoordinatorTask({ ...configured, replayBudget: { ...replayBudget, ...changed } }, { type: 'execute', owner: 'test' })).toThrow() + } +}) +it('enforces the configured job deadline when no test timeout overrides it', async () => { + const stalled = join(directory, 'budget-stalled.cjs') + await writeFile(stalled, 'process.on("message",()=>{});setInterval(()=>{},1000)') + const began = performance.now() + await expect(runCoordinatorTask({ ...options(), coordinatorPath: stalled, + replayBudget: { jobWallMs: 1000, waveformWallMs: 1000, waveformCpuSeconds: 1 } }, + { type: 'execute', owner: 'test' })).rejects.toThrow('time budget') + expect(performance.now() - began).toBeLessThan(3000) +}) +it.skipIf(process.platform === 'win32')('kills evaluator descendants when a coordinator crashes', async () => { + const script = join(directory, 'crash.cjs'), pidPath = join(directory, 'descendant.pid') + await writeFile(script, `const {spawn}=require('node:child_process');const fs=require('node:fs');process.on('message',()=>{const child=spawn(process.execPath,['-e','setInterval(()=>{},1000)'],{stdio:'ignore'});fs.writeFileSync(${JSON.stringify(pidPath)},String(child.pid));process.exit(1)})`) + await expect(runCoordinatorTask({ ...options(), coordinatorPath: script }, { type: 'collectArtifacts' })).rejects.toThrow('complete result') + const pid = Number(await readFile(pidPath, 'utf8')) + let alive = true + for (let attempt = 0; attempt < 20 && alive; attempt++) { + try { + process.kill(pid, 0) + if (process.platform === 'linux' && (await readFile(`/proc/${pid}/stat`, 'utf8')).split(' ')[2] === 'Z') alive = false + } + catch { alive = false } + if (alive) await new Promise(resolve => setTimeout(resolve, 25)) + } + expect(alive).toBe(false) +}) + +it('runs a leased evaluation and preserves report identity across the process boundary', async () => { + const sqlite = new Database(databasePath) + const db = drizzle(sqlite, { schema }) + const dataset = new DatasetStore(db).create({ side: 'left', subjectId: 'fixture', sessionIds: [1], referenceImportIds: [], timezone: 'UTC', idempotencyKey: 'execute-dataset' }) + const jobs = new BacktestJobStore(db) + const fixture = backtestFixture().job + const experiment = jobs.createExperiment(dataset.id, fixture.protocol) + const job = jobs.enqueue({ experimentId: experiment.id, evaluatorBuildHash: await workerBuildHash(workerPath), configs: fixture.configs, split: 'tuning', idempotencyKey: 'execute-job' }) + try { + expect(await runCoordinatorTask(options(), { type: 'execute', owner: 'integration' }, { healthAdmission: async () => ({ allowed: true, sample: { qualificationId: 'integration-policy', checkedAtMs: Date.now(), ingestionAgeMs: 123 } }) })).toEqual({ state: 'succeeded', jobId: job.id }) + const result = await runCoordinatorTask(options(), { type: 'readResult', input: { id: job.id } }) + expect(result?.contentHash).toBe(jobs.result(job.id)?.contentHash) + expect(result?.available).toBe(true) + expect(result?.report).toMatchObject({ jobOperations: { memory: { scope: 'sum_rss_of_replay_process_group', attempts: expect.any(Number), exactKernelPeak: false } } }) + expect(result?.report).toMatchObject({ jobOperations: { ingestion: { allChecksMeasured: true, samples: 2, maximumObservedAgeMs: 123, qualificationId: 'integration-policy' } } }) + const summary = await runCoordinatorTask(options(), { type: 'readSummary', input: { id: job.id } }) + expect(summary).toMatchObject({ contentHash: result?.contentHash, contentHashScope: 'complete_retained_report' }) + expect(JSON.stringify(summary?.report)).not.toContain('"blocks"') + expect(JSON.stringify(result?.report)).toContain('"blocks"') + } + finally { sqlite.close() } +}) + +it('cooperatively interrupts a running lease before shutdown completes', async () => { + const slow = join(directory, 'slow-evaluator.cjs') + await writeFile(slow, 'process.stdin.resume();setInterval(()=>{},1000)') + const sqlite = new Database(databasePath) + const db = drizzle(sqlite, { schema }) + const dataset = new DatasetStore(db).create({ side: 'left', subjectId: 'fixture', sessionIds: [1], referenceImportIds: [], timezone: 'UTC', idempotencyKey: 'cancel-dataset' }) + const jobs = new BacktestJobStore(db), fixture = backtestFixture().job + const experiment = jobs.createExperiment(dataset.id, fixture.protocol) + const job = jobs.enqueue({ experimentId: experiment.id, evaluatorBuildHash: await workerBuildHash(slow), configs: fixture.configs, split: 'tuning', idempotencyKey: 'cancel-job' }) + const controller = new AbortController() + const running = runCoordinatorTask({ ...options(), workerPath: slow }, { type: 'execute', owner: 'cancel-test' }, { signal: controller.signal, admit: async () => true }) + const rejected = expect(running).rejects.toThrow('cancelled') + try { + for (let i = 0; i < 100 && jobs.get(job.id).state !== 'running'; i++) await new Promise(resolve => setTimeout(resolve, 10)) + expect(jobs.get(job.id).state).toBe('running') + controller.abort() + await rejected + expect(jobs.get(job.id)).toMatchObject({ state: 'interrupted', error: 'Worker host stopped' }) + expect(jobs.result(job.id)).toBeNull() + } + finally { + controller.abort() + await running.catch(() => {}) + sqlite.close() + } +}) + +it('inspects actual archives through the isolated coordinator and persists only admitted evidence', async () => { + const archiveDirectory = join(directory, 'archive') + await mkdir(archiveDirectory) + const pythonPath = resolve('modules/piezo-processor/.venv/bin/python') + const rawLocation = { pythonPath, modulesPath: resolve('modules'), archiveDirectory } + await promisify(execFile)(pythonPath, ['-c', 'import gzip,sys;from pathlib import Path;Path(sys.argv[1]).write_bytes(gzip.compress(bytes.fromhex(\'a26373657101646461746148a164747970656178\'),mtime=0))', join(archiveDirectory, 'a.RAW.gz')]) + const scan = () => ({ type: 'inspectRawSource' as const, input: { afterName: null } }) + expect(await runCoordinatorTask({ ...options(), rawLocation }, scan())).toEqual({ status: 'deferred', nextCursor: null }) + let checks = 0 + expect(await runCoordinatorTask({ ...options(), rawLocation }, scan(), { admit: async () => ++checks === 1 })).toEqual({ status: 'deferred', nextCursor: null }) + const sqlite = new Database(databasePath) + try { + expect(sqlite.prepare('SELECT count(*) n FROM biometric_raw_sources').get()).toEqual({ n: 0 }) + const result = await runCoordinatorTask({ ...options(), rawLocation }, scan(), { admit: async () => true }) + expect(result.status).toBe('inspected') + if (result.status !== 'inspected') throw new Error('Missing inspection') + expect(result.source).toMatchObject({ availability: 'requires_reverification', inspection: { source_name: 'a.RAW.gz', retained: false, inventory: { decoded_records: 1 } } }) + expect(sqlite.prepare('SELECT count(*) n FROM biometric_raw_sources').get()).toEqual({ n: 1 }) + expect(sqlite.prepare('SELECT count(*) n FROM biometric_raw_artifacts').get()).toEqual({ n: 0 }) + expect(await runCoordinatorTask(options(), { type: 'pageRawSources', input: { startMs: 0, endMs: 1 } })).toEqual({ items: [], nextCursor: null }) + await writeFile(join(archiveDirectory, 'a.RAW.gz'), 'corrupt') + expect(await runCoordinatorTask({ ...options(), rawLocation }, scan(), { admit: async () => true })).toEqual({ status: 'source_unavailable', nextCursor: 'a.RAW.gz' }) + expect(sqlite.prepare('SELECT unavailable_at_ms unavailable FROM biometric_raw_sources').get()).toEqual({ unavailable: null }) + expect(await runCoordinatorTask({ ...options(), rawLocation }, { type: 'inspectRawSource', input: { afterName: 'a.RAW.gz' } }, { admit: async () => true })).toEqual({ status: 'complete', nextCursor: null }) + } + finally { sqlite.close() } +}) + +it('cancels archive inspection when ongoing resource admission is withdrawn', async () => { + const pythonPath = join(directory, 'slow-inspection') + await writeFile(pythonPath, `#!${process.execPath}\nprocess.stdin.resume();setInterval(()=>{},1000);setTimeout(()=>process.exit(91),10000).unref();`, { mode: 0o700 }) + let checks = 0 + expect(await runCoordinatorTask({ ...options(), rawLocation: { pythonPath, modulesPath: directory, archiveDirectory: directory } }, + { type: 'inspectRawSource', input: { afterName: null } }, { admit: async () => ++checks === 1 })).toEqual({ status: 'deferred', nextCursor: null }) + expect(checks).toBe(2) +}, 10000) + +it('collects aged RAW orphans through the admitted coordinator without touching fresh files', async () => { + const retained = join(directory, 'biometric-raw') + await mkdir(retained, { recursive: true }) + const old = join(retained, `${'a'.repeat(64)}.RAW.gz`), fresh = join(retained, `${'b'.repeat(64)}.RAW.gz`) + await writeFile(old, 'old unpublished copy') + await writeFile(fresh, 'fresh unpublished copy') + await utimes(old, 0, 0) + const rawLocation = { pythonPath: resolve('modules/piezo-processor/.venv/bin/python'), modulesPath: resolve('modules'), archiveDirectory: directory } + await runCoordinatorTask({ ...options(), rawLocation }, { type: 'collectArtifacts' }, { admit: async () => false }) + expect(await readFile(old, 'utf8')).toBe('old unpublished copy') + await runCoordinatorTask({ ...options(), rawLocation }, { type: 'collectArtifacts' }, { admit: async () => true }) + await expect(readFile(old)).rejects.toMatchObject({ code: 'ENOENT' }) + expect(await readFile(fresh, 'utf8')).toBe('fresh unpublished copy') +}) + +it('does not fall back to permissive boolean admission when the health callback denies or fails', async () => { + for (const healthAdmission of [async () => ({ allowed: false, sample: null }), async () => { + throw new Error('missing health') + }]) { + expect(await runCoordinatorTask(options(), { type: 'execute', owner: 'deny-health' }, { admit: async () => true, healthAdmission })).toEqual({ state: 'deferred' }) + } +}) + +it('shares background capacity with shadow attempts while retaining report access', async () => { + let finish!: () => void + const busy = withBiometricBackgroundSlot('work', () => new Promise((resolve) => { + finish = resolve + })) + try { + for (const task of [{ type: 'execute', owner: 'test' }, { type: 'collectArtifacts' }] as const) { + await expect(runCoordinatorTask(options(), task)).rejects.toMatchObject({ code: 'RESOURCE_LIMIT' }) + } + await expect(runCoordinatorTask(options(), { type: 'readResult', input: { id: '00000000-0000-4000-8000-000000000001' } })).rejects.toMatchObject({ code: 'NOT_FOUND' }) + } + finally { + finish() + await busy + } + expect(await runCoordinatorTask(options(), { type: 'execute', owner: 'test' })).toEqual({ state: 'deferred' }) +}) +it('does not release background capacity when coordinator cleanup cannot be confirmed', async () => { + const observation = vi.spyOn(cleanup, 'confirmShadowProcessCleanup').mockResolvedValueOnce(false) + try { + await expect(runCoordinatorTask(options(), { type: 'execute', owner: 'test' })).rejects.toMatchObject({ code: 'BIOMETRIC_COORDINATOR_CLEANUP_UNCONFIRMED' }) + await expect(withBiometricBackgroundSlot('work', async () => null)).rejects.toThrow('cleanup is unconfirmed') + } + finally { + observation.mockRestore() + delete (globalThis as typeof globalThis & { __biometricBackgroundSlotsV1?: unknown }).__biometricBackgroundSlotsV1 + } +}) diff --git a/src/biometrics/backtest/tests/datasets.test.ts b/src/biometrics/backtest/tests/datasets.test.ts new file mode 100644 index 000000000..30f0b4e42 --- /dev/null +++ b/src/biometrics/backtest/tests/datasets.test.ts @@ -0,0 +1,92 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import * as schema from '@/src/db/biometrics-schema' +import { previewReferenceImport } from '../../references/normalize' +import { ReferenceStore } from '../../references/store' +import { DatasetStore } from '../datasets' +let sqlite: Database.Database +let db: ReturnType> +let datasets: DatasetStore +let references: ReferenceStore +const startMs = 1791673200000 +const upload = { side: 'left' as const, subjectId: 'sleeper-A', source: 'fixture', rows: [ + { recordKey: 'hr', metric: 'heart_rate', value: 60, unit: 'bpm', timestampMs: startMs }, +] } +const request = { side: 'left' as const, subjectId: 'sleeper-A', sessionIds: [1], referenceImportIds: [] as string[], timezone: 'America/Los_Angeles', idempotencyKey: 'dataset-1' } +beforeEach(() => { + sqlite = new Database(':memory:') + db = drizzle(sqlite, { schema }) + migrate(db, { migrationsFolder: 'src/db/biometrics-migrations' }) + datasets = new DatasetStore(db) + references = new ReferenceStore(db) + db.insert(schema.sleepRecords).values({ id: 1, side: 'left', enteredBedAt: new Date(startMs), leftBedAt: new Date(startMs + 3600000), sleepDurationSeconds: 3600, timesExitedBed: 0 }).run() + db.insert(schema.vitals).values({ side: 'left', timestamp: new Date(startMs), heartRate: 65 }).run() +}) +afterEach(() => sqlite.close()) + +describe('retained processed datasets', () => { + it('keeps exact input bytes after reference correction/deletion and observation retention', () => { + const first = references.commit({ import: upload, previewHash: previewReferenceImport(upload).summary.contentHash }) + const dataset = datasets.create({ ...request, referenceImportIds: [first.id] }) + const before = datasets.read(dataset.id) + const correction = { ...upload, rows: [{ ...upload.rows[0], value: 90 }] } + references.commit({ import: correction, previewHash: previewReferenceImport(correction).summary.contentHash, supersedesImportId: first.id }) + references.delete(first.id) + db.delete(schema.vitals).run() + db.delete(schema.sleepRecords).run() + const after = datasets.read(dataset.id) + expect(after).toEqual(before) + expect(after.snapshot.sessions[0].vitals[0].heartRate).toBe(65) + expect(after.snapshot.references[0].measurements[0].value).toBe(60) + expect(after.manifest).toMatchObject({ capabilities: { signalReplay: false, stageReplay: true }, splitProtocol: { evidence: 'descriptive_only' } }) + expect(datasets.create({ ...request, referenceImportIds: [first.id] }).id).toBe(dataset.id) + }) + it('rejects cross-side sessions, sleeper mismatches, duplicate revisions and changed idempotency requests', () => { + expect(() => datasets.create({ ...request, side: 'right' })).toThrow('missing or belongs') + const first = references.commit({ import: upload, previewHash: previewReferenceImport(upload).summary.contentHash }) + expect(() => datasets.create({ ...request, subjectId: 'B', referenceImportIds: [first.id] })).toThrow('match') + const correction = { ...upload, rows: [{ ...upload.rows[0], value: 70 }] } + const second = references.commit({ import: correction, previewHash: previewReferenceImport(correction).summary.contentHash, supersedesImportId: first.id }) + expect(() => datasets.create({ ...request, referenceImportIds: [first.id, second.id] })).toThrow('one revision') + datasets.create(request) + expect(() => datasets.create({ ...request, timezone: 'UTC' })).toThrow('different selection') + }) + it('detects corrupted retained bytes and enforces aggregate storage quota atomically', () => { + const dataset = datasets.create(request) + sqlite.prepare('UPDATE biometric_datasets SET snapshot = json_set(snapshot, ?, ?) WHERE id = ?') + .run('$.sessions[0].vitals[0].heartRate', 99, dataset.id) + expect(() => datasets.read(dataset.id)).toThrow('integrity') + sqlite.prepare('UPDATE biometric_datasets SET byte_length = ?').run(64 * 1024 * 1024) + expect(() => datasets.create({ ...request, idempotencyKey: 'second' })).toThrow('storage budget') + expect(datasets.list('left')).toHaveLength(1) + expect(datasets.list('right')).toEqual([]) + }) + it('retains and verifies realistic multi-night rows beyond the configuration hash node budget', () => { + db.delete(schema.vitals).run() + db.delete(schema.sleepRecords).run() + const insert = sqlite.prepare('INSERT INTO vitals (side, timestamp, heart_rate) VALUES (?, ?, ?)') + sqlite.transaction(() => { + for (let night = 1; night <= 3; night++) { + const entered = startMs + night * 86400000 + db.insert(schema.sleepRecords).values({ id: night, side: 'left', enteredBedAt: new Date(entered), leftBedAt: new Date(entered + 43200000), sleepDurationSeconds: 43200, timesExitedBed: 0 }).run() + for (let minute = 0; minute < 720; minute++) insert.run('left', entered / 1000 + minute * 60, 60 + (minute % 10) / 10) + } + })() + const dataset = datasets.create({ ...request, sessionIds: [1, 2, 3] }) + expect(datasets.read(dataset.id).snapshot.sessions.reduce((total, session) => total + session.vitals.length, 0)).toBe(2160) + expect(datasets.descriptor(dataset.id).contentHash).toBe(dataset.contentHash) + }) + + it('records exact processing bundles when available and keeps missing provenance explicit', () => { + db.insert(schema.biometricProcessingRuns).values({ id: 'run-1', side: 'left', kind: 'live', bundle: { version: 'fixture' }, + configHash: 'fixture-hash', inputIdentity: { origin: 'fixture' }, startedAtMs: startMs }).run() + sqlite.prepare('UPDATE vitals SET processing_run_id = ?').run('run-1') + const frozen = datasets.read(datasets.create(request).id) + expect(frozen.snapshot.processingRuns[0]).toMatchObject({ id: 'run-1', calibrationHash: null, bundle: { version: 'fixture' } }) + expect(frozen.snapshot.sessions[0].vitals[0].processingRunId).toBe('run-1') + expect(frozen.manifest).toMatchObject({ coverage: [{ vitals: { count: 1, gapDurationMs: 3540000 } }] }) + }) +}) diff --git a/src/biometrics/backtest/tests/discovery.test.ts b/src/biometrics/backtest/tests/discovery.test.ts new file mode 100644 index 000000000..77d00f41a --- /dev/null +++ b/src/biometrics/backtest/tests/discovery.test.ts @@ -0,0 +1,61 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { afterEach, beforeEach, expect, it } from 'vitest' +import * as schema from '@/src/db/biometrics-schema' +import { listReplaySessions, sessionCapabilities } from '../discovery' +let sqlite: Database.Database +let db: ReturnType> +const start = 1791673200000 +function session(id = 1, hours = 1, side: 'left' | 'right' = 'left') { + db.insert(schema.sleepRecords).values({ id, side, enteredBedAt: new Date(start), leftBedAt: new Date(start + hours * 3600000), sleepDurationSeconds: Math.max(0, hours * 3600), timesExitedBed: 0 }).run() +} +beforeEach(() => { + sqlite = new Database(':memory:') + db = drizzle(sqlite, { schema }) + migrate(db, { migrationsFolder: 'src/db/biometrics-migrations' }) +}) +afterEach(() => sqlite.close()) +it('pages equal start times without duplicates and keeps sides separate', () => { + for (let id = 1; id <= 22; id++) session(id) + session(23, 1, 'right') + const first = listReplaySessions(db, 'left') + expect(first.items.map(row => row.id)).toEqual(Array.from({ length: 20 }, (_, i) => 22 - i)) + expect(first.nextCursor).toEqual({ startMs: start, id: 3 }) + if (!first.nextCursor) throw new Error('Expected another page') + const next = listReplaySessions(db, 'left', first.nextCursor) + expect(next.items.map(row => row.id)).toEqual([2, 1]) + expect(next.nextCursor).toBeNull() + expect(listReplaySessions(db, 'right').items.map(row => row.id)).toEqual([23]) +}) +it('reports an empty session without implying retained waveforms or stage support', () => { + session() + expect(sessionCapabilities(db, 'left', 1)).toMatchObject({ comparison: false, stageReplay: false, signalReplay: false, + processed: { legacyUnknownProvenanceCount: 0, vitals: { count: 0 }, movement: { count: 0 } }, referenceMetrics: [], + limitations: expect.arrayContaining(['No stored vitals are available for comparison or stage replay.']) }) + expect(() => sessionCapabilities(db, 'right', 1)).toThrow('not found for this side') +}) +it.each([0, -1, 37])('rejects invalid or over-budget session duration %s hours', (hours) => { + session(1, hours) + expect(() => sessionCapabilities(db, 'left', 1)).toThrow('boundaries exceed') +}) +it.each([24, 25])('distinguishes comparison from the one-day stage replay limit at %s hours', (hours) => { + session(1, hours) + db.insert(schema.biometricProcessingRuns).values({ id: 'known', side: 'left', kind: 'live', bundle: {}, configHash: 'fixture', inputIdentity: {}, startedAtMs: start }).run() + db.insert(schema.vitals).values([ + { side: 'left', timestamp: new Date(start), heartRate: 60, processingRunId: 'known' }, + { side: 'left', timestamp: new Date(start + 60000), heartRate: 61 }, + { side: 'right', timestamp: new Date(start), heartRate: 70 }, + { side: 'left', timestamp: new Date(start + hours * 3600000), heartRate: 80 }, + ]).run() + const result = sessionCapabilities(db, 'left', 1) + expect(result).toMatchObject({ comparison: true, stageReplay: hours === 24, processed: { legacyUnknownProvenanceCount: 1, vitals: { count: 2 } } }) + expect(result.limitations.includes('Stage replay currently requires a merged window of at most one day.')).toBe(hours > 24) +}) +it('bounds aggregate processed rows before returning an incomplete inventory', () => { + session(1, 6) + sqlite.exec(`WITH RECURSIVE rows(n) AS (SELECT 1 UNION ALL SELECT n+1 FROM rows WHERE n < 20001) + INSERT INTO vitals(side,timestamp,heart_rate) SELECT 'left', ${start / 1000} + n, 60 FROM rows`) + expect(() => sessionCapabilities(db, 'left', 1)).toThrow('processed-data row budget') +}) diff --git a/src/biometrics/backtest/tests/evaluator.test.ts b/src/biometrics/backtest/tests/evaluator.test.ts new file mode 100644 index 000000000..272e5a16a --- /dev/null +++ b/src/biometrics/backtest/tests/evaluator.test.ts @@ -0,0 +1,63 @@ +// @vitest-environment node +import { describe, expect, it } from 'vitest' +import { configHash } from '../../algorithms/canonical' +import { classifySleepStages } from '@/src/lib/sleep-stages' +import { evaluateBacktest } from '../evaluator' + +import { backtestFixture as fixture } from './fixtures/backtest' +const start = 1791673200000 + +describe('retained-data evaluator', () => { + it('deduplicates overlapping sessions, preserves shared classifier behavior and does not mutate input', () => { + const input = fixture(), before = JSON.stringify(input) + const report = evaluateBacktest(input) + expect(JSON.stringify(input)).toBe(before) + expect(report.nights[0].uniqueObservationCount).toBe(10) + const windows = report.nights[0].evaluations[0].windows + expect(windows).toEqual(report.nights[0].evaluations[1].windows) + const expected = classifySleepStages(input.dataset.snapshot.sessions[0].vitals.map(row => ({ ...row, timestamp: new Date(row.timestampMs) })), [], 0) + expect(windows[0].blocks).toEqual(expected.map(epoch => ({ start: epoch.start, end: Math.min(start + 600000, epoch.start + epoch.duration), stage: epoch.stage }))) + expect(windows[0].stage).toMatchObject({ pairedDurationMs: 600000, agreement: 1 }) + expect(windows[0].numeric[0]).toMatchObject({ metrics: { mae: 2 }, pairedCount: 1 }) + expect(report.status).toBe('insufficient_evidence') + expect(report.stageNightSummary).toMatchObject([{ totalNights: 1, scoredNights: 1, equalNightMeanAgreementDifference: 0 }]) + expect(report.nights[0].comparisons).toMatchObject([{ baseline: 'default', candidate: 'current', windows: [{ commonSupportScope: 'all_configurations', coverageReduced: false, differences: { commonAgreement: 0 } }] }]) + }) + it('rejects changed retained inputs and engine snapshots before evaluation', () => { + const input = fixture() + input.dataset.snapshot.sessions[0].vitals[0].heartRate = 100 + expect(() => evaluateBacktest(input)).toThrow('integrity mismatch') + const wrongEngine = fixture() + wrongEngine.job.configs[0].manifestHash = '0'.repeat(64) + expect(() => evaluateBacktest(wrongEngine)).toThrow('engine/configuration mismatch') + }) + it('honors fixed offsets for stage duration scoring without relabeling the original trace', () => { + const input = fixture() + input.job.protocol.clockOffsetMs = 60000 + input.job.protocolHash = configHash(input.job.protocol) + const report = evaluateBacktest(input) + expect(report.nights[0].evaluations[0].windows[0].stage?.pairedDurationMs).toBe(540000) + expect(report.nights[0].comparisons[0].windows[0].commonSupport.baseline.pairedDurationMs).toBe(540000) + expect(report.nights[0].comparisons[0].windows[0].commonSupport.candidate.pairedDurationMs).toBe(540000) + expect(report.nights[0].evaluations[0].windows[0].blocks[0].start).toBe(start) + }) + it('uses the frozen uncertainty policy while retaining insufficient-evidence status', () => { + const input = fixture() + const protocol = { ...input.job.protocol, nightUncertainty: { version: 'paired-night-percentile-v1' as const, + seed: 'retained-policy', resamples: 200, minimumNights: 3 } } + const report = evaluateBacktest({ ...input, job: { ...input.job, protocol, protocolHash: configHash(protocol) } }) + expect(report.stageNightSummary[0].uncertainty).toMatchObject({ policy: protocol.nightUncertainty, + status: 'insufficient_nights', scoredNights: 1, interval: null }) + expect(report.status).toBe('insufficient_evidence') + }) + it('applies the shared three-configuration support to every reported comparison', () => { + const input = fixture() + const candidate = { ...input.job.configs[0], label: 'candidate' as const } + const report = evaluateBacktest({ ...input, job: { ...input.job, configs: [...input.job.configs, candidate] } }) + expect(report.nights[0].comparisons).toHaveLength(3) + for (const comparison of report.nights[0].comparisons) { + expect(comparison.windows[0]).toMatchObject({ commonSupportScope: 'all_configurations', + commonSupport: { baseline: { pairedDurationMs: 600000 }, candidate: { pairedDurationMs: 600000 } } }) + } + }) +}) diff --git a/src/biometrics/backtest/tests/fixtures/backtest.ts b/src/biometrics/backtest/tests/fixtures/backtest.ts new file mode 100644 index 000000000..6bf9ae9e0 --- /dev/null +++ b/src/biometrics/backtest/tests/fixtures/backtest.ts @@ -0,0 +1,21 @@ +import { configHash } from '../../../algorithms/canonical' +import { defaultConfig, manifestHash } from '../../../algorithms/registry' +import { previewReferenceImport } from '../../../references/normalize' +const start = 1791673200000 +const fixture = () => { + const session = { id: 1, startMs: start, endMs: start + 600000, sleepDurationSeconds: 600, timesExitedBed: 0, presentIntervals: null, notPresentIntervals: null, + vitals: Array.from({ length: 10 }, (_, id) => ({ id, timestampMs: start + id * 60000, heartRate: 60, hrv: 30, breathingRate: 14, processingRunId: null })), movement: [] } + const dataset = { manifest: { splitProtocol: { assignments: [{ cohortId: 'night', sessionIds: [1, 2], split: 'tuning', startMs: start, endMs: start + 600000 }] } }, + snapshot: { version: 'processed-dataset-v1' as const, side: 'left' as const, subjectId: 'A', sessions: [session, { ...session, id: 2 }], processingRuns: [], references: [{ + importId: 'ref', revision: 1, contentHash: 'fixture', source: 'fixture', timePolicy: {}, measurements: previewReferenceImport({ side: 'left', subjectId: 'A', source: 'fixture', rows: [ + { recordKey: 'stage', metric: 'sleep_stages', value: 'light', unit: 'stage', startMs: start, endMs: start + 600000 }, + { recordKey: 'hr', metric: 'heart_rate', value: 62, unit: 'bpm', timestampMs: start }, + ] }).measurements, + }] } } + const protocol = { version: 'descriptive-evaluation-v1' as const, mode: 'stage_replay' as const, assumedStageCalibrationQuality: 0, pointToleranceMs: 30000, clockOffsetMs: 0 } + return { dataset, job: { version: 'backtest-job-v1' as const, experimentId: '11111111-1111-4111-8111-111111111111', datasetId: '22222222-2222-4222-8222-222222222222', + idempotencyKey: 'fixture', evaluatorBuildHash: 'f'.repeat(64), split: 'tuning' as const, configs: ['default', 'current'].map(label => ({ label: label as 'default' | 'current', revisionId: null, resolved: defaultConfig, resolvedHash: configHash(defaultConfig), manifestHash })), + datasetHash: configHash(dataset), protocol, protocolHash: configHash(protocol), holdoutUse: 0, evidence: 'descriptive_only' as const } } +} +export { fixture as backtestFixture } + diff --git a/src/biometrics/backtest/tests/fixtures/stage-objective.ts b/src/biometrics/backtest/tests/fixtures/stage-objective.ts new file mode 100644 index 000000000..10b0df5e0 --- /dev/null +++ b/src/biometrics/backtest/tests/fixtures/stage-objective.ts @@ -0,0 +1,6 @@ +import type { StageObjectivePolicy } from '../../stage-objective' +export const stageObjectiveTestPolicy: StageObjectivePolicy = { version: 'stage-objective-v1', qualificationId: 'test-only', minimumNights: 3, + minimumPairedDurationMsPerNight: 800, minimumOutputCoverage: 0.9, maximumLostOutputFraction: 0.1, + minimumAgreementImprovement: 0.125, maximumNightAgreementRegression: 0.125, maximumRegressedNightFraction: 0.34, + maximumMacroF1Regression: 0.125, maximumStageF1Regression: 0.125, minimumStageReferenceMsPerNight: 200, + requiredStages: ['wake', 'light', 'deep', 'rem'], operational: { maximumWallMs: 1000, maximumPeakRssBytes: 1000, maximumIngestionAgeMs: 1000 } } diff --git a/src/biometrics/backtest/tests/host-startup.test.ts b/src/biometrics/backtest/tests/host-startup.test.ts new file mode 100644 index 000000000..fc15ff775 --- /dev/null +++ b/src/biometrics/backtest/tests/host-startup.test.ts @@ -0,0 +1,25 @@ +// @vitest-environment node +import { afterEach, expect, it, vi } from 'vitest' +import { startConfiguredBacktestHost } from '../host-startup' +import { startBacktestHost } from '../host' +vi.mock('../host', () => ({ startBacktestHost: vi.fn(() => ({ started: true })) })) +const db = {} as Parameters[0] +const policy = { qualificationId: 'test-only', minimumAvailableMemoryBytes: 100, minimumFreeDiskBytes: 100, + maximumLoadPerCpu: 1, maximumEventLoopDelayMs: 100, maximumPiezoAgeMs: 10000, maximumTemperatureC: 65, maximumSampleAgeMs: 1000 } +afterEach(() => { + vi.unstubAllEnvs() + vi.clearAllMocks() +}) +it.each([undefined, '', '{}', '{', JSON.stringify({ ...policy, maximumLoadPerCpu: 0 })])('does not start background work without valid deployment policy: %s', (value) => { + vi.stubEnv('SLEEPYPOD_BIOMETRICS_ADMISSION_POLICY', value) + expect(startConfiguredBacktestHost(db)).toBeNull() + expect(startBacktestHost).not.toHaveBeenCalled() +}) +it('starts with a fresh admission closure and denies after policy removal', async () => { + vi.stubEnv('SLEEPYPOD_BIOMETRICS_ADMISSION_POLICY', JSON.stringify(policy)) + expect(startConfiguredBacktestHost(db)).toEqual({ started: true }) + expect(startBacktestHost).toHaveBeenCalledWith(db, expect.any(Function), expect.any(Function)) + const admit = vi.mocked(startBacktestHost).mock.calls[0][1] + vi.stubEnv('SLEEPYPOD_BIOMETRICS_ADMISSION_POLICY', undefined) + expect(await admit()).toBe(false) +}) diff --git a/src/biometrics/backtest/tests/host.test.ts b/src/biometrics/backtest/tests/host.test.ts new file mode 100644 index 000000000..a70409823 --- /dev/null +++ b/src/biometrics/backtest/tests/host.test.ts @@ -0,0 +1,122 @@ +// @vitest-environment node +import { afterEach, expect, it, vi } from 'vitest' +import { BacktestHost, startBacktestHost, stopBacktestHost } from '../host' +import { coordinatorRequest } from '../coordinator-service' +vi.mock('../coordinator-service', () => ({ coordinatorRequest: vi.fn() })) +afterEach(async () => { + await stopBacktestHost() + vi.restoreAllMocks() + vi.useRealTimers() +}) +it('never overlaps attempts and awaits aborted execution before shutdown completes', async () => { + vi.useFakeTimers() + let release: (() => void) | undefined + let observedSignal: AbortSignal | undefined + const execute = vi.fn((signal: AbortSignal) => { + observedSignal = signal + return new Promise((resolve) => { + release = resolve + }) + }) + const host = new BacktestHost({ execute, collect: vi.fn() }) + host.start() + await vi.advanceTimersByTimeAsync(0) + host.start() + await vi.advanceTimersByTimeAsync(60000) + expect(execute).toHaveBeenCalledTimes(1) + let finished = false + const stopping = host.stop().then(() => { + finished = true + }) + expect(observedSignal?.aborted).toBe(true) + await Promise.resolve() + expect(finished).toBe(false) + release?.() + await stopping + expect(finished).toBe(true) + await vi.advanceTimersByTimeAsync(60000) + expect(execute).toHaveBeenCalledTimes(1) + expect(vi.getTimerCount()).toBe(0) +}) +it('recovers from execution and cleanup errors without spinning or abandoning later work', async () => { + vi.useFakeTimers() + const execute = vi.fn().mockRejectedValueOnce(new Error('temporary')).mockResolvedValue({ state: 'idle' }) + const collect = vi.fn().mockRejectedValueOnce(new Error('temporary')).mockResolvedValue(undefined) + const onError = vi.fn() + const host = new BacktestHost({ execute, collect, onError }) + host.start() + await vi.advanceTimersByTimeAsync(0) + expect(onError.mock.calls).toEqual([['cleanup'], ['execution']]) + await vi.advanceTimersByTimeAsync(4999) + expect(execute).toHaveBeenCalledTimes(1) + await vi.advanceTimersByTimeAsync(1) + expect(execute).toHaveBeenCalledTimes(2) + expect(collect).toHaveBeenCalledTimes(1) + await vi.advanceTimersByTimeAsync(1800000) + expect(collect).toHaveBeenCalledTimes(2) + await host.stop() +}) +it('does not begin a job when shutdown arrives during cleanup', async () => { + vi.useFakeTimers() + let release: (() => void) | undefined + const execute = vi.fn() + const host = new BacktestHost({ execute, collect: () => new Promise((resolve) => { + release = resolve + }) }) + host.start() + const stopping = host.stop() + release?.() + await stopping + expect(execute).not.toHaveBeenCalled() + expect(vi.getTimerCount()).toBe(0) +}) +it('aborts the active cleanup on shutdown and waits for its acknowledgement', async () => { + vi.useFakeTimers() + const execute = vi.fn() + let aborted = false + const host = new BacktestHost({ execute, collect: signal => new Promise((resolve) => { + signal.addEventListener('abort', () => { + aborted = true + resolve() + }, { once: true }) + }) }) + host.start() + await host.stop() + expect(aborted).toBe(true) + expect(execute).not.toHaveBeenCalled() + expect(vi.getTimerCount()).toBe(0) +}) + +it('shares one background host and forwards health admission to cleanup and execution', async () => { + vi.useFakeTimers() + vi.mocked(coordinatorRequest).mockResolvedValue(null) + const db = {} as Parameters[0] + const admit = vi.fn().mockResolvedValue(false) + const healthAdmission = vi.fn().mockResolvedValue({ allowed: false, sample: null }) + const first = startBacktestHost(db, admit, healthAdmission) + expect(startBacktestHost(db, admit, healthAdmission)).toBe(first) + await vi.advanceTimersByTimeAsync(0) + expect(coordinatorRequest).toHaveBeenCalledTimes(2) + expect(coordinatorRequest).toHaveBeenNthCalledWith(1, db, { type: 'collectArtifacts' }, { signal: expect.any(AbortSignal), admit, healthAdmission }) + expect(coordinatorRequest).toHaveBeenNthCalledWith(2, db, { type: 'execute', owner: expect.stringMatching(/^core:/) }, { signal: expect.any(AbortSignal), admit, healthAdmission }) + const signal = vi.mocked(coordinatorRequest).mock.calls[0][2]?.signal + await stopBacktestHost() + expect(signal?.aborted).toBe(true) + expect(vi.getTimerCount()).toBe(0) + await stopBacktestHost() + expect(startBacktestHost(db, admit, healthAdmission)).not.toBe(first) +}) + +it('reports coordinator failures without exporting their private diagnostic content', async () => { + vi.useFakeTimers() + vi.mocked(coordinatorRequest).mockReset().mockRejectedValue(new Error('private-data')) + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + startBacktestHost({} as Parameters[0], async () => false) + await vi.advanceTimersByTimeAsync(0) + expect(warn.mock.calls).toEqual([ + ['[biometrics] Backtest cleanup failed; host will retry'], + ['[biometrics] Backtest execution failed; host will retry'], + ]) + await stopBacktestHost() + expect(vi.getTimerCount()).toBe(0) +}) diff --git a/src/biometrics/backtest/tests/job-operations.test.ts b/src/biometrics/backtest/tests/job-operations.test.ts new file mode 100644 index 000000000..851ca4d07 --- /dev/null +++ b/src/biometrics/backtest/tests/job-operations.test.ts @@ -0,0 +1,31 @@ +// @vitest-environment node +import { expect, it } from 'vitest' +import { createJobOperations } from '../job-operations' +const receipt = { allowed: true, sample: { qualificationId: 'qualified-build', checkedAtMs: 100000, ingestionAgeMs: 100 } } +it('retains bounded sampled ingestion evidence with monotonic duration and gaps', () => { + let time = 0 + const operations = createJobOperations(() => time) + operations.record(receipt) + time = 7000 + operations.record({ ...receipt, sample: { ...receipt.sample, ingestionAgeMs: 300 } }) + time = 7200 + expect(operations.finish()).toMatchObject({ wallMs: 7200, ingestion: { checks: 2, samples: 2, allChecksMeasured: true, + maximumObservedAgeMs: 300, maximumObservationGapMs: 7000, continuousMeasurement: false, qualificationId: 'qualified-build' } }) +}) +it('exposes missing, denied, and changing qualification evidence without inventing zero age', () => { + const empty = createJobOperations(() => 0) + empty.record(null) + expect(empty.finish().ingestion).toMatchObject({ allChecksMeasured: false, samples: 0, maximumObservedAgeMs: null, qualificationId: null }) + for (const bad of [null, { ...receipt, allowed: false }, { ...receipt, sample: { ...receipt.sample, ingestionAgeMs: -1 } }, + { ...receipt, sample: { ...receipt.sample, qualificationId: 'different' } }]) { + const operations = createJobOperations(() => 0) + operations.record(receipt) + operations.record(bad) + expect(operations.finish().ingestion).toMatchObject({ allChecksMeasured: false, qualificationId: null }) + } +}) +it('a single observation does not establish coverage over execution', () => { + const operations = createJobOperations(() => 0) + operations.record(receipt) + expect(operations.finish().ingestion.allChecksMeasured).toBe(false) +}) diff --git a/src/biometrics/backtest/tests/jobs.test.ts b/src/biometrics/backtest/tests/jobs.test.ts new file mode 100644 index 000000000..4dc1d1a0c --- /dev/null +++ b/src/biometrics/backtest/tests/jobs.test.ts @@ -0,0 +1,138 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import * as schema from '@/src/db/biometrics-schema' +import { configHash } from '../../algorithms/canonical' +import { defaultConfig, manifestHash } from '../../algorithms/registry' +import { DatasetStore } from '../datasets' +import { BacktestJobStore } from '../jobs' +let sqlite: Database.Database +let store: BacktestJobStore +let datasets: DatasetStore +let now: number +let datasetId: string +let experimentId: string +const config = { label: 'default' as const, revisionId: null, resolved: defaultConfig, resolvedHash: configHash(defaultConfig), manifestHash } +const protocol = { version: 'descriptive-evaluation-v1' as const, mode: 'stage_replay' as const, assumedStageCalibrationQuality: 0, pointToleranceMs: 30000, clockOffsetMs: 0 } +const enqueue = (idempotencyKey = 'job-1', split: 'tuning' | 'validation' | 'holdout' = 'tuning') => store.enqueue({ experimentId, evaluatorBuildHash: 'f'.repeat(64), idempotencyKey, split, configs: [config] }) +const lease = (job: NonNullable>) => ({ id: job.id, owner: job.leaseOwner as string, token: job.fencingToken }) +beforeEach(() => { + now = 1791673200000 + sqlite = new Database(':memory:') + const db = drizzle(sqlite, { schema }) + migrate(db, { migrationsFolder: 'src/db/biometrics-migrations' }) + for (let id = 1; id <= 3; id++) db.insert(schema.sleepRecords).values({ id, side: 'left', enteredBedAt: new Date(now + id * 86400000), + leftBedAt: new Date(now + id * 86400000 + 3600000), sleepDurationSeconds: 3600, timesExitedBed: 0 }).run() + datasets = new DatasetStore(db, () => now) + datasetId = datasets.create({ side: 'left', subjectId: 'A', sessionIds: [1, 2, 3], referenceImportIds: [], timezone: 'UTC', idempotencyKey: 'dataset' }).id + store = new BacktestJobStore(db, () => now) + experimentId = store.createExperiment(datasetId, protocol).id +}) +afterEach(() => sqlite.close()) + +describe('durable backtest leases and results', () => { + it('deduplicates requests, permits only one active worker, and atomically publishes the winning attempt', () => { + const job = enqueue() + expect(enqueue().id).toBe(job.id) + expect(() => enqueue('job-1', 'validation')).toThrow('changed') + const claimed = store.claim('worker-A') + expect(claimed).toMatchObject({ id: job.id, attempt: 1, fencingToken: 1 }) + expect(store.claim('worker-B')).toBeNull() + if (!claimed) throw new Error('Missing claim') + const token = lease(claimed) + store.heartbeat(token, { phase: 'evaluating', completedNights: 0, totalNights: 1 }) + store.complete(token, { status: 'insufficient_evidence' }) + expect(store.get(job.id).state).toBe('succeeded') + expect(sqlite.prepare('SELECT count(*) AS n FROM biometric_backtest_results').get()).toEqual({ n: 1 }) + expect(() => store.complete(token, {})).toThrow('no longer authoritative') + }) + it('releases inline report bytes idempotently without erasing holdout exposure or job identity', () => { + const job = enqueue('release-report', 'holdout') + expect(() => store.releaseResult(job.id, '0'.repeat(64))).toThrow('Only completed') + const claimed = store.claim('worker-A') + if (!claimed) throw new Error('Missing claim') + const published = store.complete(lease(claimed), { traces: ['retained until release'] }) + expect(() => store.releaseResult(job.id, '0'.repeat(64))).toThrow('hash changed') + expect(store.result(job.id)?.available).toBe(true) + const released = store.releaseResult(job.id, published.contentHash) + expect(released).toMatchObject({ available: false, report: null, releasedAtMs: now, + contentHash: published.contentHash, fencingToken: claimed.fencingToken, currentHoldoutUseCount: 1 }) + now += 1000 + expect(store.releaseResult(job.id, published.contentHash)).toEqual(released) + expect(store.result(job.id)).toEqual(released) + expect(enqueue('release-report', 'holdout').id).toBe(job.id) + expect(store.get(job.id).state).toBe('succeeded') + expect(sqlite.prepare('SELECT report FROM biometric_backtest_results').get()).toEqual({ report: '{}' }) + expect(sqlite.prepare('SELECT count(*) AS n FROM biometric_holdout_accesses').get()).toEqual({ n: 1 }) + }) + it('fences expired workers and retries crashes only within the attempt limit', () => { + const job = enqueue() + const first = store.claim('worker-A', 1000) + if (!first) throw new Error('Missing claim') + now += 1000 + expect(() => store.complete(lease(first), {})).toThrow('no longer authoritative') + const second = store.claim('worker-B', 1000) + expect(second).toMatchObject({ attempt: 2, fencingToken: 2 }) + expect(() => store.heartbeat(lease(first), { phase: 'loading', completedNights: 0, totalNights: 1 })).toThrow('no longer authoritative') + now += 1000 + expect(store.claim('worker-C', 1000)).toMatchObject({ attempt: 3 }) + now += 1000 + expect(store.claim('worker-D')).toBeNull() + expect(store.get(job.id)).toMatchObject({ state: 'failed', error: 'Worker lease expired' }) + }) + it('cancels queued jobs immediately and rejects publication after running cancellation', () => { + const queued = enqueue() + expect(store.cancel(queued.id).state).toBe('cancelled') + expect(store.claim('worker')).toBeNull() + const running = enqueue('second') + const claimed = store.claim('worker') + if (!claimed) throw new Error('Missing claim') + expect(store.cancel(running.id).state).toBe('cancel_requested') + expect(store.heartbeat(lease(claimed), { phase: 'loading', completedNights: 0, totalNights: 1 })).toEqual({ cancelRequested: true }) + expect(() => store.complete(lease(claimed), {})).toThrow('cannot publish') + store.acknowledgeCancellation(lease(claimed)) + expect(store.get(running.id).state).toBe('cancelled') + expect(sqlite.prepare('SELECT count(*) AS n FROM biometric_backtest_results').get()).toEqual({ n: 0 }) + }) + it('recovers abandoned cancellations without rerunning and rolls back failed result inserts', () => { + const job = enqueue() + const claimed = store.claim('worker', 1000) + if (!claimed) throw new Error('Missing claim') + sqlite.exec('CREATE TRIGGER reject_result BEFORE INSERT ON biometric_backtest_results BEGIN SELECT RAISE(ABORT, \'fixture failure\'); END;') + expect(() => store.complete(lease(claimed), {})).toThrow('fixture failure') + expect(store.get(job.id).state).toBe('running') + store.cancel(job.id) + now += 1000 + expect(store.claim('replacement')).toBeNull() + expect(store.get(job.id).state).toBe('cancelled') + }) + it('records holdout reuse across experiments and keeps duplicate enqueue from consuming it again', () => { + const first = enqueue('holdout', 'holdout') + expect(first.input).toMatchObject({ holdoutUse: 1, evidence: 'descriptive_only' }) + enqueue('holdout', 'holdout') + expect(store.holdoutUseCount(datasetId)).toBe(1) + experimentId = store.createExperiment(datasetId, protocol).id + expect(enqueue('holdout-again', 'holdout').input).toMatchObject({ holdoutUse: 2, evidence: 'reused_holdout_is_validation' }) + expect(store.holdoutUseCount(datasetId)).toBe(2) + const clone = datasets.create({ side: 'left', subjectId: 'different-asserted-name', sessionIds: [1, 2, 3], referenceImportIds: [], timezone: 'UTC', idempotencyKey: 'clone' }) + experimentId = store.createExperiment(clone.id, protocol).id + expect(enqueue('cloned-holdout', 'holdout').input).toMatchObject({ holdoutUse: 3, evidence: 'reused_holdout_is_validation' }) + expect(store.holdoutUseCount(datasetId)).toBe(3) + }) + it('rejects forged engine hashes and invalid progress without changing the lease', () => { + expect(() => store.enqueue({ experimentId, evaluatorBuildHash: 'f'.repeat(64), idempotencyKey: 'bad', split: 'tuning', configs: [{ ...config, resolvedHash: '0'.repeat(64) }] })).toThrow('does not match') + enqueue() + const claimed = store.claim('worker') + if (!claimed) throw new Error('Missing claim') + expect(() => store.heartbeat(lease(claimed), { phase: 'evaluating', completedNights: 2, totalNights: 1 })).toThrow('monotonic') + expect(store.get(claimed.id).progress).toMatchObject({ completedNights: 0 }) + store.fail(lease(claimed), 'bounded failure') + expect(store.get(claimed.id).state).toBe('failed') + }) +}) + +it('refuses signal experiments without immutable waveform inputs', () => { + expect(() => store.createExperiment(datasetId, { ...protocol, mode: 'signal_replay' })).toThrow('retained waveform inputs') +}) diff --git a/src/biometrics/backtest/tests/location.test.ts b/src/biometrics/backtest/tests/location.test.ts new file mode 100644 index 000000000..c13d18ebf --- /dev/null +++ b/src/biometrics/backtest/tests/location.test.ts @@ -0,0 +1,25 @@ +// @vitest-environment node +import { resolve } from 'node:path' +import { beforeEach, expect, it, vi } from 'vitest' +import { existsSync } from 'node:fs' +import { packagedWorkerPath, packagedCoordinatorPath, packagedShadowHostPath } from '../location' + +vi.mock('node:fs', () => ({ existsSync: vi.fn() })) +beforeEach(() => vi.mocked(existsSync).mockReset()) + +it.each([ + ['worker', packagedWorkerPath], + ['coordinator', packagedCoordinatorPath], + ['shadow-host', packagedShadowHostPath], +] as const)('resolves only fixed %s packages, preferring the installed bundle', (name, locate) => { + const installed = resolve(`biometrics-worker/${name}.cjs`) + const development = resolve(`.next/standalone/biometrics-worker/${name}.cjs`) + vi.mocked(existsSync).mockReturnValue(true) + expect(locate()).toBe(installed) + expect(existsSync).toHaveBeenCalledExactlyOnceWith(installed) + vi.mocked(existsSync).mockReset().mockImplementation(path => path === development) + expect(locate()).toBe(development) + expect(vi.mocked(existsSync).mock.calls).toEqual([[installed], [development]]) + vi.mocked(existsSync).mockReset().mockReturnValue(false) + expect(locate).toThrow(expect.objectContaining({ code: 'WORKER_UNAVAILABLE' })) +}) diff --git a/src/biometrics/backtest/tests/night-summary.test.ts b/src/biometrics/backtest/tests/night-summary.test.ts new file mode 100644 index 000000000..9de654059 --- /dev/null +++ b/src/biometrics/backtest/tests/night-summary.test.ts @@ -0,0 +1,56 @@ +// @vitest-environment node +import { expect, it } from 'vitest' +import { compareStageOutputs } from '../stage-comparison' +import { summarizeStageNights } from '../night-summary' +import { previewReferenceImport } from '../../references/normalize' +const startMs = 1791673200000 +function night(cohortId: string, duration: number, improved: boolean, missing = false) { + const endMs = startMs + duration + const references = missing + ? [] + : previewReferenceImport({ side: 'left', subjectId: 'fixture', source: 'test', rows: [ + { recordKey: 'stage', metric: 'sleep_stages', value: 'light', unit: 'stage', startMs, endMs }, + ] }).measurements + const score = compareStageOutputs([{ start: startMs, end: endMs, stage: improved ? 'deep' : 'light' }], + [{ start: startMs, end: endMs, stage: improved ? 'light' : 'deep' }], references, startMs, endMs) + return { cohortId, comparisons: [{ baseline: 'default', candidate: 'candidate', baselineHash: 'a', candidateHash: 'b', windows: [{ ...score, startMs, endMs }] }] } +} +it('keeps equal-night effects distinct from pooled duration effects and includes unscored nights', () => { + const result = summarizeStageNights([night('long', 900000, true), night('short', 100000, false), night('unknown', 500000, true, true)])[0] + expect(result).toMatchObject({ totalNights: 3, scoredNights: 2, unscoredNights: 1, improvedAgreementNights: 1, + regressedAgreementNights: 1, equalNightMeanAgreementDifference: 0, observedNightRange: { minimum: -1, maximum: 1 } }) + expect(result.pooledCommonSupport.candidate.agreement).toBe(0.9) + expect(result.pooledCommonSupport.baseline.agreement).toBe(0.1) + expect(result.nights[2].commonAgreementDifference).toBeNull() +}) +it('combines window confusion durations before computing nightly F1', () => { + const first = night('one', 900000, true), second = night('two', 100000, false) + second.comparisons[0].windows[0].startMs += 900000 + second.comparisons[0].windows[0].endMs += 900000 + first.comparisons[0].windows.push(second.comparisons[0].windows[0]) + const result = summarizeStageNights([first])[0] + expect(result.nights[0].commonSupport.candidate.agreement).toBe(0.9) + expect(result.nights[0].commonSupport.candidate.macroF1).toBeCloseTo(0.9 / 1.9) + expect(result.nights[0].commonSupport.candidate.pairedDurationMs).toBe(1000000) +}) +it('never merges different hashes or counts duplicated cohort/comparison evidence twice', () => { + const first = night('one', 100000, true), second = night('two', 100000, true) + second.comparisons[0].candidateHash = 'different' + expect(summarizeStageNights([first, second])).toHaveLength(2) + expect(() => summarizeStageNights([first, first])).toThrow('whole-night') + first.comparisons.push(first.comparisons[0]) + expect(() => summarizeStageNights([first])).toThrow('Duplicate comparison') +}) +it('keeps all-missing evidence unknown', () => { + const result = summarizeStageNights([night('missing', 100000, true, true)])[0] + expect(result.equalNightMeanAgreementDifference).toBeNull() + expect(result.observedNightRange).toBeNull() + expect(result.pooledCommonSupport.candidate.agreement).toBeNull() + expect(summarizeStageNights([])).toEqual([]) +}) + +it('rejects overlapping windows instead of double-counting their support', () => { + const input = night('one', 100000, true) + input.comparisons[0].windows.push(input.comparisons[0].windows[0]) + expect(() => summarizeStageNights([input])).toThrow('nonoverlapping') +}) diff --git a/src/biometrics/backtest/tests/numeric-comparison.test.ts b/src/biometrics/backtest/tests/numeric-comparison.test.ts new file mode 100644 index 000000000..87f759ef5 --- /dev/null +++ b/src/biometrics/backtest/tests/numeric-comparison.test.ts @@ -0,0 +1,59 @@ +// @vitest-environment node +import { expect, it } from 'vitest' +import { compareNumericConfigurations } from '../numeric-comparison' +import { DEFAULT_ALIGNMENT } from '../../references/alignment' +import { previewReferenceImport } from '../../references/normalize' + +const T = Date.UTC(2026, 9, 10, 23) +const refs = (rows: unknown[]) => previewReferenceImport({ side: 'left', source: 'test', subjectId: 'test', rows }).measurements +const point = (timestampMs: number, value = 60) => ({ recordKey: String(timestampMs), metric: 'heart_rate', timestampMs, value, unit: 'bpm' }) +const output = (label: 'default' | 'current' | 'candidate', rows: [number, number | null][]) => ({ label, configHash: label, + observations: rows.map(([offset, value]) => ({ timestampMs: T + offset, value })) }) + +it('exposes dropped difficult readings despite improved own-support error', () => { + const [result] = compareNumericConfigurations('heart_rate', null, + [output('default', [[0, 60], [60000, 100]]), output('candidate', [[0, 60]])], + refs([point(T), point(T + 60000)]), DEFAULT_ALIGNMENT) + expect(result).toMatchObject({ commonObservationCount: 1, lostObservationTimestamps: 1, addedObservationTimestamps: 0, + coverageReduced: true, differences: { ownMae: -20, commonMae: 0, referenceCoverage: -0.5 } }) +}) +it('recomputes interval aggregation on identical sample support', () => { + const [result] = compareNumericConfigurations('heart_rate', null, + [output('default', [[0, 60], [60000, 100]]), output('candidate', [[0, 60]])], + refs([{ recordKey: 'interval', metric: 'heart_rate', startMs: T, endMs: T + 120000, value: 60, unit: 'bpm', aggregation: 'mean' }]), DEFAULT_ALIGNMENT) + expect(result.differences).toMatchObject({ ownMae: -20, commonMae: 0 }) + expect(result.commonSupport.baseline.metrics?.mae).toBe(0) +}) +it('uses all three configurations even for the default-current comparison', () => { + const results = compareNumericConfigurations('heart_rate', null, + [output('default', [[0, 60], [60000, 90]]), output('current', [[0, 62], [60000, 60]]), output('candidate', [[0, 61]])], + refs([point(T), point(T + 60000)]), DEFAULT_ALIGNMENT) + expect(results).toHaveLength(3) + expect(results.every(result => result.commonObservationCount === 1)).toBe(true) + expect(results.find(result => result.candidate === 'current')?.differences.commonMae).toBe(2) +}) +it('does not treat nearby output timestamps as common support', () => { + const [result] = compareNumericConfigurations('heart_rate', null, + [output('default', [[0, 60]]), output('candidate', [[1, 60]])], refs([point(T)]), DEFAULT_ALIGNMENT) + expect(result).toMatchObject({ commonObservationCount: 0, differences: { commonMae: null, ownMae: 0 }, + lostObservationTimestamps: 1, addedObservationTimestamps: 1 }) +}) +it('excludes duplicate timestamps including null-value duplicates', () => { + const [result] = compareNumericConfigurations('heart_rate', null, + [output('default', [[0, 60], [0, null], [60000, 60]]), output('candidate', [[0, 60], [60000, 62]])], + refs([point(T), point(T + 60000)]), DEFAULT_ALIGNMENT) + expect(result).toMatchObject({ commonObservationCount: 1, ambiguousObservationTimestamps: { baseline: 1, candidate: 0 }, + differences: { commonMae: 2, commonSignedBias: 2 } }) +}) +it('preserves fixed clock offsets and null metrics without compatible reference evidence', () => { + const outputs = [output('default', [[0, 60]]), output('candidate', [[0, 62]])] + const [shifted] = compareNumericConfigurations('heart_rate', null, outputs, refs([point(T + 60000)]), + { ...DEFAULT_ALIGNMENT, pointToleranceMs: 0, clockOffsetMs: 60000 }) + expect(shifted.differences.commonMae).toBe(2) + const [missing] = compareNumericConfigurations('heart_rate', null, outputs, [], DEFAULT_ALIGNMENT) + expect(missing.differences.commonMae).toBeNull() + const [incompatible] = compareNumericConfigurations('window_ibi_variability_v1', 300000, outputs, + refs([{ recordKey: 'watch', metric: 'hrv_rmssd', timestampMs: T, value: 40, unit: 'ms', windowMs: 300000 }]), DEFAULT_ALIGNMENT) + expect(incompatible.commonSupport.baseline.status).toBe('incompatible') + expect(incompatible.differences.commonMae).toBeNull() +}) diff --git a/src/biometrics/backtest/tests/numeric-night-summary.test.ts b/src/biometrics/backtest/tests/numeric-night-summary.test.ts new file mode 100644 index 000000000..ba1dce8c6 --- /dev/null +++ b/src/biometrics/backtest/tests/numeric-night-summary.test.ts @@ -0,0 +1,53 @@ +// @vitest-environment node +import { createHash } from 'node:crypto' +import { expect, it } from 'vitest' +import { createNumericNightSummary } from '../numeric-night-summary' +import { createNightUncertainty } from '../uncertainty-core' +import { compareNumericConfigurations } from '../numeric-comparison' +import { DEFAULT_ALIGNMENT } from '../../references/alignment' +import { previewReferenceImport } from '../../references/normalize' +const summarize = createNumericNightSummary(createNightUncertainty(text => createHash('sha256').update(text).digest(), 'equal_night_mean_common_mae_difference')) +const T = Date.UTC(2026, 9, 10) +const policy = { version: 'paired-night-percentile-v1' as const, seed: 'numeric', minimumNights: 3, resamples: 200 } +function window(offset: number, count: number, error: number, missing = false) { + const observations = Array.from({ length: count }, (_, i) => ({ timestampMs: T + offset + i * 60000, value: 60 })) + const references = missing ? [] : previewReferenceImport({ side: 'left', source: 'fixture', subjectId: 'fixture', rows: observations.map((row, i) => ({ recordKey: String(i), metric: 'heart_rate', timestampMs: row.timestampMs, value: 60, unit: 'bpm' })) }).measurements + return { startMs: T + offset, endMs: T + offset + count * 60000, metrics: compareNumericConfigurations('heart_rate', null, [ + { label: 'default', configHash: 'baseline', observations }, + { label: 'candidate', configHash: 'candidate', observations: observations.map(row => ({ ...row, value: row.value + error })) }, + ], references, DEFAULT_ALIGNMENT) } +} +it('combines windows by paired reference count before giving nights equal weight', () => { + const [result] = summarize([ + { cohortId: 'a', numericComparisons: [window(0, 1, 10), window(120000, 3, 2)] }, + { cohortId: 'b', numericComparisons: [window(0, 1, 0)] }, + { cohortId: 'missing', numericComparisons: [window(0, 1, 0, true)] }, + ], policy) + expect(result).toMatchObject({ totalNights: 3, scoredNights: 2, unscoredNights: 1, equalNightMeanMaeDifference: 2, + pooledCommonSupport: { candidate: { mae: 3.2, pairedCount: 5 } }, + uncertainty: { status: 'insufficient_nights', estimate: 2, omittedNights: 1, statistic: 'equal_night_mean_common_mae_difference' } }) + expect(result.nights[0].commonMaeDifference).toBe(4) + expect(result.pooledCommonSupport.candidate).not.toHaveProperty('p95AbsoluteError') +}) +it('bootstraps numeric differences above one in metric units reproducibly', () => { + const nights = [2, 10, 30].map((error, i) => ({ cohortId: String(i), numericComparisons: [window(0, 1, error)] })) + const [result] = summarize(nights, policy) + expect(result.uncertainty.status).toBe('estimated') + expect(result.uncertainty).toEqual(summarize([...nights].reverse(), policy)[0].uncertainty) + expect(result.uncertainty.interval?.upper).toBeGreaterThan(10) + expect(result.uncertainty.estimate).toBe(14) +}) +it('keeps missing evidence unknown and rejects duplicate or overlapping evidence', () => { + const night = { cohortId: 'one', numericComparisons: [window(0, 1, 0, true)] } + expect(summarize([night])[0]).toMatchObject({ equalNightMeanMaeDifference: null, scoredNights: 0, uncertainty: { interval: null, estimate: null } }) + expect(() => summarize([night, night])).toThrow('cohorts') + expect(() => summarize([{ ...night, numericComparisons: [window(0, 2, 0), window(60000, 1, 0)] }])).toThrow('windows') + const duplicate = window(0, 1, 0) + duplicate.metrics.push(duplicate.metrics[0]) + expect(() => summarize([{ ...night, numericComparisons: [duplicate] }])).toThrow('Duplicate') +}) +it('does not combine different immutable configurations', () => { + const first = window(0, 1, 2), second = window(0, 1, 3) + second.metrics[0].candidateHash = 'other' + expect(summarize([{ cohortId: 'a', numericComparisons: [first] }, { cohortId: 'b', numericComparisons: [second] }])).toHaveLength(2) +}) diff --git a/src/biometrics/backtest/tests/numeric-subgroups.test.ts b/src/biometrics/backtest/tests/numeric-subgroups.test.ts new file mode 100644 index 000000000..e3476e23f --- /dev/null +++ b/src/biometrics/backtest/tests/numeric-subgroups.test.ts @@ -0,0 +1,28 @@ +// @vitest-environment node +import { expect, it } from 'vitest' +import { subgroupReferences, subgroupBoundaryExclusions, subgroupObjectiveEvidence } from '../numeric-subgroups' +import { previewReferenceImport } from '../../references/normalize' +const T = Date.UTC(2026, 9, 10) +const window = { startMs: T, endMs: T + 120000 } +const rows = previewReferenceImport({ side: 'left', source: 'test', subjectId: 'test', rows: [ + { recordKey: 'early', metric: 'heart_rate', timestampMs: T, value: 60, unit: 'bpm' }, + { recordKey: 'middle', metric: 'heart_rate', timestampMs: T + 60000, value: 60, unit: 'bpm' }, + { recordKey: 'first', metric: 'heart_rate', startMs: T, endMs: T + 60000, value: 60, unit: 'bpm', aggregation: 'mean' }, + { recordKey: 'spanning', metric: 'heart_rate', startMs: T + 30000, endMs: T + 90000, value: 60, unit: 'bpm', aggregation: 'mean' }, +] }).measurements +it('keeps point/interval semantics and midpoint boundaries without splitting reference records', () => { + expect(subgroupReferences('reference_points', rows, window).map(row => row.recordKey)).toEqual(['early', 'middle']) + expect(subgroupReferences('reference_intervals', rows, window).map(row => row.recordKey)).toEqual(['first', 'spanning']) + expect(subgroupReferences('window_first_half', rows, window).map(row => row.recordKey)).toEqual(['early', 'first']) + expect(subgroupReferences('window_second_half', rows, window).map(row => row.recordKey)).toEqual(['middle']) + expect(subgroupBoundaryExclusions('window_first_half', rows, window)).toBe(1) + expect(subgroupBoundaryExclusions('reference_intervals', rows, window)).toBe(0) +}) +it('weights supported nights equally and never hides unsupported nights in the subgroup mean', () => { + const identity = { baseline: 'default', candidate: 'candidate', metric: 'heart_rate', baselineHash: 'a', candidateHash: 'b' } + const nights = [{ commonMaeDifference: -4, commonSupport: { candidate: { pairedCount: 100 } } }, { commonMaeDifference: 6, commonSupport: { candidate: { pairedCount: 10 } } }] + const summary = { ...identity, name: 'window_second_half', nights } + expect(subgroupObjectiveEvidence([summary], identity, 10)).toEqual([{ name: 'window_second_half', pairedNights: 2, maeDifference: 1 }]) + expect(subgroupObjectiveEvidence([summary], identity, 11)).toEqual([{ name: 'window_second_half', pairedNights: 1, maeDifference: null }]) + expect(subgroupObjectiveEvidence([summary], { ...identity, candidateHash: 'different' }, 10)).toEqual([]) +}) diff --git a/src/biometrics/backtest/tests/numeric-traces.test.ts b/src/biometrics/backtest/tests/numeric-traces.test.ts new file mode 100644 index 000000000..7814acf13 --- /dev/null +++ b/src/biometrics/backtest/tests/numeric-traces.test.ts @@ -0,0 +1,43 @@ +// @vitest-environment node +import { expect, it } from 'vitest' +import { pageNumericTraces } from '../numeric-trace-page' +const hash = 'b'.repeat(64) +const request = { id: '11111111-1111-4111-8111-111111111111', expectedContentHash: 'a'.repeat(64), configuration: 'candidate' as const, limit: 2 } +function fixture() { + const reading = { side: 'left', sourceSide: 'right', configHash: hash, timestampMs: 1000, heartRate: 60, breathingRate: null, hrvIndex: null, + hrvMetric: 'window_ibi_variability_v1', qualityScore: 0, flags: null, limitations: ['unknown_context'], hrRaw: 65 } + return { version: 'waveform-backtest-report-v1', side: 'left', protocol: { mode: 'signal_replay', clockOffsetMs: 60000 }, + nights: [{ cohortId: 'one', evaluations: ['default', 'candidate'].map(label => ({ label, configHash: hash, revisionId: null, windows: [{ startMs: 1000, endMs: 5000 }] })) }], + waveform: { outputs: [{ cohortId: 'one', configHash: hash, readings: [ + { ...reading, timestampMs: 999 }, reading, { ...reading, side: 'right', heartRate: 99 }, + { ...reading, timestampMs: 2000, heartRate: null }, { ...reading, timestampMs: 4000 }, { ...reading, timestampMs: 5000 }, + ] }] } } +} +it('pages exact dataset-side readings without offset, downsampling, opposite-side values or private extra fields', () => { + const report = fixture(), before = structuredClone(report) + const first = pageNumericTraces(report, request) + expect(first).toMatchObject({ totalItems: 3, side: 'left', downsampled: false, clockOffsetMs: 60000, + provenance: 'retained_waveform_recomputation', nextCursor: { offset: 2, contentHash: request.expectedContentHash } }) + expect(first.items.map(row => row.timestampMs)).toEqual([1000, 2000]) + expect(first.items[1].heartRate).toBeNull() + expect(first.items[0]).not.toHaveProperty('hrRaw') + expect(first.items[0].sourceSide).toBe('right') + const last = pageNumericTraces(report, { ...request, offset: first.nextCursor?.offset }) + expect(last.items.map(row => row.timestampMs)).toEqual([4000]) + expect(last.nextCursor).toBeNull() + expect(pageNumericTraces(report, { ...request, configuration: 'default' }).items).toEqual(first.items) + expect(report).toEqual(before) +}) +it('rejects missing provenance, ambiguous output identity and invalid selections', () => { + expect(() => pageNumericTraces({ ...fixture(), side: undefined }, request)).toThrow('provenance') + const duplicate = fixture() + duplicate.waveform.outputs.push(duplicate.waveform.outputs[0]) + expect(() => pageNumericTraces(duplicate, request)).toThrow('ambiguous') + const mismatch = fixture() + mismatch.waveform.outputs[0].readings[1].configHash = 'c'.repeat(64) + expect(() => pageNumericTraces(mismatch, request)).toThrow('configuration mismatch') + expect(() => pageNumericTraces(fixture(), { ...request, cohortId: 'absent' })).toThrow('not evaluated') + expect(() => pageNumericTraces(fixture(), { ...request, configuration: 'current' })).toThrow('not evaluated') + expect(() => pageNumericTraces(fixture(), { ...request, offset: 4 })).toThrow('cursor') + expect(() => pageNumericTraces(fixture(), { ...request, limit: 501 })).toThrow() +}) diff --git a/src/biometrics/backtest/tests/objective-finalization.test.ts b/src/biometrics/backtest/tests/objective-finalization.test.ts new file mode 100644 index 000000000..e151a77f4 --- /dev/null +++ b/src/biometrics/backtest/tests/objective-finalization.test.ts @@ -0,0 +1,92 @@ +// @vitest-environment node +import { expect, it } from 'vitest' +import { assessNumericObjective, type ObjectivePolicy, type ObjectiveEvidence } from '../objective-policy' +import { finalizeObjectiveReport, objectiveOperations } from '../objective-finalization' +import type { createJobOperations } from '../job-operations' +const policy: ObjectivePolicy = { version: 'numeric-objective-v1', qualificationId: 'qualified-test', metric: 'heart_rate', minimumNights: 3, + minimumPairedReferencesPerNight: 10, maximumLostObservationFraction: 0.1, minimumMaeImprovement: 2, maximumNightMaeRegression: 1, + maximumRegressedNightFraction: 0.34, maximumSubgroupMaeRegression: 1, requiredSubgroups: [], + operational: { maximumWallMs: 1000, maximumPeakRssBytes: 1000, maximumIngestionAgeMs: 1000, + observations: { version: 'sampled-process-group-v1', maximumMemoryGapMs: 1000, maximumIngestionGapMs: 5000 } } } +const evidence: ObjectiveEvidence = { metricCompatible: true, split: 'holdout', holdoutUse: 1, intervalUpper: -2, subgroups: [], operational: null, + nights: ['a', 'b', 'c'].map(cohortId => ({ cohortId, pairedReferences: 10, maeDifference: -2, baselineObservationCount: 10, lostObservations: 1 })) } +const operations = (): ReturnType['finish']> => ({ version: 'backtest-job-operations-v1', wallMs: 1000, + scope: 'admission_through_prepublication', memory: { version: 'process-group-rss-observations-v1', scope: 'sum_rss_of_replay_process_group', + attempts: 3, samples: 3, allSamplesAvailable: true, peakObservedRssBytes: 1000, maximumObservationGapMs: 1000, intervalMs: 1000, exactKernelPeak: false }, + ingestion: { source: 'supervisor_live_piezo_receipt_age', checks: 2, samples: 2, allChecksMeasured: true, qualificationId: 'qualified-test', + maximumObservedAgeMs: 1000, maximumObservationGapMs: 5000, nominalIntervalMs: 5000, continuousMeasurement: false } }) +const report = () => ({ objectiveAssessments: ['default', 'current'].map(baseline => ({ baseline, candidate: 'candidate', baselineHash: baseline, + candidateHash: 'candidate', ...assessNumericObjective(policy, evidence, null) })) }) +it('finalizes measured operational evidence only under explicit sampled policy and trusted designation', () => { + const result = finalizeObjectiveReport(report(), policy, operations(), policy) + expect(result.objectiveAssessments).toEqual(expect.arrayContaining([expect.objectContaining({ status: 'improved', evidence: { ...evidence, operational: objectiveOperations(operations(), policy) } })])) + expect(finalizeObjectiveReport(report(), policy, operations(), null).objectiveAssessments).toEqual(expect.arrayContaining([expect.objectContaining({ status: 'insufficient_evidence', missingEvidence: ['policy_not_designated'] })])) +}) +it('does not repurpose sampled peaks for legacy policies or incomplete observations', () => { + const legacy = { ...policy, operational: { maximumWallMs: 1000, maximumPeakRssBytes: 1000, maximumIngestionAgeMs: 1000 } } + expect(objectiveOperations(operations(), legacy)).toBeNull() + for (const patch of [{ memory: null }, { memory: { ...operations().memory, samples: 1 } }, + { ingestion: { ...operations().ingestion, allChecksMeasured: false } }, { ingestion: { ...operations().ingestion, qualificationId: null } }]) { + expect(objectiveOperations({ ...operations(), ...patch }, policy)).toBeNull() + } +}) +it('treats sampling gaps and qualification mismatches as missing evidence at strict boundaries', () => { + for (const patch of [{ maximumIngestionGapMs: 5001 }, { maximumMemoryGapMs: 1001 }, { qualificationId: 'different' }]) { + const operational = objectiveOperations(operations(), policy) + if (!operational?.observations) throw new Error('Missing measured operations') + expect(assessNumericObjective(policy, { ...evidence, operational: { ...operational, observations: { ...operational.observations, ...patch } } }, policy).status).toBe('insufficient_evidence') + } +}) +it('replaces worker-supplied operational evidence and rejects mismatched policies or missing comparisons', () => { + const forged = report() + for (const row of forged.objectiveAssessments) row.evidence.operational = { wallMs: 0, peakRssBytes: 0, maximumIngestionAgeMs: 0 } + expect(finalizeObjectiveReport(forged, policy, null, policy).objectiveAssessments).toEqual(expect.arrayContaining([expect.objectContaining({ status: 'insufficient_evidence', missingEvidence: ['operational_evidence_unavailable'] })])) + expect(() => finalizeObjectiveReport(report(), { ...policy, minimumMaeImprovement: 3 }, operations(), policy)).toThrow('identity') + expect(() => finalizeObjectiveReport({ objectiveAssessments: report().objectiveAssessments.slice(0, 1) }, policy, operations(), policy)).toThrow() +}) + +const metrics = ['heart_rate', 'breathing_rate', 'window_ibi_variability_v1'] as const +const policies = metrics.map(metric => ({ ...policy, metric })) +const multiReport = () => ({ objectiveAssessments: policies.flatMap(item => ['default', 'current'].map(baseline => ({ + baseline, candidate: 'candidate', baselineHash: baseline, candidateHash: 'candidate', ...assessNumericObjective(item, evidence, null), +}))) }) + +it('reassesses both baselines independently for all three numeric metrics', () => { + const result = finalizeObjectiveReport(multiReport(), policies, operations(), policies) + expect(result.objectiveAssessments).toHaveLength(6) + for (const metric of metrics) { + expect(result.objectiveAssessments).toEqual(expect.arrayContaining(['default', 'current'].map(baseline => expect.objectContaining({ + baseline, policy: expect.objectContaining({ metric }), status: 'improved', designated: true, + })))) + } + const partial = finalizeObjectiveReport(multiReport(), policies, operations(), policies.slice(0, 2)) + expect(partial.objectiveAssessments).toEqual(expect.arrayContaining([expect.objectContaining({ + policy: expect.objectContaining({ metric: 'window_ibi_variability_v1' }), status: 'insufficient_evidence', missingEvidence: ['policy_not_designated'], + })])) +}) + +it('rejects missing, duplicated, substituted or policy-mismatched metric/baseline pairs', () => { + const source = multiReport() + const rows = source.objectiveAssessments + for (const objectiveAssessments of [rows.slice(1), [...rows, rows[0]], [rows[0], rows[0], ...rows.slice(2)], + rows.map((row, index) => index === 5 ? { ...row, policy: { ...row.policy, minimumMaeImprovement: 3 } } : row)]) { + expect(() => finalizeObjectiveReport({ objectiveAssessments }, policies, operations(), policies)).toThrow() + } + expect(() => finalizeObjectiveReport(source, [policies[0], policies[0], policies[2]], operations(), policies)).toThrow('Duplicate') + expect(() => finalizeObjectiveReport(source, policies, operations(), [policies[0], policies[0]])).toThrow('Duplicate') +}) + +it('uses each metric policy for operational bounds and replaces child telemetry for every metric', () => { + const strict = policies.map(item => item.metric === 'breathing_rate' + ? { ...item, operational: { ...item.operational, maximumWallMs: 999 } } + : item) + const raw = { objectiveAssessments: strict.flatMap(item => ['default', 'current'].map(baseline => ({ + baseline, candidate: 'candidate', baselineHash: baseline, candidateHash: 'candidate', + ...assessNumericObjective(item, { ...evidence, operational: { wallMs: 0, peakRssBytes: 0, maximumIngestionAgeMs: 0 } }, item), + }))) } + const result = finalizeObjectiveReport(raw, strict, operations(), strict) + for (const row of result.objectiveAssessments as Array<{ policy: ObjectivePolicy, status: string, evidence: ObjectiveEvidence }>) { + expect(row.evidence.operational?.wallMs).toBe(1000) + expect(row.status).toBe(row.policy.metric === 'breathing_rate' ? 'regressed' : 'improved') + } +}) diff --git a/src/biometrics/backtest/tests/objective-policy.test.ts b/src/biometrics/backtest/tests/objective-policy.test.ts new file mode 100644 index 000000000..4453abf9a --- /dev/null +++ b/src/biometrics/backtest/tests/objective-policy.test.ts @@ -0,0 +1,64 @@ +// @vitest-environment node +import { expect, it } from 'vitest' +import { assessNumericObjective, type ObjectiveEvidence, type ObjectivePolicy } from '../objective-policy' +import { experimentProtocolSchema } from '../contracts' +import { configHash } from '../../algorithms/canonical' +const policy: ObjectivePolicy = { version: 'numeric-objective-v1', qualificationId: 'test-only', metric: 'heart_rate', minimumNights: 3, + minimumPairedReferencesPerNight: 10, maximumLostObservationFraction: 0.1, minimumMaeImprovement: 2, + maximumNightMaeRegression: 1, maximumRegressedNightFraction: 0.34, maximumSubgroupMaeRegression: 1, requiredSubgroups: ['motion'], + operational: { maximumWallMs: 1000, maximumPeakRssBytes: 1000, maximumIngestionAgeMs: 1000 } } +const evidence = (): ObjectiveEvidence => ({ metricCompatible: true, split: 'holdout', holdoutUse: 1, + nights: ['a', 'b', 'c'].map(cohortId => ({ cohortId, pairedReferences: 10, maeDifference: -2, baselineObservationCount: 10, lostObservations: 1 })), + intervalUpper: -2, subgroups: [{ name: 'motion', pairedNights: 3, maeDifference: 1 }], operational: { wallMs: 1000, peakRssBytes: 1000, maximumIngestionAgeMs: 1000 } }) +it('only a designated qualified policy can award improvement at exact thresholds', () => { + expect(assessNumericObjective(policy, evidence(), policy).status).toBe('improved') + expect(assessNumericObjective(policy, evidence(), null)).toMatchObject({ status: 'insufficient_evidence', missingEvidence: ['policy_not_designated'] }) + expect(assessNumericObjective(policy, evidence(), { ...policy, qualificationId: 'different' }).status).toBe('insufficient_evidence') +}) +it('requires independent support, uncertainty, subgroups and measured operations', () => { + for (const patch of [{ split: 'tuning' as const }, { holdoutUse: 2 }, { intervalUpper: null }, { subgroups: [] }, { operational: null }, + { nights: evidence().nights.slice(0, 2) }, { nights: evidence().nights.map(row => ({ ...row, pairedReferences: 9 })) }]) { + expect(assessNumericObjective(policy, { ...evidence(), ...patch }, policy).status).toBe('insufficient_evidence') + } + expect(assessNumericObjective(policy, { ...evidence(), metricCompatible: false }, policy).status).toBe('incompatible') +}) +it('checks strict operational and coverage boundaries', () => { + for (const key of ['wallMs', 'peakRssBytes', 'maximumIngestionAgeMs'] as const) { + expect(assessNumericObjective(policy, { ...evidence(), operational: { wallMs: 1000, peakRssBytes: 1000, maximumIngestionAgeMs: 1000, [key]: 1001 } }, policy).status).toBe('regressed') + } + const data = evidence() + data.nights[0].lostObservations = 2 + expect(assessNumericObjective(policy, data, policy).failedThresholds).toContain('coverage_loss') +}) +it('does not hide night or subgroup regressions behind a better mean', () => { + const data = evidence() + data.nights[0].maeDifference = 1.01 + expect(assessNumericObjective(policy, data, policy).failedThresholds).toContain('night_regression') + data.nights[0].maeDifference = 1 + data.nights[1].maeDifference = 1 + expect(assessNumericObjective(policy, data, policy).failedThresholds).toContain('regressed_night_fraction') + data.subgroups[0].maeDifference = 1.01 + expect(assessNumericObjective(policy, data, policy).failedThresholds).toContain('subgroup_regression:motion') + expect(assessNumericObjective(policy, { ...evidence(), intervalUpper: -1.99 }, policy).status).toBe('no_material_change') +}) +it('pins thresholds in protocol identity and rejects duplicate evidence', () => { + const base = { version: 'descriptive-evaluation-v1', mode: 'signal_replay', assumedStageCalibrationQuality: 0, pointToleranceMs: 30000, clockOffsetMs: 0, objectivePolicy: policy } + expect(experimentProtocolSchema.safeParse({ ...base, mode: 'comparison' }).success).toBe(false) + const first = experimentProtocolSchema.parse(base) + const second = experimentProtocolSchema.parse({ ...base, objectivePolicy: { ...policy, minimumMaeImprovement: 3 } }) + expect(configHash(first)).not.toBe(configHash(second)) + const data = evidence() + data.nights.push(data.nights[0]) + expect(() => assessNumericObjective(policy, data, policy)).toThrow('identity') +}) + +it('freezes one policy per numeric metric without rewriting legacy protocols', () => { + const base = { version: 'descriptive-evaluation-v1', mode: 'signal_replay', assumedStageCalibrationQuality: 0, pointToleranceMs: 30000, clockOffsetMs: 0 } + const policies = ['heart_rate', 'breathing_rate', 'window_ibi_variability_v1'].map(metric => ({ ...policy, metric })) + expect(experimentProtocolSchema.parse({ ...base, objectivePolicies: policies }).objectivePolicies).toEqual(policies) + expect(experimentProtocolSchema.parse({ ...base, objectivePolicy: policy })).not.toHaveProperty('objectivePolicies') + for (const patch of [{ objectivePolicies: [] }, { objectivePolicies: [policy, policy] }, { objectivePolicies: policies, objectivePolicy: policy }, + { objectivePolicies: policies, mode: 'comparison' }, { objectivePolicies: policies, mode: 'stage_replay' }]) { + expect(experimentProtocolSchema.safeParse({ ...base, ...patch }).success).toBe(false) + } +}) diff --git a/src/biometrics/backtest/tests/process-group-memory.test.ts b/src/biometrics/backtest/tests/process-group-memory.test.ts new file mode 100644 index 000000000..7f52919b2 --- /dev/null +++ b/src/biometrics/backtest/tests/process-group-memory.test.ts @@ -0,0 +1,74 @@ +// @vitest-environment node +import { mkdtemp, mkdir, writeFile, rm } from 'node:fs/promises' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { afterEach, expect, it, vi } from 'vitest' +import { monitorProcessGroupMemory, sampleProcessGroupRss } from '../process-group-memory' +let root: string | undefined +afterEach(async () => { + vi.useRealTimers() + if (root) await rm(root, { recursive: true, force: true }) + root = undefined +}) +async function processFixture(pid: number, group: number, rss: string | null, state = 'S') { + if (!root) root = await mkdtemp(join(tmpdir(), 'replay-proc-')) + const directory = join(root, String(pid)) + await mkdir(directory, { recursive: true }) + await writeFile(join(directory, 'stat'), `${pid} (name with ) parentheses) ${state} 1 ${group} 0 0\n`) + if (rss !== null) await writeFile(join(directory, 'status'), rss) + return directory +} +it('sums only the owned process group and treats disappearing members as an observation race', async () => { + await processFixture(10, 10, 'VmRSS:\t100 kB\n') + await processFixture(11, 10, 'VmRSS:\t200 kB\n') + await processFixture(12, 12, 'VmRSS:\t999 kB\n') + await processFixture(13, 10, null) + await processFixture(14, 10, 'Name: zombie\n', 'Z') + expect(await sampleProcessGroupRss(10, root)).toBe(300 * 1024) + expect(await sampleProcessGroupRss(99, root)).toBeNull() +}) +it('reports unavailable memory for malformed owned RSS and excessive group size', async () => { + const directory = await processFixture(10, 10, 'VmRSS: unknown\n') + expect(await sampleProcessGroupRss(10, root)).toBeNull() + await writeFile(join(directory, 'status'), 'VmRSS: 99999999999999999999999 kB\n') + expect(await sampleProcessGroupRss(10, root)).toBeNull() + for (let pid = 10; pid < 75; pid++) await processFixture(pid, 10, 'VmRSS: 1 kB\n') + expect(await sampleProcessGroupRss(10, root)).toBeNull() +}) +it('retains sampled peaks, missing samples and gaps and stops its periodic work', async () => { + vi.useFakeTimers() + let time = 0 + const read = vi.fn().mockResolvedValueOnce(100).mockResolvedValueOnce(null).mockResolvedValue(300) + const monitor = monitorProcessGroupMemory(1, { sample: read, now: () => time }) + await monitor.sample() + time = 2000 + await monitor.sample() + time = 2200 + await monitor.sample() + expect(monitor.snapshot()).toMatchObject({ attempts: 3, samples: 2, allSamplesAvailable: false, + peakObservedRssBytes: 300, maximumObservationGapMs: 2000, exactKernelPeak: false }) + await monitor.stop() + await vi.advanceTimersByTimeAsync(5000) + await monitor.sample() + expect(read).toHaveBeenCalledTimes(3) +}) +it('serializes slow reads and waits for them when stopped', async () => { + let resolve: (value: number) => void = () => {} + const read = vi.fn(() => new Promise((done) => { + resolve = done + })) + const monitor = monitorProcessGroupMemory(1, { sample: read }) + const first = monitor.sample(), second = monitor.sample() + await Promise.resolve() + expect(read).toHaveBeenCalledTimes(1) + let stopped = false + const stopping = monitor.stop().then(() => { + stopped = true + }) + await Promise.resolve() + expect(stopped).toBe(false) + resolve(500) + await Promise.all([first, second, stopping]) + expect(stopped).toBe(true) + expect(monitor.snapshot().peakObservedRssBytes).toBe(500) +}) diff --git a/src/biometrics/backtest/tests/raw-artifact-references.test.ts b/src/biometrics/backtest/tests/raw-artifact-references.test.ts new file mode 100644 index 000000000..585f5bae4 --- /dev/null +++ b/src/biometrics/backtest/tests/raw-artifact-references.test.ts @@ -0,0 +1,95 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { afterEach, beforeEach, expect, it } from 'vitest' +import * as schema from '@/src/db/biometrics-schema' + +let sqlite: Database.Database +let db: ReturnType> +beforeEach(() => { + sqlite = new Database(':memory:') + sqlite.pragma('foreign_keys = ON') + db = drizzle(sqlite, { schema }) + migrate(db, { migrationsFolder: 'src/db/biometrics-migrations' }) + for (const id of ['dataset-a', 'dataset-b']) db.insert(schema.biometricDatasets).values({ id, side: 'left', subjectId: 'fixture', + idempotencyKey: id, requestHash: id, contentHash: id, manifest: {}, snapshot: {}, byteLength: 4, createdAtMs: 1000 }).run() + for (const contentHash of ['hash-a', 'hash-b']) db.insert(schema.biometricRawArtifacts).values({ contentHash, + name: `${contentHash}.RAW.gz`, compressedBytes: 100, expandedBytes: 200, createdAtMs: 1000 }).run() +}) +afterEach(() => sqlite.close()) + +it('shares retained bytes across datasets while preserving unique immutable positions', () => { + const link = { id: 'link-a', datasetId: 'dataset-a', artifactHash: 'hash-a', position: 0 } + db.insert(schema.biometricDatasetRawArtifacts).values(link).run() + db.insert(schema.biometricDatasetRawArtifacts).values({ ...link, id: 'link-b', datasetId: 'dataset-b' }).run() + expect(() => db.insert(schema.biometricDatasetRawArtifacts).values({ ...link, id: 'duplicate', position: 1 }).run()).toThrow('UNIQUE') + expect(() => db.insert(schema.biometricDatasetRawArtifacts).values({ ...link, id: 'reordered', artifactHash: 'hash-b' }).run()).toThrow('UNIQUE') + // Releasing one association leaves another live reference and retains the + // original position/identity for audit even after all references are released. + sqlite.prepare('UPDATE biometric_dataset_raw_artifacts SET released_at_ms = 2000 WHERE id = ?').run('link-a') + expect(sqlite.prepare('SELECT dataset_id FROM biometric_dataset_raw_artifacts WHERE artifact_hash = ? AND released_at_ms IS NULL').all('hash-a')) + .toEqual([{ dataset_id: 'dataset-b' }]) + expect(() => db.delete(schema.biometricRawArtifacts).run()).toThrow('FOREIGN KEY') + expect(() => db.delete(schema.biometricDatasets).run()).toThrow('FOREIGN KEY') +}) + +it('rejects dangling artifact references and duplicate storage names', () => { + expect(() => db.insert(schema.biometricDatasetRawArtifacts).values({ id: 'missing-dataset', datasetId: 'missing', artifactHash: 'hash-a', position: 0 }).run()).toThrow('FOREIGN KEY') + expect(() => db.insert(schema.biometricDatasetRawArtifacts).values({ id: 'missing-artifact', datasetId: 'dataset-a', artifactHash: 'missing', position: 0 }).run()).toThrow('FOREIGN KEY') + expect(() => db.insert(schema.biometricRawArtifacts).values({ contentHash: 'another-hash', name: 'hash-a.RAW.gz', + compressedBytes: 100, expandedBytes: 200, createdAtMs: 1000 }).run()).toThrow('UNIQUE') +}) + +it('releases only its own references and makes shared bytes collectible after the final release', async () => { + const { DatasetStore } = await import('../datasets') + let now = 2000 + const store = new DatasetStore(db, () => now) + for (const datasetId of ['dataset-a', 'dataset-b']) db.insert(schema.biometricDatasetRawArtifacts).values({ + id: datasetId, datasetId, artifactHash: 'hash-a', position: 0, + }).run() + expect(store.release('dataset-a').releasedAtMs).toBe(2000) + expect(sqlite.prepare('SELECT released_at_ms FROM biometric_raw_artifacts WHERE content_hash = ?').get('hash-a')).toEqual({ released_at_ms: null }) + expect(sqlite.prepare('SELECT released_at_ms FROM biometric_dataset_raw_artifacts WHERE dataset_id = ?').get('dataset-b')).toEqual({ released_at_ms: null }) + now = 3000 + store.release('dataset-b') + expect(sqlite.prepare('SELECT released_at_ms FROM biometric_raw_artifacts WHERE content_hash = ?').get('hash-a')).toEqual({ released_at_ms: 3000 }) + // Unrelated unreferenced artifacts are not touched by a dataset release. + expect(sqlite.prepare('SELECT released_at_ms FROM biometric_raw_artifacts WHERE content_hash = ?').get('hash-b')).toEqual({ released_at_ms: null }) + now = 4000 + expect(store.release('dataset-a').releasedAtMs).toBe(2000) + expect(sqlite.prepare('SELECT released_at_ms FROM biometric_dataset_raw_artifacts WHERE dataset_id = ?').get('dataset-a')).toEqual({ released_at_ms: 2000 }) + expect(sqlite.prepare('SELECT released_at_ms FROM biometric_raw_artifacts WHERE content_hash = ?').get('hash-a')).toEqual({ released_at_ms: 3000 }) +}) + +it.each(['queued', 'running', 'cancel_requested', 'interrupted'] as const)('preserves all retained inputs while a job is %s', async (state) => { + const { DatasetStore } = await import('../datasets') + db.insert(schema.biometricDatasetRawArtifacts).values({ id: 'link', datasetId: 'dataset-a', artifactHash: 'hash-a', position: 0 }).run() + db.insert(schema.biometricExperiments).values({ id: 'experiment', datasetId: 'dataset-a', side: 'left', protocol: {}, protocolHash: 'fixture', createdAtMs: 1000 }).run() + db.insert(schema.biometricBacktestJobs).values({ id: 'job', experimentId: 'experiment', idempotencyKey: 'job', requestHash: 'fixture', + input: {}, state, progress: {}, createdAtMs: 1000, updatedAtMs: 1000 }).run() + expect(() => new DatasetStore(db).release('dataset-a')).toThrow('Cancel or finish') + for (const table of ['biometric_datasets', 'biometric_raw_artifacts', 'biometric_dataset_raw_artifacts']) { + expect(sqlite.prepare(`SELECT count(*) AS n FROM ${table} WHERE released_at_ms IS NOT NULL`).get()).toEqual({ n: 0 }) + } +}) + +it('rolls back reference release if persisting the dataset tombstone fails', async () => { + const { DatasetStore } = await import('../datasets') + db.insert(schema.biometricDatasetRawArtifacts).values({ id: 'link', datasetId: 'dataset-a', artifactHash: 'hash-a', position: 0 }).run() + sqlite.exec('CREATE TRIGGER fail_release BEFORE UPDATE ON biometric_datasets BEGIN SELECT RAISE(ABORT, \'injected failure\'); END') + expect(() => new DatasetStore(db).release('dataset-a')).toThrow('injected failure') + expect(sqlite.prepare('SELECT released_at_ms FROM biometric_dataset_raw_artifacts').get()).toEqual({ released_at_ms: null }) + expect(sqlite.prepare('SELECT released_at_ms FROM biometric_raw_artifacts WHERE content_hash = ?').get('hash-a')).toEqual({ released_at_ms: null }) +}) + +it.each(['cancelled', 'succeeded', 'failed'] as const)('allows release after a job becomes %s', async (state) => { + const { DatasetStore } = await import('../datasets') + db.insert(schema.biometricDatasetRawArtifacts).values({ id: 'link', datasetId: 'dataset-a', artifactHash: 'hash-a', position: 0 }).run() + db.insert(schema.biometricExperiments).values({ id: 'experiment', datasetId: 'dataset-a', side: 'left', protocol: {}, protocolHash: 'fixture', createdAtMs: 1000 }).run() + db.insert(schema.biometricBacktestJobs).values({ id: 'job', experimentId: 'experiment', idempotencyKey: 'job', requestHash: 'fixture', + input: {}, state, progress: {}, createdAtMs: 1000, updatedAtMs: 1000 }).run() + expect(new DatasetStore(db, () => 2000).release('dataset-a').releasedAtMs).toBe(2000) + expect(sqlite.prepare('SELECT released_at_ms FROM biometric_dataset_raw_artifacts').get()).toEqual({ released_at_ms: 2000 }) + expect(sqlite.prepare('SELECT state FROM biometric_backtest_jobs').get()).toEqual({ state }) +}) diff --git a/src/biometrics/backtest/tests/raw-catalog.test.ts b/src/biometrics/backtest/tests/raw-catalog.test.ts new file mode 100644 index 000000000..d019aadb1 --- /dev/null +++ b/src/biometrics/backtest/tests/raw-catalog.test.ts @@ -0,0 +1,107 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import * as schema from '@/src/db/biometrics-schema' +import { RawSourceCatalog } from '../raw-catalog' + +let sqlite: Database.Database +let catalog: RawSourceCatalog +let now: number +const digest = 'a'.repeat(64) +function inspection(source = 'capture.RAW.gz', sha256 = digest) { + return { source_name: source, retained: false, artifact: { name: `${sha256}.RAW.gz`, sha256, compressed_bytes: 100, expanded_bytes: 200 }, + inventory: { version: 'retained-raw-inventory-v1', artifact_sha256: sha256, + decoded_records: 2, piezo_records: 1, pump_records: 1, usable_dual_frames: 1, empty_channel_frames: 0, + oversize_channel_frames: 0, invalid_timestamps: 0, timestamp_reversals: 0, duplicate_timestamps: 0, + first_event_ms: 1000, last_event_ms: 2000, + channels: { left1: { samples: 10, lost_markers: 1, truncated_tail_bytes: 0 }, right1: { samples: 10, lost_markers: 0, truncated_tail_bytes: 0 } }, + ordered_waveform_available: true, limitations: ['Fixture bounds do not establish continuous coverage.'] } } +} +beforeEach(() => { + now = 3000 + sqlite = new Database(':memory:') + sqlite.pragma('foreign_keys = ON') + const db = drizzle(sqlite, { schema }) + migrate(db, { migrationsFolder: 'src/db/biometrics-migrations' }) + catalog = new RawSourceCatalog(db, () => now) +}) +afterEach(() => sqlite.close()) + +describe('inspected RAW source catalog', () => { + it('persists bounded evidence without claiming retained or current availability', () => { + const input = inspection() + const id = catalog.record(input) + expect(catalog.read(id)).toEqual({ id, inspection: input, inspectedAtMs: 3000, unavailableAtMs: null, availability: 'requires_reverification' }) + now = 4000 + expect(catalog.record(input)).toBe(id) + const reopened = new RawSourceCatalog(drizzle(sqlite, { schema })) + expect(reopened.read(id).inspectedAtMs).toBe(4000) + expect(sqlite.prepare('SELECT * FROM biometric_raw_artifacts').all()).toEqual([]) + expect(sqlite.prepare('SELECT * FROM biometric_datasets').all()).toEqual([]) + }) + + it('preserves replaced identities and prevents a late failure from invalidating newer bytes', () => { + const oldId = catalog.record(inspection()) + now = 5000 + const newId = catalog.record(inspection('capture.RAW.gz', 'b'.repeat(64))) + expect(newId).not.toBe(oldId) + expect(catalog.read(oldId)).toMatchObject({ unavailableAtMs: 5000, availability: 'unavailable', inspection: inspection() }) + now = 6000 + catalog.markUnavailable(oldId) + expect(catalog.read(oldId).unavailableAtMs).toBe(5000) + expect(catalog.page({ startMs: 1000, endMs: 2001 }).items.map(row => row.id)).toEqual([newId]) + catalog.markUnavailable(newId) + expect(catalog.page({ startMs: 1000, endMs: 2001 }).items).toEqual([]) + now = 7000 + expect(catalog.record(inspection())).toBe(oldId) + expect(catalog.read(oldId)).toMatchObject({ unavailableAtMs: null, inspectedAtMs: 7000 }) + }) + + it('rejects conflicting evidence atomically and validates cross-field identities', () => { + const id = catalog.record(inspection()) + const conflict = inspection() + conflict.artifact.expanded_bytes++ + expect(() => catalog.record(conflict)).toThrow('different inspection evidence') + expect(catalog.read(id).inspection).toEqual(inspection()) + const changed = inspection() + changed.inventory.artifact_sha256 = 'b'.repeat(64) + expect(() => catalog.record(changed)).toThrow('inspected bytes') + expect(() => catalog.record({ ...inspection(), retained: true })).toThrow() + expect(() => catalog.record(inspection('../escape.RAW.gz'))).toThrow() + expect(() => catalog.record({ ...inspection(), extra: true })).toThrow() + expect(() => catalog.read('0'.repeat(64))).toThrow('not found') + }) + + it('rejects inconsistent counts, event bounds, descriptors and capabilities', () => { + const changes = [ + { first_event_ms: null }, { first_event_ms: 3000 }, { decoded_records: 1 }, + { empty_channel_frames: 1 }, { ordered_waveform_available: false }, { invalid_timestamps: 1 }, + { timestamp_reversals: 1 }, { oversize_channel_frames: 1 }, + { channels: { ...inspection().inventory.channels, left1: { samples: 1, lost_markers: 2, truncated_tail_bytes: 0 } } }, + ] + for (const change of changes) expect(() => catalog.record({ ...inspection(), inventory: { ...inspection().inventory, ...change } })).toThrow() + expect(() => catalog.record({ ...inspection(), artifact: { ...inspection().artifact, name: 'wrong.RAW.gz' } })).toThrow('content hash') + expect(() => catalog.record({ ...inspection(), artifact: { ...inspection().artifact, expanded_bytes: Number.MAX_SAFE_INTEGER + 1 } })).toThrow() + }) + + it('pages potential overlap with half-open request boundaries and no duplicate identities', () => { + const ids = Array.from({ length: 52 }, (_, index) => catalog.record(inspection(`capture-${index}.RAW.gz`))).sort() + const first = catalog.page({ startMs: 2000, endMs: 3000 }) + expect(first.items.map(row => row.id)).toEqual(ids.slice(0, 50)) + expect(first.nextCursor).toBe(ids[49]) + const second = catalog.page({ startMs: 2000, endMs: 3000, afterId: first.nextCursor ?? undefined }) + expect(second.items.map(row => row.id)).toEqual(ids.slice(50)) + expect(second.nextCursor).toBeNull() + expect(catalog.page({ startMs: 0, endMs: 1000 }).items).toEqual([]) + expect(catalog.page({ startMs: 2001, endMs: 3000 }).items).toEqual([]) + expect(() => catalog.page({ startMs: 1000, endMs: 1000 })).toThrow('nonempty') + }) + + it('detects corrupted indexed metadata before returning an inspected descriptor', () => { + const id = catalog.record(inspection()) + sqlite.prepare('UPDATE biometric_raw_sources SET first_event_ms = 999 WHERE id = ?').run(id) + expect(() => catalog.read(id)).toThrow('integrity check') + }) +}) diff --git a/src/biometrics/backtest/tests/raw-collection-child.test.ts b/src/biometrics/backtest/tests/raw-collection-child.test.ts new file mode 100644 index 000000000..125ab0947 --- /dev/null +++ b/src/biometrics/backtest/tests/raw-collection-child.test.ts @@ -0,0 +1,41 @@ +// @vitest-environment node +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, expect, it } from 'vitest' +import { runRawCollectionChild } from '../raw-collection-child' +let directory: string, executable: string +const location = () => ({ pythonPath: executable, modulesPath: directory, databasePath: join(directory, 'db'), retainedDirectory: join(directory, 'raw') }) +beforeEach(async () => { + directory = await mkdtemp(join(tmpdir(), 'collection-child-')) + executable = join(directory, 'worker') +}) +afterEach(() => rm(directory, { recursive: true, force: true })) +async function worker(body: string) { + await writeFile(executable, `#!${process.execPath}\nlet input='';process.stdin.on('data',c=>input+=c);process.stdin.on('end',()=>{${body}});`, { mode: 0o700 }) +} +const result = { version: 'raw-collection-result-v1', released: 2, orphans: 1 } +const emit = `process.stdout.write(${JSON.stringify(JSON.stringify(result))});` +it('uses fixed module, trusted paths and an input with no caller-controlled retention settings', async () => { + const args = ['-B', '-m', 'common.biometrics.collection_worker', '--database-path', location().databasePath, '--retained-directory', location().retainedDirectory] + await worker(`if(JSON.stringify(process.argv.slice(2))!==${JSON.stringify(JSON.stringify(args))} || input!==${JSON.stringify(JSON.stringify({ version: 'raw-collection-work-v1' }))})process.exit(1);${emit}`) + expect(await runRawCollectionChild(location())).toEqual(result) +}) +it('rejects malformed output and unsuccessful exit', async () => { + await worker('process.stdout.write("{}");') + await expect(runRawCollectionChild(location())).rejects.toThrow('invalid or incomplete') + await worker(`${emit}process.exitCode=1;`) + await expect(runRawCollectionChild(location())).rejects.toThrow('invalid or incomplete') +}) +it('bounds output, diagnostics and time, and cancels a running child', async () => { + await worker('process.stdout.write("x".repeat(1025));') + await expect(runRawCollectionChild(location())).rejects.toThrow('output exceeded') + await worker('process.stderr.write("x".repeat(16385));') + await expect(runRawCollectionChild(location())).rejects.toThrow('diagnostics exceeded') + await worker('setInterval(()=>{},1000);') + await expect(runRawCollectionChild(location(), { timeoutMs: 50 })).rejects.toThrow('time budget') + const controller = new AbortController() + const promise = runRawCollectionChild(location(), { signal: controller.signal }) + setTimeout(() => controller.abort(), 30) + await expect(promise).rejects.toThrow('cancelled') +}) diff --git a/src/biometrics/backtest/tests/raw-inspection-child.test.ts b/src/biometrics/backtest/tests/raw-inspection-child.test.ts new file mode 100644 index 000000000..89d9ca649 --- /dev/null +++ b/src/biometrics/backtest/tests/raw-inspection-child.test.ts @@ -0,0 +1,58 @@ +// @vitest-environment node +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, expect, it } from 'vitest' +import { runRawInspectionChild } from '../raw-inspection-child' +let directory: string, executable: string +const input = { afterName: null } +const location = () => ({ pythonPath: executable, modulesPath: directory, archiveDirectory: directory }) +beforeEach(async () => { + directory = await mkdtemp(join(tmpdir(), 'inspection-child-')) + executable = join(directory, 'worker') +}) +afterEach(() => rm(directory, { recursive: true, force: true })) +async function worker(body: string) { + await writeFile(executable, `#!${process.execPath}\nsetTimeout(() => process.exit(91), 3000).unref(); let input='';process.stdin.on('data',c=>input+=c);process.stdin.on('end',()=>{${body}});`, { mode: 0o700 }) +} +const emit = (value: unknown) => `process.stdout.write(${JSON.stringify(JSON.stringify(value))});` +it('uses the fixed module and clean environment and sends only a validated cursor', async () => { + await worker(`if(JSON.stringify(process.argv.slice(2))!==${JSON.stringify(JSON.stringify(['-B', '-m', 'common.biometrics.inspection_worker', '--archive-directory', directory]))} || process.env.RAW_TEST_SECRET || process.env.PYTHONPATH!==${JSON.stringify(directory)} || input!==${JSON.stringify(JSON.stringify({ version: 'raw-inspection-work-v1', after_name: null }))})process.exit(1);${emit({ status: 'complete', nextCursor: null })}`) + process.env.RAW_TEST_SECRET = 'private' + try { + expect(await runRawInspectionChild(location(), input)).toEqual({ status: 'complete', nextCursor: null }) + } + finally { delete process.env.RAW_TEST_SECRET } + expect(() => runRawInspectionChild(location(), { afterName: '../a.RAW.gz' })).toThrow() + expect(() => runRawInspectionChild(location(), input, { timeoutMs: 120001 })).toThrow('time budget') +}) +it.each([ + '{', '{}', '{"status":"complete","nextCursor":null}\n{}', + '{"status":"source_unavailable","nextCursor":"../a.RAW.gz"}', + '{"status":"complete","nextCursor":null,"extra":true}', +])('rejects malformed or additional output %s', async (output) => { + await worker(`process.stdout.write(${JSON.stringify(output)});`) + await expect(runRawInspectionChild(location(), input)).rejects.toThrow('invalid or incomplete') +}) +it('requires advancing cursors, valid UTF-8 and successful exit', async () => { + await worker(emit({ status: 'source_unavailable', nextCursor: 'a.RAW.gz' })) + await expect(runRawInspectionChild(location(), { afterName: 'a.RAW.gz' })).rejects.toThrow('invalid or incomplete') + await worker(`${emit({ status: 'complete', nextCursor: null })}process.stdout.write(Buffer.from([0xc3]));`) + await expect(runRawInspectionChild(location(), input)).rejects.toThrow('invalid or incomplete') + await worker(`${emit({ status: 'complete', nextCursor: null })}process.exitCode=1;`) + await expect(runRawInspectionChild(location(), input)).rejects.toThrow('invalid or incomplete') +}) +it('bounds output, diagnostics, elapsed time and cancellation', async () => { + await worker('process.stdout.write(\'x\'.repeat(65537));') + await expect(runRawInspectionChild(location(), input)).rejects.toThrow('output exceeded') + await worker('process.stderr.write(\'x\'.repeat(16385));') + await expect(runRawInspectionChild(location(), input)).rejects.toThrow('diagnostics exceeded') + await worker('setInterval(()=>{},1000);') + await expect(runRawInspectionChild(location(), input, { timeoutMs: 30 })).rejects.toThrow('time budget') + const controller = new AbortController() + const running = runRawInspectionChild(location(), input, { signal: controller.signal }) + controller.abort() + await expect(running).rejects.toThrow('cancelled') + await expect(runRawInspectionChild(location(), input, { signal: controller.signal })).rejects.toThrow('cancelled') + await expect(runRawInspectionChild({ ...location(), pythonPath: join(directory, 'missing') }, input)).rejects.toThrow('could not start') +}) diff --git a/src/biometrics/backtest/tests/raw-publication-child.test.ts b/src/biometrics/backtest/tests/raw-publication-child.test.ts new file mode 100644 index 000000000..6df9e4c90 --- /dev/null +++ b/src/biometrics/backtest/tests/raw-publication-child.test.ts @@ -0,0 +1,57 @@ +// @vitest-environment node +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, expect, it } from 'vitest' +import { runRawPublicationChild } from '../raw-publication-child' +let directory: string, executable: string +const hash = 'a'.repeat(64) +// This protocol fixture never runs Python or publishes rows. +const work = { prepared: { contentHash: hash }, selections: [] } as unknown as Parameters[1] +const result = { type: 'result', id: '11111111-1111-4111-8111-111111111111', contentHash: hash, available: true } +const location = () => ({ pythonPath: executable, modulesPath: directory, archiveDirectory: directory, retainedDirectory: join(directory, 'raw'), databasePath: join(directory, 'test.db') }) +beforeEach(async () => { + directory = await mkdtemp(join(tmpdir(), 'publication-child-')) + executable = join(directory, 'worker') +}) +afterEach(() => rm(directory, { recursive: true, force: true })) +async function worker(body: string) { + await writeFile(executable, `#!${process.execPath}\nsetTimeout(() => process.exit(91), 3000).unref(); let input='';process.stdin.on('data',c=>input+=c);process.stdin.on('end',()=>{${body}});`, { mode: 0o700 }) +} +const emit = (value: unknown) => `process.stdout.write(${JSON.stringify(JSON.stringify(value))});` +it('uses fixed publication arguments, a clean environment and the versioned envelope', async () => { + await worker(`if(JSON.stringify(process.argv.slice(2))!==${JSON.stringify(JSON.stringify(['-B', '-m', 'common.biometrics.publication_worker', '--database', location().databasePath, '--archive-directory', directory, '--retained-directory', location().retainedDirectory]))} || process.env.RAW_TEST_SECRET || process.env.PYTHONPATH!==${JSON.stringify(directory)} || input!==${JSON.stringify(JSON.stringify({ version: 'waveform-publication-work-v1', ...work }))})process.exit(1);${emit(result)}`) + process.env.RAW_TEST_SECRET = 'private' + try { + expect(await runRawPublicationChild(location(), work)).toEqual(result) + } + finally { delete process.env.RAW_TEST_SECRET } + expect(() => runRawPublicationChild(location(), work, { timeoutMs: 120001 })).toThrow('time budget') +}) +it.each([ + {}, { ...result, contentHash: 'b'.repeat(64) }, { ...result, id: 'bad' }, { ...result, extra: true }, +])('rejects invalid identities or unexpected fields %#', async (output) => { + await worker(emit(output)) + await expect(runRawPublicationChild(location(), work)).rejects.toThrow('invalid or incomplete') +}) +it('rejects extra output, invalid UTF-8 and unsuccessful exit after a response', async () => { + for (const suffix of ['process.stdout.write("{}");', 'process.stdout.write(Buffer.from([0xc3]));', 'process.exitCode=1;']) { + await worker(`${emit(result)}${suffix}`) + await expect(runRawPublicationChild(location(), work)).rejects.toThrow('invalid or incomplete') + } +}) +it('bounds input, output, diagnostics and execution, and cancels work', async () => { + await expect(runRawPublicationChild(location(), { ...work, padding: 'x'.repeat(10 * 1024 * 1024) } as typeof work)).rejects.toThrow('input exceeds') + await worker('process.stdout.write("x".repeat(65537));') + await expect(runRawPublicationChild(location(), work)).rejects.toThrow('output exceeded') + await worker('process.stderr.write("x".repeat(16385));') + await expect(runRawPublicationChild(location(), work)).rejects.toThrow('diagnostics exceeded') + await worker('setInterval(()=>{},1000);') + await expect(runRawPublicationChild(location(), work, { timeoutMs: 30 })).rejects.toThrow('time budget') + const controller = new AbortController() + const running = runRawPublicationChild(location(), work, { signal: controller.signal }) + controller.abort() + await expect(running).rejects.toThrow('cancelled') + await expect(runRawPublicationChild(location(), work, { signal: controller.signal })).rejects.toThrow('cancelled') + await expect(runRawPublicationChild({ ...location(), pythonPath: join(directory, 'missing') }, work)).rejects.toThrow('could not start') +}) diff --git a/src/biometrics/backtest/tests/reference-traces.test.ts b/src/biometrics/backtest/tests/reference-traces.test.ts new file mode 100644 index 000000000..a7477fe3d --- /dev/null +++ b/src/biometrics/backtest/tests/reference-traces.test.ts @@ -0,0 +1,45 @@ +// @vitest-environment node +import { expect, it } from 'vitest' +import { pageReferenceTraces } from '../reference-trace-page' +import { evaluateBacktest } from '../evaluator' +import { backtestFixture } from './fixtures/backtest' +import { previewReferenceImport } from '../../references/normalize' +const request = { id: '11111111-1111-4111-8111-111111111111', expectedContentHash: 'a'.repeat(64), cohortId: 'night', metric: 'heart_rate' as const, limit: 1 } +function fixture() { + const work = backtestFixture(), report = evaluateBacktest(work) + return { report, dataset: { ...work.dataset, contentHash: report.datasetHash } } +} +it('pages frozen source revisions without annotations or submitted identity and preserves point times', () => { + const { report, dataset } = fixture() + const row = dataset.snapshot.references[0].measurements[1] + row.original.annotation = 'private note' + row.original.subjectId = 'private subject' + dataset.snapshot.references.push({ ...dataset.snapshot.references[0], importId: 'second', revision: 2, source: 'another source' }) + const first = pageReferenceTraces(report, dataset, request) + expect(first).toMatchObject({ totalItems: 2, side: 'left', provenance: 'frozen_dataset_reference', timestampBasis: 'reference_source_time', downsampled: false, + nextCursor: { offset: 1, contentHash: request.expectedContentHash }, items: [{ source: 'fixture', importRevision: 1, temporalKind: 'point', value: 62, startMs: row.startMs }] }) + expect(JSON.stringify(first)).not.toContain('private') + expect(first.items[0]).not.toHaveProperty('original') + const second = pageReferenceTraces(report, dataset, { ...request, offset: first.nextCursor?.offset }) + expect(second.items[0].importId).toBe('second') + expect(second.nextCursor).toBeNull() +}) +it('keeps interval semantics and incompatible HRV visible without claiming a paired score', () => { + const { report, dataset } = fixture() + const start = dataset.snapshot.sessions[0].startMs + dataset.snapshot.references[0].measurements.push(...previewReferenceImport({ side: 'left', source: 'watch', subjectId: 'A', rows: [ + { recordKey: 'hrv', metric: 'hrv_rmssd', value: 30, unit: 'ms', startMs: start, endMs: start + 300000, windowMs: 300000 }, + ] }).measurements) + const page = pageReferenceTraces(report, dataset, { ...request, metric: 'window_ibi_variability_v1' }) + expect(page.items[0]).toMatchObject({ temporalKind: 'interval', metric: 'hrv_rmssd', compatibility: 'incompatible_metric_definition', endMs: start + 300000 }) + expect(page.items[0]).not.toHaveProperty('paired') +}) +it('filters scoring windows and fails on stale identities or invalid pages', () => { + const { report, dataset } = fixture() + const row = dataset.snapshot.references[0].measurements[1] + dataset.snapshot.references[0].measurements.push({ ...row, startMs: dataset.snapshot.sessions[0].endMs, endMs: dataset.snapshot.sessions[0].endMs }) + expect(pageReferenceTraces(report, dataset, request).totalItems).toBe(1) + expect(() => pageReferenceTraces(report, { ...dataset, contentHash: 'other' }, request)).toThrow('hash changed') + expect(() => pageReferenceTraces(report, dataset, { ...request, cohortId: 'absent' })).toThrow('not evaluated') + expect(() => pageReferenceTraces(report, dataset, { ...request, offset: 2 })).toThrow('cursor') +}) diff --git a/src/biometrics/backtest/tests/report-summary.test.ts b/src/biometrics/backtest/tests/report-summary.test.ts new file mode 100644 index 000000000..8b19c486f --- /dev/null +++ b/src/biometrics/backtest/tests/report-summary.test.ts @@ -0,0 +1,34 @@ +// @vitest-environment node +import { expect, it } from 'vitest' +import { evaluateBacktest } from '../evaluator' +import { dataHash } from '../../algorithms/canonical' +import { backtestFixture } from './fixtures/backtest' +import { summarizeBacktestResult } from '../report-summary' +import { pageStageTraces } from '../trace-page' + +it('preserves all scores and the retained artifact hash while omitting trace rows', () => { + const report = evaluateBacktest(backtestFixture()) + const contentHash = dataHash(report) + const descriptor = { report, available: true, contentHash } + const summary = summarizeBacktestResult(descriptor) + if (!summary) throw new Error('Expected a summary') + expect(summary).toMatchObject({ contentHash, contentHashScope: 'complete_retained_report', projection: 'scores_without_stage_trace_rows_v1' }) + const expected = structuredClone(report) as unknown as { nights: Array<{ evaluations: Array<{ windows: Array<{ blocks?: unknown }> }> }> } + for (const night of expected.nights) { + for (const evaluation of night.evaluations) { + for (const window of evaluation.windows) delete window.blocks + } + } + expect(summary.report).toEqual(expected) + expect(dataHash(report)).toBe(contentHash) + expect(pageStageTraces(report, { id: '11111111-1111-4111-8111-111111111111', expectedContentHash: contentHash, configuration: 'default' }).totalItems).toBeGreaterThan(0) + expect(JSON.stringify(summary.report).length).toBeLessThan(JSON.stringify(report).length) +}) +it('distinguishes missing and released reports without manufacturing content', () => { + expect(summarizeBacktestResult(null)).toBeNull() + expect(summarizeBacktestResult({ available: false, report: null, contentHash: 'retained' })).toMatchObject({ report: null, available: false, contentHash: 'retained' }) +}) +it('rejects unknown versions and oversize scores instead of silently truncating them', () => { + expect(() => summarizeBacktestResult({ available: true, report: { version: 'future' } })).toThrow('summary format') + expect(() => summarizeBacktestResult({ available: true, report: { version: 'processed-backtest-report-v1', nights: [], extra: 'x'.repeat(2 * 1024 * 1024) } })).toThrow('transport limit') +}) diff --git a/src/biometrics/backtest/tests/runner.test.ts b/src/biometrics/backtest/tests/runner.test.ts new file mode 100644 index 000000000..ac4b2ba22 --- /dev/null +++ b/src/biometrics/backtest/tests/runner.test.ts @@ -0,0 +1,176 @@ +import { ReferenceStore } from '../../references/store' +import { previewReferenceImport } from '../../references/normalize' +import { configHash } from '../../algorithms/canonical' +import { defaultConfig, manifestHash } from '../../algorithms/registry' +import { stageObjectiveTestPolicy } from './fixtures/stage-objective' +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { execFile } from 'node:child_process' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { promisify } from 'node:util' +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest' +import * as schema from '@/src/db/biometrics-schema' +import { DatasetStore } from '../datasets' +import { BacktestJobStore } from '../jobs' +import { workerBuildHash } from '../child' +import { executeNextBacktest } from '../runner' +import { backtestFixture } from './fixtures/backtest' +let sqlite: Database.Database +let db: ReturnType> +let jobs: BacktestJobStore +let jobId: string +let directory: string +let evaluatorBuildHash: string +beforeAll(async () => { + directory = await mkdtemp(join(tmpdir(), 'biometric-runner-test-')) + await promisify(execFile)(process.execPath, ['scripts/build/biometrics-worker.mjs', directory]) + evaluatorBuildHash = await workerBuildHash(join(directory, 'worker.cjs')) +}) +afterAll(async () => rm(directory, { recursive: true, force: true })) +beforeEach(() => { + sqlite = new Database(':memory:') + db = drizzle(sqlite, { schema }) + migrate(db, { migrationsFolder: 'src/db/biometrics-migrations' }) + const fixture = backtestFixture() + const session = fixture.dataset.snapshot.sessions[0] + db.insert(schema.sleepRecords).values({ id: 1, side: 'left', enteredBedAt: new Date(session.startMs), leftBedAt: new Date(session.endMs), sleepDurationSeconds: 600, timesExitedBed: 0 }).run() + for (const row of session.vitals) db.insert(schema.vitals).values({ side: 'left', timestamp: new Date(row.timestampMs), heartRate: row.heartRate, hrv: row.hrv, breathingRate: row.breathingRate }).run() + const dataset = new DatasetStore(db).create({ side: 'left', subjectId: 'A', sessionIds: [1], referenceImportIds: [], timezone: 'UTC', idempotencyKey: 'run' }) + jobs = new BacktestJobStore(db) + const experiment = jobs.createExperiment(dataset.id, fixture.job.protocol) + jobId = jobs.enqueue({ evaluatorBuildHash, experimentId: experiment.id, configs: fixture.job.configs, + idempotencyKey: 'run', split: 'tuning' }).id +}) +afterEach(() => sqlite.close()) +const options = { owner: 'integration-test', get workerPath() { + return join(directory, 'worker.cjs') +}, admit: async () => true } + +describe('durable replay execution', () => { + it('runs retained data through a child and publishes an integrity-checked result without changing canonical observations', async () => { + const before = sqlite.prepare('SELECT * FROM vitals').all() + expect(await executeNextBacktest(db, options)).toEqual({ state: 'succeeded', jobId }) + expect(jobs.result(jobId)?.report).toMatchObject({ version: 'processed-backtest-report-v1', status: 'insufficient_evidence', nights: [{ uniqueObservationCount: 10 }] }) + expect(jobs.result(jobId)?.report).toMatchObject({ jobOperations: { ingestion: { allChecksMeasured: false, samples: 0, maximumObservedAgeMs: null } } }) + expect(sqlite.prepare('SELECT * FROM vitals').all()).toEqual(before) + expect(await executeNextBacktest(db, options)).toEqual({ state: 'idle' }) + }) + it('retains supervisor observations in the committed report and marks legacy boolean admission unmeasured', async () => { + let checks = 0 + expect(await executeNextBacktest(db, { ...options, admit: async () => { + checks++ + return true + }, + admissionReceipt: () => ({ allowed: true, sample: { qualificationId: 'test-policy', checkedAtMs: Date.now(), ingestionAgeMs: checks * 100 } }), + })).toEqual({ state: 'succeeded', jobId }) + expect(jobs.result(jobId)?.report).toMatchObject({ jobOperations: { version: 'backtest-job-operations-v1', + ingestion: { allChecksMeasured: true, checks: 2, samples: 2, maximumObservedAgeMs: 200, qualificationId: 'test-policy' } } }) + }) + it('finalizes retained stage objectives after the real scoring child completes', async () => { + const datasetId = jobs.getExperiment(jobs.get(jobId).experimentId).datasetId + jobs.cancel(jobId) + const fixture = backtestFixture() + const experiment = jobs.createExperiment(datasetId, { ...fixture.job.protocol, stageObjectivePolicy: stageObjectiveTestPolicy }) + const staged = jobs.enqueue({ evaluatorBuildHash, experimentId: experiment.id, idempotencyKey: 'stage-objective', split: 'tuning', + configs: [...fixture.job.configs, { ...fixture.job.configs[0], label: 'candidate' }] }) + expect(await executeNextBacktest(db, options)).toEqual({ state: 'succeeded', jobId: staged.id }) + expect(jobs.result(staged.id)?.report).toMatchObject({ status: 'no_material_change', stageObjectiveAssessments: [ + { baseline: 'default', status: 'insufficient_evidence', evidence: { operational: null } }, + { baseline: 'current', status: 'insufficient_evidence', evidence: { operational: null } }, + ] }) + }) + it.each([['wake', 'improved'], ['deep', 'regressed']] as const)('publishes %s candidate evidence as %s through the real scoring worker', async (stage, status) => { + jobs.cancel(jobId) + const first = 1791673200000 + const ids = Array.from({ length: 12 }, (_, index) => index + 10) + const rows = ids.flatMap((id, index) => { + const start = first + id * 86400000, end = start + 600000 + db.insert(schema.sleepRecords).values({ id, side: 'left', enteredBedAt: new Date(start), leftBedAt: new Date(end), sleepDurationSeconds: 600, timesExitedBed: 0 }).run() + for (let minute = 0; minute < 10; minute++) db.insert(schema.vitals).values({ side: 'left', timestamp: new Date(start + minute * 60000), heartRate: 60, hrv: 30, breathingRate: 14 }).run() + const middle = start + 360000 + index % 3 * 60000 + return [{ recordKey: `${id}-wake`, metric: 'sleep_stages', value: 'wake', unit: 'stage', startMs: start, endMs: middle }, + { recordKey: `${id}-light`, metric: 'sleep_stages', value: 'light', unit: 'stage', startMs: middle, endMs: end }] + }) + const input = { side: 'left' as const, source: 'synthetic-runner-test', subjectId: 'fixture', rows } + const reference = new ReferenceStore(db).commit({ import: input, previewHash: previewReferenceImport(input).summary.contentHash }) + const dataset = new DatasetStore(db).create({ side: 'left', subjectId: 'fixture', sessionIds: ids, referenceImportIds: [reference.id], timezone: 'UTC', idempotencyKey: 'qualified-test' }) + const policy = { ...stageObjectiveTestPolicy, requiredStages: ['wake' as const], maximumMacroF1Regression: 0.5, + operational: { maximumWallMs: 60000, maximumPeakRssBytes: 1000000000, maximumIngestionAgeMs: 1000, + observations: { version: 'sampled-process-group-v1' as const, maximumMemoryGapMs: 60000, maximumIngestionGapMs: 60000 } } } + const protocol = { ...backtestFixture().job.protocol, assumedStageCalibrationQuality: 1, stageObjectivePolicy: policy, + nightUncertainty: { version: 'paired-night-percentile-v1' as const, seed: 'runner-qualified', resamples: 200, minimumNights: 3 } } + const experiment = jobs.createExperiment(dataset.id, protocol) + const candidate = structuredClone(defaultConfig) + candidate.sleep_stages.ruleset.rules = [{ stage, when: [] }] + const configurations = [...backtestFixture().job.configs, { label: 'candidate' as const, revisionId: '33333333-3333-4333-8333-333333333333', + resolved: candidate, resolvedHash: configHash(candidate), manifestHash }] + const request = { evaluatorBuildHash, experimentId: experiment.id, configs: configurations, split: 'holdout' as const } + const staged = jobs.enqueue({ ...request, idempotencyKey: 'qualified' }) + const registry = join(directory, 'promotion-qualifications.json') + await writeFile(registry, JSON.stringify({ version: 'biometric-promotion-qualifications-v1', entries: [{ + qualificationId: policy.qualificationId, manifestHash, evaluatorBuildHash, protocol, + fixtureEvidenceHash: 'a'.repeat(64), podEvidenceHash: 'b'.repeat(64), waveformContext: null, + }] })) + const before = sqlite.prepare('SELECT * FROM vitals').all() + const measuredOptions = { ...options, admissionReceipt: () => ({ allowed: true, + sample: { qualificationId: policy.qualificationId, checkedAtMs: Date.now(), ingestionAgeMs: 10 }, + memory: { version: 'process-group-rss-observations-v1' as const, scope: 'sum_rss_of_replay_process_group' as const, + attempts: 2, samples: 2, allSamplesAvailable: true, peakObservedRssBytes: 1000, maximumObservationGapMs: 1, intervalMs: 1000 as const, exactKernelPeak: false as const } }) } + try { + expect(await executeNextBacktest(db, measuredOptions)).toEqual({ state: 'succeeded', jobId: staged.id }) + expect(jobs.result(staged.id)?.report).toMatchObject({ status, + stageObjectiveAssessments: [{ status, designated: true }, { status, designated: true }] }) + if (status === 'improved') expect(jobs.result(staged.id)?.report).toMatchObject({ evidence: 'qualified_reference_agreement' }) + expect(sqlite.prepare('SELECT * FROM vitals').all()).toEqual(before) + const reused = jobs.enqueue({ ...request, idempotencyKey: 'reused' }) + expect(await executeNextBacktest(db, measuredOptions)).toEqual({ state: 'succeeded', jobId: reused.id }) + expect(jobs.result(reused.id)?.report).toMatchObject({ status: 'insufficient_evidence' }) + } + finally { await rm(registry, { force: true }) } + }) + + it('refuses to execute a different worker bundle after enqueue', async () => { + sqlite.prepare('UPDATE biometric_backtest_jobs SET input = json_set(input, \'$.evaluatorBuildHash\', ?)').run('0'.repeat(64)) + expect(await executeNextBacktest(db, options)).toEqual({ state: 'stopped', jobId }) + expect(jobs.get(jobId).state).toBe('failed') + expect(jobs.result(jobId)).toBeNull() + }) + + it('defers work before claim when resource admission is denied or shutdown arrives', async () => { + expect(await executeNextBacktest(db, { ...options, admit: async () => false })).toEqual({ state: 'deferred' }) + expect(jobs.get(jobId).attempt).toBe(0) + const controller = new AbortController() + expect(await executeNextBacktest(db, { ...options, signal: controller.signal, admit: async () => { + controller.abort() + return true + } })).toEqual({ state: 'deferred' }) + expect(jobs.get(jobId).attempt).toBe(0) + }) + it('interrupts a running child when health declines and never publishes its result', async () => { + const stalledPath = join(directory, 'stalled.cjs') + await writeFile(stalledPath, 'process.stdin.resume(); setInterval(() => {}, 1000)') + const hash = await workerBuildHash(stalledPath) + sqlite.prepare('UPDATE biometric_backtest_jobs SET input = json_set(input, \'$.evaluatorBuildHash\', ?)').run(hash) + let checks = 0 + const result = await executeNextBacktest(db, { ...options, workerPath: stalledPath, admit: async () => ++checks === 1 }) + expect(result).toEqual({ state: 'stopped', jobId }) + expect(checks).toBe(2) + expect(jobs.get(jobId)).toMatchObject({ state: 'interrupted', error: 'Resource admission withdrawn' }) + expect(jobs.result(jobId)).toBeNull() + }, 10000) + + it('withholds publication when the final resource check fails', async () => { + let checks = 0 + expect(await executeNextBacktest(db, { ...options, admit: async () => { + if (++checks === 1) return true + throw new Error('Health collector unavailable') + } })).toEqual({ state: 'stopped', jobId }) + expect(checks).toBe(2) + expect(jobs.get(jobId)).toMatchObject({ state: 'interrupted', error: 'Resource admission withdrawn' }) + expect(jobs.result(jobId)).toBeNull() + }) +}) diff --git a/src/biometrics/backtest/tests/splits.test.ts b/src/biometrics/backtest/tests/splits.test.ts new file mode 100644 index 000000000..0e430f95a --- /dev/null +++ b/src/biometrics/backtest/tests/splits.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from 'vitest' +import { assignWholeNightSplits } from '../splits' +import { describeCoverage } from '../coverage' +const session = (id: number, start: string, end: string) => ({ id, startMs: Date.parse(start), endMs: Date.parse(end) }) + +describe('frozen whole-night split assignment', () => { + it('keeps naps and overnight sessions in the same local-noon cohort across DST', () => { + const result = assignWholeNightSplits([ + session(1, '2026-11-01T00:00:00-07:00', '2026-11-01T07:00:00-08:00'), + session(2, '2026-10-31T14:00:00-07:00', '2026-10-31T15:00:00-07:00'), + session(3, '2026-11-02T00:00:00-08:00', '2026-11-02T08:00:00-08:00'), + session(4, '2026-11-03T00:00:00-08:00', '2026-11-03T08:00:00-08:00'), + ], 'America/Los_Angeles') + expect(result.assignments.map(group => [group.split, group.sessionIds])).toEqual([ + ['tuning', [2, 1]], ['validation', [3]], ['holdout', [4]], + ]) + expect(result.assignments[0].nightLabels).toEqual(['2026-10-31']) + expect(result.evidence).toBe('partitioned_not_validated') + }) + it('joins sessions bridging adjacent nights instead of leaking correlated data', () => { + const result = assignWholeNightSplits([ + session(1, '2026-10-01T23:00:00Z', '2026-10-02T13:00:00Z'), + session(2, '2026-10-02T12:00:00Z', '2026-10-02T15:00:00Z'), + session(3, '2026-10-02T23:00:00Z', '2026-10-03T08:00:00Z'), + ], 'UTC') + expect(result.assignments).toHaveLength(1) + expect(result.assignments[0].sessionIds).toEqual([1, 2, 3]) + expect(result.evidence).toBe('descriptive_only') + }) + it('uses all small datasets descriptively and rejects malformed or duplicate sessions', () => { + const rows = [session(1, '2026-10-01T23:00:00Z', '2026-10-02T08:00:00Z')] + expect(assignWholeNightSplits(rows, 'UTC').assignments[0].split).toBe('tuning') + expect(() => assignWholeNightSplits([...rows, ...rows], 'UTC')).toThrow('Duplicate') + expect(() => assignWholeNightSplits(rows, 'not-a-timezone')).toThrow() + expect(() => assignWholeNightSplits([{ ...rows[0], endMs: rows[0].startMs }], 'UTC')).toThrow() + }) +}) + +describe('coverage gaps', () => { + it('reports leading, internal and trailing gaps without substituting zeros', () => { + expect(describeCoverage([100, 100, 200, 700], 0, 1000, 100)).toMatchObject({ count: 3, duplicateTimestamps: 1, + firstMs: 100, lastMs: 700, gapDurationMs: 700, gaps: [{ startMs: 0, endMs: 100 }, { startMs: 300, endMs: 700 }, { startMs: 800, endMs: 1000 }] }) + expect(describeCoverage([], 0, 1000, 100)).toMatchObject({ firstMs: null, lastMs: null, gapDurationMs: 1000 }) + }) +}) diff --git a/src/biometrics/backtest/tests/stage-comparison.test.ts b/src/biometrics/backtest/tests/stage-comparison.test.ts new file mode 100644 index 000000000..f32f4fb8c --- /dev/null +++ b/src/biometrics/backtest/tests/stage-comparison.test.ts @@ -0,0 +1,40 @@ +// @vitest-environment node +import { expect, it } from 'vitest' +import { previewReferenceImport } from '../../references/normalize' +import { compareStageOutputs } from '../stage-comparison' +const references = previewReferenceImport({ side: 'left', subjectId: 'fixture', source: 'test', rows: [ + { recordKey: 'first', metric: 'sleep_stages', value: 'light', unit: 'stage', startMs: 1791673200000, endMs: 1791673210000 }, + { recordKey: 'second', metric: 'sleep_stages', value: 'deep', unit: 'stage', startMs: 1791673210000, endMs: 1791673220000 }, +] }).measurements +it('exposes apparent gains from dropping difficult intervals', () => { + const comparison = compareStageOutputs([{ start: 1791673200000, end: 1791673220000, stage: 'light' }], + [{ start: 1791673200000, end: 1791673210000, stage: 'light' }], references, 1791673200000, 1791673220000) + expect(comparison.ownSupport.baseline.agreement).toBe(0.5) + expect(comparison.ownSupport.candidate.agreement).toBe(1) + expect(comparison.differences).toMatchObject({ ownAgreement: 0.5, commonAgreement: 0, referenceCoverage: -0.5, outputCoverage: -0.5, pairedDurationMs: -10000 }) + expect(comparison.coverageReduced).toBe(true) + expect(comparison.commonSupport.baseline.pairedDurationMs).toBe(10000) + expect(comparison.commonSupport.candidate.pairedDurationMs).toBe(10000) +}) +it('measures genuine paired improvement without penalizing unchanged support', () => { + const comparison = compareStageOutputs([{ start: 1791673200000, end: 1791673220000, stage: 'light' }], + [{ start: 1791673200000, end: 1791673210000, stage: 'light' }, { start: 1791673210000, end: 1791673220000, stage: 'deep' }], references, 1791673200000, 1791673220000) + expect(comparison.differences).toMatchObject({ commonAgreement: 0.5, referenceCoverage: 0, outputCoverage: 0 }) + expect(comparison.coverageReduced).toBe(false) + expect(comparison.interpretation).toBe('descriptive_agreement_not_validated_improvement') +}) +it('keeps no common support unknown instead of reporting zero error', () => { + const comparison = compareStageOutputs([{ start: 1791673200000, end: 1791673210000, stage: 'light' }], + [{ start: 1791673210000, end: 1791673220000, stage: 'deep' }], references, 1791673200000, 1791673220000) + expect(comparison.commonSupport.baseline.pairedDurationMs).toBe(0) + expect(comparison.differences.commonAgreement).toBeNull() + expect(comparison.differences.commonMacroF1).toBeNull() +}) +it('excludes conflicting references identically on both sides and never fills gaps', () => { + const conflict = { ...references[0], recordKey: 'conflict', stage: 'rem' as const } + const comparison = compareStageOutputs([{ start: 1791673200000, end: 1791673220000, stage: 'light' }], + [{ start: 1791673200000, end: 1791673205000, stage: 'light' }, { start: 1791673215000, end: 1791673220000, stage: 'deep' }], [...references, conflict], 1791673200000, 1791673220000) + expect(comparison.commonSupport.baseline.pairedDurationMs).toBe(5000) + expect(comparison.commonSupport.candidate.pairedDurationMs).toBe(5000) + expect(comparison.differences.commonAgreement).toBe(1) +}) diff --git a/src/biometrics/backtest/tests/stage-objective.test.ts b/src/biometrics/backtest/tests/stage-objective.test.ts new file mode 100644 index 000000000..536f31ccc --- /dev/null +++ b/src/biometrics/backtest/tests/stage-objective.test.ts @@ -0,0 +1,135 @@ +import { stageObjectiveTestPolicy as policy } from './fixtures/stage-objective' +// @vitest-environment node +import { expect, it } from 'vitest' +import { assessStageObjective, type StageObjectiveEvidence } from '../stage-objective' +import { experimentProtocolSchema } from '../contracts' +import { configHash } from '../../algorithms/canonical' +import { finalizeStageObjectiveReport } from '../objective-finalization' +import { evaluateBacktest } from '../evaluator' +import { backtestFixture } from './fixtures/backtest' +const evidence = (): StageObjectiveEvidence => ({ split: 'holdout', holdoutUse: 1, intervalLower: 0.125, + operational: { wallMs: 1000, peakRssBytes: 1000, maximumIngestionAgeMs: 1000 }, nights: ['a', 'b', 'c'].map(cohortId => ({ + cohortId, pairedDurationMs: 800, agreementDifference: 0.125, macroF1Difference: 0.125, candidateOutputCoverage: 0.9, + baselineOutputMs: 1000, commonOutputMs: 900, + perStage: policy.requiredStages.map(stage => ({ stage, referenceMs: 200, f1Difference: 0.125 })), + })) }) +it('awards reference agreement improvement only with designated independent supported evidence at exact boundaries', () => { + expect(assessStageObjective(policy, evidence(), policy).status).toBe('improved') + expect(assessStageObjective(policy, evidence(), null).missingEvidence).toEqual(['policy_not_designated']) + for (const patch of [{ holdoutUse: 2 }, { split: 'validation' as const }, { intervalLower: null }, { operational: null }]) { + expect(assessStageObjective(policy, { ...evidence(), ...patch }, policy).status).toBe('insufficient_evidence') + } + expect(assessStageObjective(policy, { ...evidence(), intervalLower: 0.124 }, policy).status).toBe('no_material_change') +}) +it('does not hide coverage, macro-F1 or required-stage regressions behind improved overall agreement', () => { + const data = evidence() + data.nights[0].commonOutputMs = 899 + data.nights[0].candidateOutputCoverage = 0.89 + data.nights[0].macroF1Difference = -0.126 + data.nights[0].perStage[3].f1Difference = -0.126 + expect(assessStageObjective(policy, data, policy)).toMatchObject({ status: 'regressed', failedThresholds: ['output_coverage', 'coverage_loss', 'macro_f1_regression', 'stage_regression:rem'] }) +}) +it('requires supported stages on every night and bounds night-level regressions', () => { + const data = evidence() + data.nights[0].perStage[3].referenceMs-- + data.nights[0].pairedDurationMs-- + expect(assessStageObjective(policy, data, policy).missingEvidence).toContain('stage_support:rem') + const regressed = evidence() + regressed.nights[0].agreementDifference = -0.126 + regressed.nights[1].agreementDifference = -0.01 + expect(assessStageObjective(policy, regressed, policy).failedThresholds).toEqual(['night_regression', 'regressed_night_fraction']) +}) +it('rejects duplicate or inconsistent support and freezes stage thresholds only with stage replay', () => { + const data = evidence() + data.nights.push(data.nights[0]) + expect(() => assessStageObjective(policy, data, policy)).toThrow('evidence') + const invalid = evidence() + invalid.nights[0].commonOutputMs = 1001 + expect(() => assessStageObjective(policy, invalid, policy)).toThrow('evidence') + const protocol = { ...backtestFixture().job.protocol, stageObjectivePolicy: policy } + expect(experimentProtocolSchema.safeParse(protocol).success).toBe(true) + expect(experimentProtocolSchema.safeParse({ ...protocol, mode: 'comparison' }).success).toBe(false) + expect(experimentProtocolSchema.safeParse({ ...protocol, mode: 'signal_replay', waveformStages: false }).success).toBe(false) + expect(configHash(protocol)).not.toBe(configHash({ ...protocol, stageObjectivePolicy: { ...policy, minimumAgreementImprovement: 0.25 } })) +}) +it('scores stage evidence in retained replay and replaces claimed operational evidence at publication', () => { + const work = backtestFixture() + const protocol = { ...work.job.protocol, stageObjectivePolicy: policy } + const report = evaluateBacktest({ ...work, job: { ...work.job, protocol, protocolHash: configHash(protocol), + configs: [...work.job.configs, { ...work.job.configs[0], label: 'candidate' }] } }) + expect(report.stageObjectiveAssessments).toHaveLength(2) + expect(report.stageObjectiveAssessments[0]).toMatchObject({ status: 'insufficient_evidence', designated: false, + evidence: { nights: [{ cohortId: 'night', commonOutputMs: 600000 }], operational: null } }) + const claimed = { stageObjectiveAssessments: ['default', 'current'].map(baseline => ({ baseline, candidate: 'candidate', baselineHash: baseline, candidateHash: 'candidate', + ...assessStageObjective(policy, evidence(), policy) })) } + expect(finalizeStageObjectiveReport(claimed, policy, null, policy).stageObjectiveAssessments).toEqual(expect.arrayContaining([expect.objectContaining({ status: 'insufficient_evidence', missingEvidence: ['operational_evidence_unavailable'] })])) +}) + +it('retains specific missing evidence for sparse nights and unavailable metrics', () => { + const cases: [string, (data: StageObjectiveEvidence) => void][] = [ + ['independent_holdout_required', (data) => { data.holdoutUse = 2 }], + ['uncertainty_unavailable', (data) => { data.intervalLower = null }], + ['coverage_unavailable', (data) => { data.nights[0].candidateOutputCoverage = null }], + ['macro_f1_unavailable', (data) => { data.nights[0].macroF1Difference = null }], + ['stage_support:rem', (data) => { data.nights[0].perStage[3].f1Difference = null }], + ] + for (const [reason, change] of cases) { + const data = evidence() + change(data) + expect(assessStageObjective(policy, data, policy)).toMatchObject({ status: 'insufficient_evidence', missingEvidence: [reason], failedThresholds: [] }) + } + for (const change of [ + (data: StageObjectiveEvidence) => { data.nights[0].agreementDifference = null }, + (data: StageObjectiveEvidence) => { + data.nights[0].pairedDurationMs-- + data.nights[0].perStage[0].referenceMs-- + }, + ]) { + const data = evidence() + change(data) + expect(assessStageObjective(policy, data, policy).missingEvidence).toEqual(expect.arrayContaining(['insufficient_supported_nights', 'unsupported_nights'])) + } + const empty = evidence() + empty.nights = [] + expect(assessStageObjective(policy, empty, policy)).toMatchObject({ status: 'insufficient_evidence', equalNightMeanAgreementDifference: null, missingEvidence: ['insufficient_supported_nights'] }) +}) + +it('requires actual mean improvement and preserves inclusive regression limits', () => { + const data = evidence() + data.nights[0].agreementDifference = -0.125 + data.nights[1].agreementDifference = 0 + data.nights[2].agreementDifference = 0.5 + data.nights[0].macroF1Difference = -0.125 + data.nights[0].perStage[3].f1Difference = -0.125 + const exact = { ...policy, maximumRegressedNightFraction: 1 / 3 } + expect(assessStageObjective(exact, data, exact)).toMatchObject({ status: 'improved', failedThresholds: [], equalNightMeanAgreementDifference: 0.125 }) + data.nights[2].agreementDifference = 0.499 + expect(assessStageObjective(exact, data, exact).status).toBe('no_material_change') +}) + +it('rejects duplicated stages, inconsistent paired durations, and altered designation', () => { + const duplicate = evidence() + duplicate.nights[0].perStage[3].stage = 'wake' + expect(() => assessStageObjective(policy, duplicate, policy)).toThrow('Invalid stage objective evidence') + const unpaired = evidence() + unpaired.nights[0].commonOutputMs = 799 + expect(() => assessStageObjective(policy, unpaired, policy)).toThrow('Invalid stage objective evidence') + const mismatch = evidence() + mismatch.nights[0].perStage[0].referenceMs++ + expect(() => assessStageObjective(policy, mismatch, policy)).toThrow('Invalid stage objective evidence') + expect(() => assessStageObjective({ ...policy, requiredStages: ['wake', 'wake'] }, evidence(), policy)).toThrow('Duplicate required stages') + expect(assessStageObjective(policy, evidence(), { ...policy, qualificationId: 'other' }).missingEvidence).toEqual(['policy_not_designated']) +}) + +it('reports absent stage support and zero baseline coverage without inventing a score', () => { + const data = evidence() + data.nights[0].baselineOutputMs = 0 + data.nights[0].commonOutputMs = 0 + data.nights[0].pairedDurationMs = 0 + data.nights[0].agreementDifference = null + data.nights[0].macroF1Difference = null + data.nights[0].perStage = [] + expect(assessStageObjective(policy, data, policy)).toMatchObject({ status: 'insufficient_evidence', failedThresholds: [], + missingEvidence: ['insufficient_supported_nights', 'unsupported_nights', 'coverage_unavailable', 'macro_f1_unavailable', + 'stage_support:wake', 'stage_support:light', 'stage_support:deep', 'stage_support:rem'] }) +}) diff --git a/src/biometrics/backtest/tests/traces.test.ts b/src/biometrics/backtest/tests/traces.test.ts new file mode 100644 index 000000000..8e035669c --- /dev/null +++ b/src/biometrics/backtest/tests/traces.test.ts @@ -0,0 +1,27 @@ +// @vitest-environment node +import { expect, it } from 'vitest' +import { pageStageTraces } from '../traces' +const request = { id: '11111111-1111-4111-8111-111111111111', expectedContentHash: 'a'.repeat(64), configuration: 'candidate' as const, limit: 2 } +const report = { version: 'processed-backtest-report-v1', protocol: { mode: 'stage_replay', clockOffsetMs: 60000 }, nights: ['one', 'two'].map((cohortId, index) => ({ + cohortId, evaluations: [{ label: 'candidate', configHash: 'b'.repeat(64), revisionId: null, windows: [{ startMs: index * 300, endMs: index * 300 + 300, + blocks: [0, 1, 2].map(i => ({ start: index * 300 + i * 100, end: index * 300 + (i + 1) * 100, stage: 'light' })) }] }], +})) } +it('pages every retained interval exactly once with a stable hash cursor and no timestamp offset', () => { + const before = JSON.stringify(report) + const first = pageStageTraces(report, request) + const second = pageStageTraces(report, { ...request, offset: first.nextCursor?.offset }) + const third = pageStageTraces(report, { ...request, offset: second.nextCursor?.offset }) + expect(first).toMatchObject({ totalItems: 6, downsampled: false, clockOffsetMs: 60000, nextCursor: { offset: 2, contentHash: request.expectedContentHash } }) + expect([...first.items, ...second.items, ...third.items].map(row => row.startMs)).toEqual([0, 100, 200, 300, 400, 500]) + expect(third.nextCursor).toBeNull() + expect(pageStageTraces(report, { ...request, offset: 6 }).items).toEqual([]) + expect(JSON.stringify(report)).toBe(before) +}) +it('filters the cohort before counting and rejects out-of-range or missing selections', () => { + expect(pageStageTraces(report, { ...request, cohortId: 'two' })).toMatchObject({ totalItems: 3, items: [{ cohortId: 'two' }, { cohortId: 'two' }] }) + expect(() => pageStageTraces(report, { ...request, cohortId: 'missing' })).toThrow('not evaluated') + expect(() => pageStageTraces(report, { ...request, configuration: 'default' })).toThrow('not evaluated') + expect(() => pageStageTraces(report, { ...request, offset: 7 })).toThrow('cursor') + expect(() => pageStageTraces(report, { ...request, limit: 501 })).toThrow() + expect(() => pageStageTraces({ ...report, protocol: { ...report.protocol, mode: 'comparison' } }, request)).toThrow('did not produce') +}) diff --git a/src/biometrics/backtest/tests/uncertainty.test.ts b/src/biometrics/backtest/tests/uncertainty.test.ts new file mode 100644 index 000000000..a617374e0 --- /dev/null +++ b/src/biometrics/backtest/tests/uncertainty.test.ts @@ -0,0 +1,42 @@ +// @vitest-environment node +import { expect, it } from 'vitest' +import { nightUncertainty, type NightBootstrapPolicy } from '../uncertainty' +import { experimentProtocolSchema } from '../contracts' +import { configHash } from '../../algorithms/canonical' +const policy: NightBootstrapPolicy = { version: 'paired-night-percentile-v1', seed: 'fixture', resamples: 200, minimumNights: 3 } +const nights = [{ cohortId: 'a', difference: -0.5 }, { cohortId: 'b', difference: 0.2 }, { cohortId: 'c', difference: 0.8 }] +it('resamples paired nights reproducibly irrespective of input ordering', () => { + const result = nightUncertainty(nights, policy) + expect(result.status).toBe('estimated') + expect(result).toEqual(nightUncertainty([...nights].reverse(), policy)) + expect(result.interval?.lower).toBeLessThan(result.estimate ?? 0) + expect(result.interval?.upper).toBeGreaterThan(result.estimate ?? 0) + expect(result).toMatchObject({ scoredNights: 3, omittedNights: 0, samplingUnit: 'whole_night_cohort' }) +}) +it('does not fabricate precision from missing or identical nights', () => { + expect(nightUncertainty(nights)).toMatchObject({ status: 'not_requested', interval: null }) + expect(nightUncertainty([...nights.slice(0, 2), { cohortId: 'missing', difference: null }], policy)) + .toMatchObject({ status: 'insufficient_nights', scoredNights: 2, omittedNights: 1, interval: null }) + expect(nightUncertainty(nights.map(row => ({ ...row, difference: 0.5 })), policy)).toMatchObject({ status: 'degenerate', interval: null }) +}) +it('bounds computation and rejects invalid or duplicated evidence', () => { + expect(() => nightUncertainty([...nights, nights[0]], policy)).toThrow('Invalid paired-night') + expect(() => nightUncertainty([{ cohortId: 'bad', difference: NaN }], policy)).toThrow('Invalid paired-night') + expect(() => nightUncertainty(nights, { ...policy, resamples: 5001 })).toThrow() + expect(() => nightUncertainty(nights, { ...policy, minimumNights: 2 })).toThrow() +}) +it('pins uncertainty parameters in the experiment protocol hash', () => { + const base = { version: 'descriptive-evaluation-v1', mode: 'stage_replay', assumedStageCalibrationQuality: 0, pointToleranceMs: 30000, clockOffsetMs: 0 } + const first = experimentProtocolSchema.parse({ ...base, nightUncertainty: policy }) + const second = experimentProtocolSchema.parse({ ...base, nightUncertainty: { ...policy, seed: 'different' } }) + expect(configHash(first)).not.toBe(configHash(second)) + expect(experimentProtocolSchema.parse(base)).toEqual(base) +}) + +it('matches a separately computed Python SHA-256/quantile golden', () => { + const fixture = Array.from({ length: 12 }, (_, i) => ({ cohortId: String(i).padStart(2, '0'), difference: ((i * 7) % 11 - 5) / 10 })) + const result = nightUncertainty(fixture, policy) + expect(result.estimate).toBeCloseTo(-0.041666666666666664, 14) + expect(result.interval?.lower).toBeCloseTo(-0.22520833333333334, 14) + expect(result.interval?.upper).toBeCloseTo(0.125, 14) +}) diff --git a/src/biometrics/backtest/tests/waveform-build.test.ts b/src/biometrics/backtest/tests/waveform-build.test.ts new file mode 100644 index 000000000..87a780718 --- /dev/null +++ b/src/biometrics/backtest/tests/waveform-build.test.ts @@ -0,0 +1,39 @@ +// @vitest-environment node +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' +import { afterEach, beforeEach, expect, it } from 'vitest' +import { probeWaveformBuild, waveformEvaluatorBuildIdentity } from '../waveform-build' +let directory: string +const location = { pythonPath: resolve('modules/piezo-processor/.venv/bin/python'), modulesPath: resolve('modules') } +beforeEach(async () => { + directory = await mkdtemp(join(tmpdir(), 'waveform-build-')) +}) +afterEach(() => rm(directory, { recursive: true, force: true })) +it('verifies a real Python identity and binds both shipped Node bundles', async () => { + const coordinator = join(directory, 'coordinator.cjs'), worker = join(directory, 'worker.cjs') + await writeFile(coordinator, 'orchestration-v1') + await writeFile(worker, 'scoring-v1') + const first = await waveformEvaluatorBuildIdentity(coordinator, worker, location) + expect(first.python).toEqual(await probeWaveformBuild(location)) + expect(first.python.identity.sources).toHaveProperty('common/biometrics/replay_worker.py') + expect(first).toEqual(await waveformEvaluatorBuildIdentity(coordinator, worker, location)) + await writeFile(coordinator, 'orchestration-v2') + const second = await waveformEvaluatorBuildIdentity(coordinator, worker, location) + expect(second.hash).not.toBe(first.hash) + expect(second.python.hash).toBe(first.python.hash) + await writeFile(worker, 'scoring-v2') + expect((await waveformEvaluatorBuildIdentity(coordinator, worker, location)).hash).not.toBe(second.hash) +}) +it('rejects missing interpreters, invalid output and forged build hashes', async () => { + await expect(probeWaveformBuild({ ...location, pythonPath: join(directory, 'missing') })).rejects.toThrow('unavailable') + const fixture = join(directory, 'fixture') + await writeFile(fixture, `#!${process.execPath}\nprocess.stdout.write('{}');`, { mode: 0o700 }) + await expect(probeWaveformBuild({ ...location, pythonPath: fixture })).rejects.toThrow() + const build = await probeWaveformBuild(location) + await writeFile(fixture, `#!${process.execPath}\nprocess.stdout.write(${JSON.stringify(JSON.stringify({ ...build, hash: '0'.repeat(64) }))});`, { mode: 0o700 }) + await expect(probeWaveformBuild({ ...location, pythonPath: fixture })).rejects.toThrow('integrity mismatch') + const controller = new AbortController() + controller.abort() + await expect(probeWaveformBuild(location, controller.signal)).rejects.toThrow('unavailable') +}) diff --git a/src/biometrics/backtest/tests/waveform-child.test.ts b/src/biometrics/backtest/tests/waveform-child.test.ts new file mode 100644 index 000000000..184e3a4d0 --- /dev/null +++ b/src/biometrics/backtest/tests/waveform-child.test.ts @@ -0,0 +1,161 @@ +// @vitest-environment node +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import { runWaveformChild } from '../waveform-child' + +let directory: string +let executable: string +const hash = 'a'.repeat(64) +const reading = { side: 'left', timestampMs: 1000, heartRate: 60, hrvIndex: null, hrvMetric: 'window_ibi_variability_v1', + breathingRate: 12, qualityScore: 0.8, flags: null, hrRaw: 60, sourceSide: 'left', configHash: hash, limitations: [] } +const result = { version: 'retained-waveform-run-v1', build_hash: hash, decoded_records: 1, raw_records: 1, + counts: { records: 1, accepted_frames: 1, empty_channel_frames: 0, pump_gated_frames: 0, preroll_outputs: 0, scored_outputs: 1 }, + limitations: [], config_hashes: { left: hash, right: hash }, artifact_hashes: [hash] } +const events = [{ type: 'reading', reading }, { type: 'result', result }] +const location = () => ({ pythonPath: executable, modulesPath: directory, retainedDirectory: directory }) +beforeEach(async () => { + directory = await mkdtemp(join(tmpdir(), 'waveform-child-')) + executable = join(directory, 'fixture-worker') +}) +afterEach(() => rm(directory, { recursive: true, force: true })) +async function worker(body: string) { + await writeFile(executable, `#!${process.execPath}\nsetTimeout(() => process.exit(91), 3000).unref(); process.stdin.resume(); process.stdin.on('end', () => { ${body} });\n`, { mode: 0o700 }) +} +const emit = (value: unknown) => `process.stdout.write(${JSON.stringify(JSON.stringify(value) + '\n')});` + +it('passes a trusted CPU limit to the fixed worker and rejects invalid limits before spawning', async () => { + await worker(`if (process.argv.slice(-2).join(' ') !== '--cpu-seconds 1800') process.exit(1); ${events.map(emit).join('')}`) + expect(await runWaveformChild(location(), {}, { cpuSeconds: 1800 })).toEqual({ readings: [reading], run: result }) + for (const cpuSeconds of [0, -1, 1.5, NaN, Infinity, 3601]) { + await expect(runWaveformChild(location(), {}, { cpuSeconds })).rejects.toThrow('Invalid waveform CPU budget') + } +}) + +it('returns readings only after a valid success event and zero exit with a clean environment', async () => { + await worker(`if (process.env.WAVEFORM_TEST_SECRET + || JSON.stringify(process.argv.slice(2)) !== ${JSON.stringify(JSON.stringify(['-B', '-m', 'common.biometrics.replay_worker', '--retained-directory', directory]))} + || process.env.PYTHONPATH !== ${JSON.stringify(directory)} || process.env.NODE_ENV !== 'production' + || process.env.LANG !== 'C.UTF-8' || process.env.TZ !== 'UTC' + || ['OPENBLAS_NUM_THREADS','OMP_NUM_THREADS','MKL_NUM_THREADS','NUMEXPR_NUM_THREADS'].some(key => process.env[key] !== '1')) process.exit(1); + ${events.map(emit).join('')}`) + process.env.WAVEFORM_TEST_SECRET = 'must not reach worker' + try { + expect(await runWaveformChild(location(), {})).toEqual({ readings: [reading], run: result }) + } + finally { + delete process.env.WAVEFORM_TEST_SECRET + } +}) + +it.each([ + [events[0]], + [...events, events[1]], + [...events, events[0]], + [events[0], { type: 'result', result: { ...result, counts: { ...result.counts, scored_outputs: 2 } } }], + [{ type: 'reading', reading: { ...reading, configHash: 'b'.repeat(64) } }, events[1]], + [{ type: 'reading', reading: { ...reading, hrvMetric: 'rmssd' } }, events[1]], + [events[0], { type: 'result', result: { ...result, decoded_records: 2 } }], + [events[0], { type: 'result', result: { ...result, counts: { ...result.counts, records: 2 } } }], + [events[0], { type: 'reading', reading: { ...reading, configHash: 'b'.repeat(64) } }, + { type: 'result', result: { ...result, counts: { ...result.counts, scored_outputs: 2 } } }], +].map(output => ({ output })))('rejects incomplete, inconsistent, or out-of-order protocol output %#', async ({ output }) => { + await worker(output.map(emit).join('')) + await expect(runWaveformChild(location(), {})).rejects.toThrow(/invalid event|incomplete result/) +}) + +it('rejects trailing partial data and nonzero exits even after a success event', async () => { + await worker(`${events.map(emit).join('')} process.stdout.write('partial');`) + await expect(runWaveformChild(location(), {})).rejects.toThrow('incomplete result') + await worker(`${events.map(emit).join('')} process.stdout.write(Buffer.from([0xc3]));`) + await expect(runWaveformChild(location(), {})).rejects.toThrow('incomplete result') + await worker(`${events.map(emit).join('')} process.exitCode = 1;`) + await expect(runWaveformChild(location(), {})).rejects.toThrow('incomplete result') +}) + +it('bounds individual output events, diagnostics and serialized input', async () => { + await worker('process.stdout.write(\'x\'.repeat(70000));') + await expect(runWaveformChild(location(), {})).rejects.toThrow('event exceeds') + await worker('process.stderr.write(\'x\'.repeat(17000));') + await expect(runWaveformChild(location(), {})).rejects.toThrow('diagnostic output') + await expect(runWaveformChild(location(), 'x'.repeat(8 * 1024 * 1024))).rejects.toThrow('input exceeds') +}) + +it('kills overdue and cancelled work and handles missing executables', async () => { + await worker('setInterval(() => {}, 1000);') + await expect(runWaveformChild(location(), {}, { timeoutMs: 30 })).rejects.toThrow('time budget') + const controller = new AbortController() + const pending = runWaveformChild(location(), {}, { signal: controller.signal }) + controller.abort() + await expect(pending).rejects.toThrow('cancelled') + await expect(runWaveformChild(location(), {}, { signal: controller.signal })).rejects.toThrow('cancelled') + await expect(runWaveformChild({ ...location(), pythonPath: join(directory, 'missing') }, {})).rejects.toThrow('could not start') +}) + +it('escalates cancellation when a running worker ignores SIGTERM', async () => { + const ready = join(directory, 'ready') + await worker(`process.on('SIGTERM', () => {}); require('node:fs').writeFileSync(${JSON.stringify(ready)}, String(process.pid)); setInterval(() => {}, 1000);`) + const controller = new AbortController() + const pending = runWaveformChild(location(), {}, { signal: controller.signal }) + const rejected = expect(pending).rejects.toThrow('cancelled') + await vi.waitFor(async () => expect(Number(await readFile(ready, 'utf8'))).toBeGreaterThan(0), { timeout: 1500 }) + const pid = Number(await readFile(ready, 'utf8')) + const started = performance.now() + controller.abort() + await rejected + expect(performance.now() - started).toBeLessThan(1500) + expect(() => process.kill(pid, 0)).toThrow() +}) + +it('enforces time-budget bounds while accepting their exact endpoints', async () => { + await worker(events.map(emit).join('')) + for (const timeoutMs of [0, 9, 1.5, Infinity, NaN, 3600001]) { + await expect(runWaveformChild(location(), {}, { timeoutMs })).rejects.toThrow('Invalid waveform time budget') + } + expect(await runWaveformChild(location(), {}, { timeoutMs: 3600000 })).toEqual({ readings: [reading], run: result }) + await worker('setInterval(() => {}, 1000);') + await expect(runWaveformChild(location(), {}, { timeoutMs: 10 })).rejects.toThrow('exceeded its time budget') +}) + +it('accepts exact input and diagnostic byte budgets', async () => { + await worker(`process.stderr.write('x'.repeat(16384)); ${events.map(emit).join('')}`) + expect(await runWaveformChild(location(), 'x'.repeat(8 * 1024 * 1024 - 2))).toEqual({ readings: [reading], run: result }) +}) + +it('rejects empty event lines without treating them as absent output', async () => { + await worker(`process.stdout.write('\\n'); ${events.map(emit).join('')}`) + await expect(runWaveformChild(location(), {})).rejects.toThrow('invalid event') +}) + +it('rejects excessive valid readings before exposing a result', async () => { + await worker(`for (let i=0; i<20001; i++) { ${emit(events[0])} } + ${emit({ type: 'result', result: { ...result, counts: { ...result.counts, scored_outputs: 20001 } } })}`) + await expect(runWaveformChild(location(), {})).rejects.toThrow('invalid event') +}) + +it('bounds aggregate output even when every event is valid and small enough', async () => { + const large = { type: 'reading', reading: { ...reading, limitations: Array.from({ length: 40 }, () => 'x'.repeat(200)) } } + await worker(`for (let i=0; i<2500; i++) { ${emit(large)} } + ${emit({ type: 'result', result: { ...result, counts: { ...result.counts, scored_outputs: 2500 } } })}`) + await expect(runWaveformChild(location(), {})).rejects.toThrow('output exceeds 16 MiB') +}) + +it('pins the requested Python build in the envelope and rejects mismatched completion', async () => { + await writeFile(executable, `#!${process.execPath}\nlet input='';process.stdin.on('data',chunk=>input+=chunk);process.stdin.on('end',()=>{if(JSON.parse(input).expected_build_hash!==${JSON.stringify(hash)})process.exitCode=1;${events.map(emit).join('')}});`, { mode: 0o700 }) + expect((await runWaveformChild(location(), {}, { expectedBuildHash: hash })).run.build_hash).toBe(hash) + await worker(events.map(emit).join('')) + await expect(runWaveformChild(location(), {}, { expectedBuildHash: 'b'.repeat(64) })).rejects.toThrow('incomplete result') + expect(() => runWaveformChild(location(), {}, { expectedBuildHash: 'invalid' })).toThrow() +}) + +it('retains valid measured process resources and rejects fabricated scopes or negative counts', async () => { + const resources = { version: 'waveform-process-resources-v1', wallMs: 120, cpuMs: 80, peakRssBytes: 102400, + scope: 'python_process_until_result', memoryScope: 'process_lifetime_high_water_mark' } + await worker([events[0], { type: 'result', result: { ...result, resources } }].map(emit).join('')) + expect((await runWaveformChild(location(), {})).run.resources).toEqual(resources) + for (const patch of [{ wallMs: -1 }, { cpuMs: -1 }, { peakRssBytes: 0 }, { scope: 'whole_job' }]) { + await worker([events[0], { type: 'result', result: { ...result, resources: { ...resources, ...patch } } }].map(emit).join('')) + await expect(runWaveformChild(location(), {})).rejects.toThrow('invalid event') + } +}) diff --git a/src/biometrics/backtest/tests/waveform-dataset.test.ts b/src/biometrics/backtest/tests/waveform-dataset.test.ts new file mode 100644 index 000000000..4367fe98d --- /dev/null +++ b/src/biometrics/backtest/tests/waveform-dataset.test.ts @@ -0,0 +1,101 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { afterEach, beforeEach, expect, it } from 'vitest' +import * as schema from '@/src/db/biometrics-schema' +import { dataHash } from '../../algorithms/canonical' +import { DatasetStore } from '../datasets' +import { datasetManifestSchema } from '../contracts' +import { prepareWaveformDataset } from '../waveform-dataset' +import { retainedWaveformInputSchema, waveformDatasetInputsSchema } from '../waveform-contracts' + +let sqlite: Database.Database +let db: ReturnType> +let datasets: DatasetStore +let base: ReturnType +const start = 1791673200000 +beforeEach(() => { + sqlite = new Database(':memory:') + db = drizzle(sqlite, { schema }) + migrate(db, { migrationsFolder: 'src/db/biometrics-migrations' }) + for (const id of [1, 2, 3]) db.insert(schema.sleepRecords).values({ id, side: 'left', + enteredBedAt: new Date(start + id * 86400000), leftBedAt: new Date(start + id * 86400000 + 3600000), + sleepDurationSeconds: 3600, timesExitedBed: 0 }).run() + datasets = new DatasetStore(db) + base = datasets.read(datasets.create({ side: 'left', subjectId: 'fixture', sessionIds: [1, 2, 3], + referenceImportIds: [], timezone: 'UTC', idempotencyKey: 'base' }).id) +}) +afterEach(() => sqlite.close()) +function inputs() { + return datasetManifestSchema.parse(base.manifest).splitProtocol.assignments.map((cohort, index) => ({ + cohortId: cohort.cohortId, + input: { version: 'retained-waveform-input-v1' as const, retained_start_ms: cohort.startMs - 300000, + score_start_ms: cohort.startMs, score_end_ms: cohort.endMs, contexts: [], + artifacts: [{ artifact: { name: `${String(index + 1).repeat(64)}.RAW.gz`, sha256: String(index + 1).repeat(64), compressed_bytes: 100, expanded_bytes: 200 }, + first_event_ms: cohort.startMs - 300000, last_event_ms: cohort.endMs - 1 }] }, + })) +} +function source() { + return { ...base, manifest: datasetManifestSchema.parse(base.manifest) } +} + +it('creates a separate immutable snapshot with stable request/content hashes and original observations', () => { + const before = structuredClone(base) + const first = prepareWaveformDataset(source(), inputs().reverse(), 'raw', 1000) + const retry = prepareWaveformDataset(source(), inputs(), 'raw', 2000) + expect(first.requestHash).toBe(retry.requestHash) + expect(first.contentHash).toBe(retry.contentHash) + expect(first.id).not.toBe(retry.id) + expect(base).toEqual(before) + expect(first.snapshot.sessions).toEqual(base.snapshot.sessions) + expect(first.manifest.parentDataset).toEqual({ id: base.id, contentHash: base.contentHash }) + expect(first.manifest.capabilities.signalReplay).toBe(true) + expect(first.contentHash).toBe(dataHash({ manifest: first.manifest, snapshot: first.snapshot })) + expect(datasets.list('left')).toHaveLength(1) // Preparation performs no publication. +}) + +it('preserves waveform inputs through the shared dataset reader and detects later corruption', () => { + const prepared = prepareWaveformDataset(source(), inputs(), 'raw') + db.insert(schema.biometricDatasets).values({ ...prepared, + byteLength: Buffer.byteLength(JSON.stringify(prepared.manifest)) + Buffer.byteLength(JSON.stringify(prepared.snapshot)) }).run() + const retained = datasets.read(prepared.id) + expect(retained.snapshot).toEqual(prepared.snapshot) + sqlite.prepare('UPDATE biometric_datasets SET snapshot = json_set(snapshot, \'$.waveformInputs[0].input.artifacts[0].artifact.expanded_bytes\', 999) WHERE id = ?').run(prepared.id) + expect(() => datasets.read(prepared.id)).toThrow('integrity') +}) + +it('rejects missing, duplicated and mismatched whole-night inputs and forged source identity', () => { + expect(() => prepareWaveformDataset(source(), inputs().slice(1), 'raw')).toThrow('every whole-night') + const duplicated = inputs() + duplicated[1] = duplicated[0] + expect(() => prepareWaveformDataset(source(), duplicated, 'raw')).toThrow('each waveform cohort once') + const wrongWindow = inputs() + wrongWindow[0].input.score_start_ms++ + expect(() => prepareWaveformDataset(source(), wrongWindow, 'raw')).toThrow('whole-night cohort') + expect(() => prepareWaveformDataset({ ...source(), contentHash: '0'.repeat(64) }, inputs(), 'raw')).toThrow('integrity') +}) + +it('rejects overlapping source/context windows, invalid modes and inconsistent shared evidence', () => { + const input = inputs()[0].input + expect(() => retainedWaveformInputSchema.parse({ ...input, artifacts: [...input.artifacts, input.artifacts[0]] })).toThrow('distinct') + const context = { start_ms: input.retained_start_ms, end_ms: input.score_end_ms, + active_sides: ['right', 'left'], left_occupied: null, right_occupied: null, pump_context_known: false } + expect(() => retainedWaveformInputSchema.parse({ ...input, contexts: [context] })).toThrow('sleeper mode') + expect(() => retainedWaveformInputSchema.parse({ ...input, contexts: [{ ...context, active_sides: null }, { ...context, active_sides: null }] })).toThrow('nonoverlapping') + const shared = [{ cohortId: 'one', input }, { cohortId: 'two', input: structuredClone(input) }] + expect(waveformDatasetInputsSchema.parse(shared)).toEqual(shared) + shared[1].input.artifacts[0].artifact.expanded_bytes++ + expect(() => waveformDatasetInputsSchema.parse(shared)).toThrow('Conflicting evidence') +}) + +it('detaches opaque historical context from caller-owned objects before hashing', () => { + const mutable = source() + const interval = { start: start + 1, end: start + 2 } + mutable.snapshot.sessions[0].presentIntervals = [interval] + mutable.contentHash = dataHash({ manifest: mutable.manifest, snapshot: mutable.snapshot }) + const prepared = prepareWaveformDataset(mutable, inputs(), 'raw') + interval.end++ + expect(prepared.snapshot.sessions[0].presentIntervals).toEqual([{ start: start + 1, end: start + 2 }]) + expect(dataHash({ manifest: prepared.manifest, snapshot: prepared.snapshot })).toBe(prepared.contentHash) +}) diff --git a/src/biometrics/backtest/tests/waveform-evaluator.test.ts b/src/biometrics/backtest/tests/waveform-evaluator.test.ts new file mode 100644 index 000000000..9b0208730 --- /dev/null +++ b/src/biometrics/backtest/tests/waveform-evaluator.test.ts @@ -0,0 +1,150 @@ +import { previewReferenceImport } from '../../references/normalize' +// @vitest-environment node +import { expect, it } from 'vitest' +import { backtestFixture } from './fixtures/backtest' +import { dataHash, configHash } from '../../algorithms/canonical' +import { evaluateBacktest } from '../evaluator' +import { summarizeBacktestResult } from '../report-summary' +import { defaultConfig, manifestHash } from '../../algorithms/registry' + +function fixture() { + const base = backtestFixture() + const cohort = base.dataset.manifest.splitProtocol.assignments[0] + const digest = 'a'.repeat(64), build = 'b'.repeat(64), resolvedHash = configHash(defaultConfig) + const dataset = { ...base.dataset, snapshot: { ...base.dataset.snapshot, version: 'waveform-dataset-v1' as const, + waveformInputs: [{ cohortId: cohort.cohortId, input: { version: 'retained-waveform-input-v1' as const, + retained_start_ms: cohort.startMs, score_start_ms: cohort.startMs, score_end_ms: cohort.endMs, contexts: [], + artifacts: [{ artifact: { sha256: digest, name: `${digest}.RAW.gz`, compressed_bytes: 1, expanded_bytes: 2 }, + first_event_ms: cohort.startMs, last_event_ms: cohort.endMs - 1 }] } }] } } + const protocol = { ...base.job.protocol, mode: 'signal_replay' as const, waveformStages: false } + const readings = [{ side: 'left' as const, timestampMs: cohort.startMs, heartRate: 62, hrvIndex: null, + hrvMetric: 'window_ibi_variability_v1' as const, breathingRate: 12, qualityScore: 1, flags: [], hrRaw: 62, + sourceSide: 'left' as const, configHash: resolvedHash, limitations: ['unknown_context'] }] + const outputs = [{ cohortId: cohort.cohortId, configHash: resolvedHash, readings, run: { + version: 'retained-waveform-run-v1' as const, build_hash: build, decoded_records: 1, raw_records: 1, + counts: { records: 1, accepted_frames: 1, empty_channel_frames: 0, pump_gated_frames: 0, preroll_outputs: 0, scored_outputs: 1 }, + limitations: ['unknown_context'], config_hashes: { left: resolvedHash, right: resolvedHash }, artifact_hashes: [digest], + } }] + return { dataset, job: { ...base.job, datasetHash: dataHash(dataset), protocol, protocolHash: configHash(protocol), + waveform: { pythonBuildHash: build, otherSide: { manifestHash, resolvedHash, resolved: defaultConfig } } }, + waveform: { hash: dataHash(outputs), outputs } } +} +it('scores derived readings for matching labels without replacing immutable dataset identity', () => { + const work = fixture() + const before = structuredClone(work) + const report = evaluateBacktest(work) + expect(report.version).toBe('waveform-backtest-report-v1') + expect(report.datasetHash).toBe(work.job.datasetHash) + expect(report.nights[0].uniqueObservationCount).toBe(1) + expect(report.nights[0].evaluations[0].windows[0].observationCount).toBe(1) + expect(report.nights[0].evaluations[0].windows[0].numeric[0]).toMatchObject({ pairedCount: 1, metrics: { mae: 0 } }) + expect(report.nights[0].evaluations[1].windows).toEqual(report.nights[0].evaluations[0].windows) + expect(report.nights[0].evaluations[0].windows[0].blocks).toEqual([]) + expect(report.nights[0].numericComparisons?.[0].metrics).toEqual(expect.arrayContaining([ + expect.objectContaining({ metric: 'heart_rate', baseline: 'default', candidate: 'current', + commonObservationCount: 1, differences: expect.objectContaining({ commonMae: 0 }), coverageReduced: false }), + ])) + const summary = summarizeBacktestResult({ available: true, report }) + expect(summary?.report).toMatchObject({ nights: [expect.objectContaining({ numericComparisons: report.nights[0].numericComparisons })] }) + expect(report.numericNightSummary?.[0]).toMatchObject({ metric: 'heart_rate', scoredNights: 1, equalNightMeanMaeDifference: 0, uncertainty: { statistic: 'equal_night_mean_common_mae_difference', status: 'not_requested' } }) + expect(work).toEqual(before) +}) +it('rejects incomplete, mismatched or extraneous derived waveform evidence', () => { + for (const mutate of [ + (work: ReturnType) => { work.waveform.hash = '0'.repeat(64) }, + (work: ReturnType) => { work.waveform.outputs = [] }, + (work: ReturnType) => { work.waveform.outputs.push(structuredClone(work.waveform.outputs[0])) }, + (work: ReturnType) => { work.waveform.outputs[0].run.build_hash = '0'.repeat(64) }, + (work: ReturnType) => { work.waveform.outputs[0].run.config_hashes.right = '0'.repeat(64) }, + (work: ReturnType) => { work.waveform.outputs[0].run.artifact_hashes = ['0'.repeat(64)] }, + (work: ReturnType) => { work.waveform.outputs[0].readings[0].timestampMs-- }, + (work: ReturnType) => { work.waveform.outputs[0].readings[0].configHash = '0'.repeat(64) }, + ]) { + const work = fixture() + mutate(work) + if (work.waveform.hash !== '0'.repeat(64)) work.waveform.hash = dataHash(work.waveform.outputs) + expect(() => evaluateBacktest(work)).toThrow() + } +}) +it('rejects waveform evidence supplied to processed evaluation and missing evidence in signal replay', () => { + expect(() => evaluateBacktest({ ...backtestFixture(), waveform: fixture().waveform })).toThrow('Unexpected waveform') + const work = fixture() + expect(() => evaluateBacktest({ ...work, waveform: undefined })).toThrow('Missing or corrupt') +}) + +it('retains objective thresholds but cannot promote without qualified measured evidence', () => { + const work = fixture() + const protocol = { ...work.job.protocol, objectivePolicy: { version: 'numeric-objective-v1' as const, qualificationId: 'test-only', metric: 'heart_rate' as const, + minimumNights: 3, minimumPairedReferencesPerNight: 10, maximumLostObservationFraction: 0, minimumMaeImprovement: 2, + maximumNightMaeRegression: 1, maximumRegressedNightFraction: 0, maximumSubgroupMaeRegression: 1, requiredSubgroups: [], + operational: { maximumWallMs: 1000, maximumPeakRssBytes: 1000, maximumIngestionAgeMs: 1000 } } } + const report = evaluateBacktest({ ...work, job: { ...work.job, protocol, protocolHash: configHash(protocol), + configs: [...work.job.configs, { ...work.job.configs[0], label: 'candidate' }] } }) + expect(report.objectiveAssessments).toHaveLength(2) + expect(report.objectiveAssessments[0]).toMatchObject({ status: 'insufficient_evidence', designated: false, + missingEvidence: expect.arrayContaining(['policy_not_designated', 'operational_evidence_unavailable']) }) + expect(report.protocol.objectivePolicy).toEqual(protocol.objectivePolicy) +}) + +it('retains measured waveform costs through compact report projection without claiming whole-job health', () => { + const work = fixture() + const outputs = work.waveform.outputs.map(output => ({ ...output, run: { ...output.run, resources: { + version: 'waveform-process-resources-v1' as const, wallMs: 120, cpuMs: 80, peakRssBytes: 102400, + scope: 'python_process_until_result' as const, memoryScope: 'process_lifetime_high_water_mark' as const, + } } })) + const report = evaluateBacktest({ ...work, waveform: { hash: dataHash(outputs), outputs } }) + const summary = summarizeBacktestResult({ available: true, report }) + expect(summary?.report).toMatchObject({ waveform: { resources: { + expectedProcesses: outputs.length, measuredProcesses: outputs.length, totalWallMs: outputs.length * 120, + totalCpuMs: outputs.length * 80, largestProcessPeakRssBytes: 102400, ingestionHealthMeasured: false, + } } }) +}) + +it('exposes a late-window regression hidden by lower whole-night error', () => { + const work = fixture(), start = work.dataset.snapshot.sessions[0].startMs + const baseline = work.waveform.outputs[0] + baseline.readings = [80, 60].map((heartRate, index) => ({ ...baseline.readings[0], heartRate, timestampMs: start + index * 300000 })) + baseline.run.counts.scored_outputs = 2 + const candidate = structuredClone(defaultConfig) + candidate.heart_rate.trackingToleranceBpm = 20 + const candidateHash = configHash(candidate) + const output = { ...baseline, configHash: candidateHash, + readings: baseline.readings.map((row, index) => ({ ...row, heartRate: index ? 65 : 60, configHash: candidateHash })), + run: { ...baseline.run, config_hashes: { ...baseline.run.config_hashes, left: candidateHash } } } + work.dataset.snapshot.references[0].measurements = previewReferenceImport({ side: 'left', source: 'fixture', subjectId: 'A', rows: [ + { recordKey: 'early', metric: 'heart_rate', timestampMs: start, value: 60, unit: 'bpm' }, + { recordKey: 'late', metric: 'heart_rate', timestampMs: start + 300000, value: 60, unit: 'bpm' }, + ] }).measurements + const protocol = { ...work.job.protocol, objectivePolicy: { version: 'numeric-objective-v1' as const, qualificationId: 'test-only', metric: 'heart_rate' as const, + minimumNights: 3, minimumPairedReferencesPerNight: 1, maximumLostObservationFraction: 0, minimumMaeImprovement: 2, + maximumNightMaeRegression: 1, maximumRegressedNightFraction: 0, maximumSubgroupMaeRegression: 1, requiredSubgroups: ['window_second_half'], + operational: { maximumWallMs: 1000, maximumPeakRssBytes: 1000, maximumIngestionAgeMs: 1000 } } } + const outputs = [baseline, output] + const report = evaluateBacktest({ ...work, waveform: { outputs, hash: dataHash(outputs) }, job: { ...work.job, + datasetHash: dataHash(work.dataset), protocol, protocolHash: configHash(protocol), + configs: [...work.job.configs, { ...work.job.configs[0], label: 'candidate', resolved: candidate, resolvedHash: candidateHash }] } }) + expect(report.numericNightSummary?.find(row => row.baseline === 'default' && row.candidate === 'candidate' && row.metric === 'heart_rate')?.equalNightMeanMaeDifference).toBe(-7.5) + expect(report.numericSubgroupSummary?.find(row => row.name === 'window_second_half' && row.summary.baseline === 'default' && row.summary.candidate === 'candidate')?.summary.equalNightMeanMaeDifference).toBe(5) + expect(report.objectiveAssessments[0].evidence.subgroups).toContainEqual({ name: 'window_second_half', pairedNights: 1, maeDifference: 5 }) + expect(report.objectiveAssessments[0].evidence.subgroups).toContainEqual({ name: 'reference_intervals', pairedNights: 0, maeDifference: null }) +}) + +it('evaluates each numeric policy against both baselines without mixing metric subgroups', () => { + const work = fixture() + const objectivePolicies = (['heart_rate', 'breathing_rate', 'window_ibi_variability_v1'] as const).map(metric => ({ + version: 'numeric-objective-v1' as const, qualificationId: 'multi-test', metric, minimumNights: 3, + minimumPairedReferencesPerNight: 1, maximumLostObservationFraction: 0, minimumMaeImprovement: 1, + maximumNightMaeRegression: 1, maximumRegressedNightFraction: 0, maximumSubgroupMaeRegression: 1, + requiredSubgroups: ['reference_points'], operational: { maximumWallMs: 1000, maximumPeakRssBytes: 1000, maximumIngestionAgeMs: 1000 }, + })) + const protocol = { ...work.job.protocol, objectivePolicies } + const report = evaluateBacktest({ ...work, job: { ...work.job, protocol, protocolHash: configHash(protocol), + configs: [...work.job.configs, { ...work.job.configs[0], label: 'candidate' }] } }) + expect(report.objectiveAssessments).toHaveLength(6) + for (const policy of objectivePolicies) { + const assessments = report.objectiveAssessments.filter(row => row.policy.metric === policy.metric) + expect(assessments.map(row => row.baseline).sort()).toEqual(['current', 'default']) + expect(assessments.every(row => row.status === 'insufficient_evidence' && !row.designated)).toBe(true) + expect(report.numericSubgroupSummary?.filter(row => row.summary.metric === policy.metric)).toHaveLength(12) + } +}) diff --git a/src/biometrics/backtest/tests/waveform-publication.integration.test.ts b/src/biometrics/backtest/tests/waveform-publication.integration.test.ts new file mode 100644 index 000000000..f2f966676 --- /dev/null +++ b/src/biometrics/backtest/tests/waveform-publication.integration.test.ts @@ -0,0 +1,230 @@ +// @vitest-environment node +import { execFileSync } from 'node:child_process' +import { mkdtempSync, readdirSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { expect, it } from 'vitest' +import * as schema from '@/src/db/biometrics-schema' +import { configHash } from '../../algorithms/canonical' +import { BacktestJobStore } from '../jobs' +import { defaultConfig, manifestHash } from '../../algorithms/registry' +import { DatasetStore } from '../datasets' +import { datasetManifestSchema } from '../contracts' +import { prepareSelectedWaveformDataset } from '../waveform-selection' +import { RawSourceCatalog } from '../raw-catalog' +import { waveformEvaluatorBuildIdentity } from '../waveform-build' +import { runCoordinatorTask } from '../coordinator' +import { runWaveformChild } from '../waveform-child' +import type { z } from 'zod' +import type { retainedWaveformInputSchema } from '../waveform-contracts' + +type Input = z.infer +const python = resolve('modules/piezo-processor/.venv/bin/python') +const environment: NodeJS.ProcessEnv = { PYTHONPATH: resolve('modules'), OPENBLAS_NUM_THREADS: '1', LANG: 'C.UTF-8', TZ: 'UTC', NODE_ENV: 'test' } + +it('publishes, verifies, replays, releases and collects a retained waveform dataset without changing observations', async () => { + const directory = mkdtempSync(join(tmpdir(), 'waveform-publication-integration-')) + const database = join(directory, 'biometrics.db') + const archive = join(directory, 'archive') + const retained = join(directory, 'biometric-raw') + const sqlite = new Database(database) + try { + // Keep the kernel lock real but isolate its inode from /run and other tests. + // Production has no CLI/environment override for the host execution lock. + const fixturePython = join(directory, 'fixture-python') + writeFileSync(fixturePython, `#!${python} +import runpy, sys +args = sys.argv[1:] +if args[0] == '-B': + args = args[1:] +assert args[0] == '-m' +module = args[1] +sys.argv = [module, *args[2:]] +if module == 'common.biometrics.replay_worker': + from common.biometrics import replay_worker + from common.biometrics.execution_lock import acquire_execution_lock + replay_worker.acquire_execution_lock = lambda: acquire_execution_lock(${JSON.stringify(join(directory, 'execution.lock'))}) + sys.exit(replay_worker.main()) +runpy.run_module(module, run_name='__main__') +`, { mode: 0o700 }) + const db = drizzle(sqlite, { schema }) + migrate(db, { migrationsFolder: 'src/db/biometrics-migrations' }) + const fixture = JSON.parse(execFileSync(python, ['-B', '-c', ` +import json, gzip, sys +from dataclasses import asdict +from pathlib import Path +import cbor2 +import numpy as np +from common.biometrics.raw_sources import ArchiveSource +root=Path(sys.argv[1]); root.mkdir() +start=1700000000000 +records=[{'type':'frzHealth','ts':start/1000,'left':{'pumpRpm':0},'right':{'pumpRpm':0}}] +for second in range(150): + samples=(np.sin((np.arange(500)+second*500)/500*2*np.pi*1.2)*1000000).astype(' now) + const base = datasets.read(datasets.create({ side: 'left', subjectId: 'fixture', sessionIds: [1], + referenceImportIds: [], timezone: 'UTC', idempotencyKey: 'processed' }).id) + const manifest = datasetManifestSchema.parse(base.manifest) + const catalog = new RawSourceCatalog(db) + const sourceIds = fixture.inspections.map(inspection => catalog.record(inspection)) + const request = { parentDatasetId: base.id, expectedContentHash: base.contentHash, + cohorts: [{ cohortId: manifest.splitProtocol.assignments[0].cohortId, sourceIds }], idempotencyKey: 'waveform' } + const { prepared } = prepareSelectedWaveformDataset(db, request, now) + execFileSync(process.execPath, ['scripts/build/biometrics-worker.mjs', directory]) + const options = { replayBudget: { jobWallMs: 300000, waveformWallMs: 200000, waveformCpuSeconds: 180 }, databasePath: database, coordinatorPath: join(directory, 'coordinator.cjs'), workerPath: join(directory, 'worker.cjs'), + modulePath: resolve('node_modules'), rawLocation: { pythonPath: fixturePython, modulesPath: resolve('modules'), archiveDirectory: archive } } + expect(await runCoordinatorTask(options, { type: 'createWaveformDataset', input: request })).toEqual({ status: 'deferred' }) + let admissionChecks = 0 + expect(await runCoordinatorTask(options, { type: 'createWaveformDataset', input: request }, { admit: async () => ++admissionChecks === 1 })).toEqual({ status: 'deferred' }) + expect(admissionChecks).toBe(2) + expect(readdirSync(directory)).not.toContain('biometric-raw') + const slowPublisher = join(directory, 'slow-publisher') + writeFileSync(slowPublisher, `#!${process.execPath}\nprocess.stdin.resume();setInterval(()=>{},1000);setTimeout(()=>process.exit(91),10000).unref();`, { mode: 0o700 }) + let resourceChecks = 0 + expect(await runCoordinatorTask({ ...options, rawLocation: { ...options.rawLocation, pythonPath: slowPublisher } }, + { type: 'createWaveformDataset', input: request }, { admit: async () => ++resourceChecks <= 2 })).toEqual({ status: 'deferred' }) + expect(resourceChecks).toBe(3) + expect(sqlite.prepare('SELECT count(*) n FROM biometric_datasets').get()).toEqual({ n: 1 }) + const publish = async (input = request) => { + const result = await runCoordinatorTask(options, { type: 'createWaveformDataset', input }, { admit: async () => true }) + if (result.status !== 'published') throw new Error('Publication was deferred') + return { type: 'result', id: result.dataset.id, contentHash: result.dataset.contentHash, available: result.dataset.available } + } + const published = await publish() + expect(published).toEqual({ type: 'result', id: expect.any(String), contentHash: prepared.contentHash, available: true }) + const frozen = datasets.read(published.id) + expect(frozen.snapshot).toEqual(prepared.snapshot) // Includes shared cross-language content-hash verification. + expect(sqlite.prepare('SELECT count(*) AS count FROM biometric_dataset_raw_artifacts').get()).toEqual({ count: 2 }) + rmSync(archive, { recursive: true }) + for (const id of sourceIds) catalog.markUnavailable(id) + expect(await publish()).toEqual(published) + const retryWithoutAdmission = await runCoordinatorTask(options, { type: 'createWaveformDataset', input: { ...request, cohorts: request.cohorts.map(item => ({ ...item, sourceIds: [...item.sourceIds].reverse() })) } }) + expect(retryWithoutAdmission).toMatchObject({ status: 'published', dataset: { id: published.id } }) + if (frozen.snapshot.version !== 'waveform-dataset-v1') throw new Error('Missing waveform inputs') + const build = await waveformEvaluatorBuildIdentity(options.coordinatorPath, options.workerPath, options.rawLocation) + const replay = await runWaveformChild({ pythonPath: fixturePython, modulesPath: resolve('modules'), retainedDirectory: retained }, { + version: 'retained-waveform-work-v1', input: frozen.snapshot.waveformInputs[0].input, + configs: { left: defaultConfig, right: defaultConfig }, + }, { expectedBuildHash: build.python.hash }) + expect(replay.run.build_hash).toBe(build.python.hash) + expect(replay.readings.length).toBeGreaterThan(0) + expect(replay.run.raw_records).toBe(151) + expect(sqlite.prepare('SELECT heart_rate FROM vitals').all()).toEqual([{ heart_rate: 65 }]) + const jobs = new BacktestJobStore(db) + const experiment = jobs.createExperiment(published.id, { version: 'descriptive-evaluation-v1', mode: 'signal_replay', waveformStages: true, + assumedStageCalibrationQuality: 0, pointToleranceMs: 30000, clockOffsetMs: 0, + objectivePolicy: { version: 'numeric-objective-v1', qualificationId: 'integration-only', metric: 'heart_rate', + minimumNights: 3, minimumPairedReferencesPerNight: 10, maximumLostObservationFraction: 0.1, minimumMaeImprovement: 2, + maximumNightMaeRegression: 1, maximumRegressedNightFraction: 0.34, maximumSubgroupMaeRegression: 1, requiredSubgroups: [], + operational: { maximumWallMs: 30000, maximumPeakRssBytes: 536870912, maximumIngestionAgeMs: 5000, + observations: { version: 'sampled-process-group-v1', maximumMemoryGapMs: 2000, maximumIngestionGapMs: 7000 } } } }) + const candidate = structuredClone(defaultConfig) + candidate.heart_rate.trackingToleranceBpm = 20 + const jobInput = { experimentId: experiment.id, evaluatorBuildHash: build.hash, split: 'tuning' as const, + waveform: { pythonBuildHash: build.python.hash, otherSide: { manifestHash, resolvedHash: configHash(defaultConfig), resolved: defaultConfig } }, + configs: [ + { label: 'default' as const, revisionId: null, manifestHash, resolvedHash: configHash(defaultConfig), resolved: defaultConfig }, + { label: 'current' as const, revisionId: null, manifestHash, resolvedHash: configHash(defaultConfig), resolved: defaultConfig }, + { label: 'candidate' as const, revisionId: null, manifestHash, resolvedHash: configHash(candidate), resolved: candidate }, + ] } + expect(() => jobs.enqueue({ ...jobInput, waveform: undefined, idempotencyKey: 'unpinned-signal' })).toThrow('pinned Python') + expect(() => jobs.enqueue({ ...jobInput, waveform: { ...jobInput.waveform, otherSide: { ...jobInput.waveform.otherSide, resolvedHash: '0'.repeat(64) } }, idempotencyKey: 'wrong-other-side' })).toThrow('Other-side configuration') + const job = jobs.enqueue({ ...jobInput, idempotencyKey: 'signal-job' }) + expect(await runCoordinatorTask(options, { type: 'execute', owner: 'signal-integration' }, { admit: async () => true })).toEqual({ state: 'succeeded', jobId: job.id }) + const report = jobs.result(job.id)?.report + expect(report).toMatchObject({ version: 'waveform-backtest-report-v1', datasetHash: published.contentHash, + evaluatorBuildHash: build.hash, protocol: { mode: 'signal_replay' }, waveform: { outputs: [{}, {}] }, + nights: [{ evaluations: [{ label: 'default' }, { label: 'current' }, { label: 'candidate' }] }] }) + expect(report).toMatchObject({ objectiveAssessments: [ + { baseline: 'default', evidence: { operational: null }, status: 'insufficient_evidence' }, + { baseline: 'current', evidence: { operational: null }, status: 'insufficient_evidence' }, + ] }) + const summary = await runCoordinatorTask(options, { type: 'readSummary', input: { id: job.id } }) + expect(JSON.stringify(summary?.report)).not.toContain('"readings"') + expect(JSON.stringify(report)).toContain('"readings"') + if (!summary) throw new Error('Missing retained report') + const referenceTraceInput = { id: job.id, expectedContentHash: summary.contentHash, + cohortId: prepared.manifest.splitProtocol.assignments[0].cohortId, metric: 'heart_rate' as const } + const references = await runCoordinatorTask(options, { type: 'readReferenceTrace', input: referenceTraceInput }) + expect(references).toMatchObject({ provenance: 'frozen_dataset_reference', datasetHash: published.contentHash, items: [], totalItems: 0 }) + await expect(runCoordinatorTask(options, { type: 'readReferenceTrace', input: { + ...referenceTraceInput, expectedContentHash: '0'.repeat(64), + } })).rejects.toMatchObject({ code: 'REVISION_CONFLICT' }) + const numericTrace = await runCoordinatorTask(options, { type: 'readNumericTrace', input: { + id: job.id, expectedContentHash: summary.contentHash, configuration: 'candidate', limit: 2, + } }) + expect(numericTrace).toMatchObject({ provenance: 'retained_waveform_recomputation', side: 'left', downsampled: false }) + expect(numericTrace.totalItems).toBeGreaterThan(0) + expect(numericTrace.items.every(row => row.side === 'left')).toBe(true) + await expect(runCoordinatorTask(options, { type: 'readNumericTrace', input: { + id: job.id, expectedContentHash: '0'.repeat(64), configuration: 'candidate', + } })).rejects.toMatchObject({ code: 'REVISION_CONFLICT' }) + const wrongBuild = jobs.enqueue({ ...jobInput, evaluatorBuildHash: '0'.repeat(64), idempotencyKey: 'wrong-signal-build' }) + expect(await runCoordinatorTask(options, { type: 'execute', owner: 'wrong-build' }, { admit: async () => true })).toEqual({ state: 'stopped', jobId: wrongBuild.id }) + expect(jobs.get(wrongBuild.id).state).toBe('failed') + expect(jobs.result(wrongBuild.id)).toBeNull() + jobs.releaseResult(job.id, summary.contentHash) + await expect(runCoordinatorTask(options, { type: 'readReferenceTrace', input: referenceTraceInput })).rejects.toMatchObject({ code: 'RESULT_RELEASED' }) + await expect(runCoordinatorTask(options, { type: 'readNumericTrace', input: { + id: job.id, expectedContentHash: summary.contentHash, configuration: 'candidate', + } })).rejects.toMatchObject({ code: 'RESULT_RELEASED' }) + expect(sqlite.prepare('SELECT heart_rate FROM vitals').all()).toEqual([{ heart_rate: 65 }]) + const reused = await publish({ ...request, idempotencyKey: 'reuse-retained' }) + expect(reused.contentHash).toBe(published.contentHash) + expect(reused.id).not.toBe(published.id) + datasets.release(reused.id) + const lostReply = join(directory, 'lost-reply') + writeFileSync(lostReply, `#!${process.execPath}\nconst {spawnSync}=require('node:child_process');const input=require('node:fs').readFileSync(0);const result=spawnSync(${JSON.stringify(python)},process.argv.slice(2),{input,env:process.env});process.exit(result.status===0?42:1);`, { mode: 0o700 }) + const recovered = await runCoordinatorTask({ ...options, rawLocation: { ...options.rawLocation, pythonPath: lostReply } }, + { type: 'createWaveformDataset', input: { ...request, idempotencyKey: 'lost-reply' } }, { admit: async () => true }) + expect(recovered.status).toBe('published') + if (recovered.status !== 'published') throw new Error('Lost committed response was not recovered') + expect(recovered.dataset.contentHash).toBe(published.contentHash) + datasets.release(recovered.dataset.id) + datasets.release(published.id) + expect(await publish()).toEqual({ ...published, available: false }) + const collected = JSON.parse(execFileSync(python, ['-B', '-c', ` +import json, sqlite3, sys +from common.biometrics.raw_artifacts import RawArtifactStore +connection=sqlite3.connect(sys.argv[1]) +print(json.dumps(RawArtifactStore(sys.argv[2]).collect_released(connection,now_ms=int(sys.argv[3]),minimum_age_ms=0))) +connection.close() +`, database, retained, String(now)], { env: environment, encoding: 'utf8', timeout: 20000 })) as string[] + expect(collected).toHaveLength(2) + expect(readdirSync(retained).filter(name => name.endsWith('.RAW.gz'))).toEqual([]) + expect(datasets.read(base.id)).toEqual(base) + datasets.release(base.id) + expect(await publish()).toEqual({ ...published, available: false }) + await expect(publish({ ...request, expectedContentHash: '0'.repeat(64) })).rejects.toMatchObject({ code: 'REVISION_CONFLICT' }) + } + finally { + sqlite.close() + rmSync(directory, { recursive: true, force: true }) + } +}, 30000) diff --git a/src/biometrics/backtest/tests/waveform-resources.test.ts b/src/biometrics/backtest/tests/waveform-resources.test.ts new file mode 100644 index 000000000..4c6b79854 --- /dev/null +++ b/src/biometrics/backtest/tests/waveform-resources.test.ts @@ -0,0 +1,20 @@ +// @vitest-environment node +import { expect, it } from 'vitest' +import { summarizeWaveformResources } from '../waveform-resources' +const resources = { version: 'waveform-process-resources-v1' as const, wallMs: 100, cpuMs: 80, peakRssBytes: 500, + scope: 'python_process_until_result' as const, memoryScope: 'process_lifetime_high_water_mark' as const } +it('sums sequential work without summing process memory peaks or claiming live health', () => { + expect(summarizeWaveformResources([{ run: { resources } }, { run: { resources: { ...resources, peakRssBytes: 700 } } }])).toMatchObject({ + expectedProcesses: 2, measuredProcesses: 2, totalWallMs: 200, totalCpuMs: 160, largestProcessPeakRssBytes: 700, ingestionHealthMeasured: false, + }) +}) +it('does not turn absent legacy measurements or missing kernel telemetry into zero', () => { + expect(summarizeWaveformResources([])).toMatchObject({ totalWallMs: null, totalCpuMs: null, largestProcessPeakRssBytes: null }) + expect(summarizeWaveformResources([{ run: {} }, { run: { resources } }])).toMatchObject({ measuredProcesses: 1, totalWallMs: null, totalCpuMs: null, largestProcessPeakRssBytes: null }) + expect(summarizeWaveformResources([{ run: { resources: { ...resources, peakRssBytes: null } } }])).toMatchObject({ totalWallMs: 100, totalCpuMs: 80, largestProcessPeakRssBytes: null }) +}) +it('rejects invalid measurement units and nonfinite counters', () => { + for (const bad of [{ wallMs: -1 }, { cpuMs: Infinity }, { peakRssBytes: 0 }, { peakRssBytes: 2 ** 53 }, { scope: 'whole_job' }]) { + expect(() => summarizeWaveformResources([{ run: { resources: { ...resources, ...bad } as typeof resources } }])).toThrow() + } +}) diff --git a/src/biometrics/backtest/tests/waveform-selection.test.ts b/src/biometrics/backtest/tests/waveform-selection.test.ts new file mode 100644 index 000000000..ee98a39e2 --- /dev/null +++ b/src/biometrics/backtest/tests/waveform-selection.test.ts @@ -0,0 +1,98 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { afterEach, beforeEach, expect, it } from 'vitest' +import * as schema from '@/src/db/biometrics-schema' +import { DatasetStore } from '../datasets' +import { RawSourceCatalog } from '../raw-catalog' +import { createWaveformDatasetSchema, prepareSelectedWaveformDataset, recordedWaveformContexts } from '../waveform-selection' +import { datasetManifestSchema } from '../contracts' + +let sqlite: Database.Database +let db: ReturnType> +let parent: ReturnType +const start = 1700000000000 +let sourceIds: string[] +beforeEach(() => { + sqlite = new Database(':memory:') + db = drizzle(sqlite, { schema }) + migrate(db, { migrationsFolder: 'src/db/biometrics-migrations' }) + db.insert(schema.sleepRecords).values({ id: 1, side: 'left', enteredBedAt: new Date(start), leftBedAt: new Date(start + 600000), + sleepDurationSeconds: 600, timesExitedBed: 1, presentIntervals: [[start / 1000, start / 1000 + 60]], + notPresentIntervals: [[start / 1000 + 60, start / 1000 + 120]] }).run() + const store = new DatasetStore(db) + parent = store.read(store.create({ side: 'left', subjectId: 'test', sessionIds: [1], referenceImportIds: [], timezone: 'UTC', idempotencyKey: 'parent' }).id) + sourceIds = [0, 1].map((index) => { + const sha256 = String(index + 1).repeat(64) + return new RawSourceCatalog(db).record({ source_name: `${index}.RAW.gz`, retained: false, + artifact: { name: `${sha256}.RAW.gz`, sha256, compressed_bytes: 100, expanded_bytes: 200 }, + inventory: { version: 'retained-raw-inventory-v1', artifact_sha256: sha256, decoded_records: 2, piezo_records: 1, pump_records: 1, + usable_dual_frames: 1, empty_channel_frames: 0, oversize_channel_frames: 0, invalid_timestamps: 0, timestamp_reversals: 0, + duplicate_timestamps: 0, first_event_ms: start - 300000 + index * 450000, last_event_ms: start + 149999 + index * 450000, + channels: { left1: { samples: 10, lost_markers: 0, truncated_tail_bytes: 0 }, right1: { samples: 10, lost_markers: 0, truncated_tail_bytes: 0 } }, + ordered_waveform_available: true, limitations: [] } }) + }) +}) +afterEach(() => sqlite.close()) +const request = () => ({ parentDatasetId: parent.id, expectedContentHash: parent.contentHash, idempotencyKey: 'waveform', + cohorts: [{ cohortId: datasetManifestSchema.parse(parent.manifest).splitProtocol.assignments[0].cohortId, sourceIds: [...sourceIds].reverse() }] }) + +it('resolves ordered immutable source identities and recorded occupancy without reading current state', () => { + sqlite.prepare('UPDATE sleep_records SET present_intervals = NULL, not_present_intervals = NULL').run() + const result = prepareSelectedWaveformDataset(db, request(), 123) + expect(result.selections.map(item => item.source_name)).toEqual(['0.RAW.gz', '1.RAW.gz']) + if (result.prepared.snapshot.version !== 'waveform-dataset-v1') throw new Error('Wrong snapshot') + expect(result.prepared.snapshot.waveformInputs[0].input).toMatchObject({ retained_start_ms: start - 300000, + score_start_ms: start, score_end_ms: start + 600000, contexts: [ + { start_ms: start, end_ms: start + 60000, left_occupied: true, right_occupied: null, active_sides: null, pump_context_known: false }, + { start_ms: start + 60000, end_ms: start + 120000, left_occupied: false, right_occupied: null, active_sides: null, pump_context_known: false }, + ] }) + const retry = request() + retry.cohorts[0].sourceIds.reverse() + expect(prepareSelectedWaveformDataset(db, retry, 456).prepared.contentHash).toBe(result.prepared.contentHash) + expect(sqlite.prepare('SELECT count(*) n FROM biometric_datasets').get()).toEqual({ n: 1 }) + expect(sqlite.prepare('SELECT count(*) n FROM biometric_raw_artifacts').get()).toEqual({ n: 0 }) +}) + +it('rejects stale, duplicate, incomplete and forged selections', () => { + expect(() => prepareSelectedWaveformDataset(db, { ...request(), expectedContentHash: '0'.repeat(64) })).toThrow('expected immutable') + expect(() => prepareSelectedWaveformDataset(db, { ...request(), cohorts: [...request().cohorts, ...request().cohorts] })).toThrow('each whole-night') + const duplicate = request() + duplicate.cohorts[0].sourceIds = [sourceIds[0], sourceIds[0]] + expect(() => prepareSelectedWaveformDataset(db, duplicate)).toThrow('each RAW source') + const missing = request() + missing.cohorts[0].cohortId = 'missing' + expect(() => prepareSelectedWaveformDataset(db, missing)).toThrow('Missing whole-night') + expect(() => createWaveformDatasetSchema.parse({ ...request(), archiveDirectory: '/tmp' })).toThrow() + new RawSourceCatalog(db).markUnavailable(sourceIds[0]) + expect(() => prepareSelectedWaveformDataset(db, request())).toThrow('unavailable') +}) + +it('retains unknown context for gaps, malformed evidence and contradictory intervals', () => { + const snapshot = structuredClone(parent.snapshot) + snapshot.sessions[0].presentIntervals = [[start / 1000 - 10, start / 1000 + 60], ['bad', 10], [Infinity, 50], [10, 0]] + snapshot.sessions[0].notPresentIntervals = [[start / 1000 + 30, start / 1000 + 90]] + expect(recordedWaveformContexts(snapshot, start - 300000, start + 600000)).toEqual([ + { start_ms: start, end_ms: start + 30000, left_occupied: true, right_occupied: null, active_sides: null, pump_context_known: false }, + { start_ms: start + 60000, end_ms: start + 90000, left_occupied: false, right_occupied: null, active_sides: null, pump_context_known: false }, + ]) + snapshot.side = 'right' + expect(recordedWaveformContexts(snapshot, start + 10000, start + 20000)).toEqual([ + { start_ms: start + 10000, end_ms: start + 20000, right_occupied: true, left_occupied: null, active_sides: null, pump_context_known: false }, + ]) + snapshot.sessions[0].presentIntervals = null + snapshot.sessions[0].notPresentIntervals = { unrecognized: true } + expect(recordedWaveformContexts(snapshot, start, start + 600000)).toEqual([]) +}) + +it('merges consistent overlapping intervals, preserves fractional seconds and bounds recorded evidence', () => { + const snapshot = structuredClone(parent.snapshot) + snapshot.sessions[0].presentIntervals = [[start / 1000 + 0.125, start / 1000 + 40], [start / 1000 + 30, start / 1000 + 60]] + snapshot.sessions[0].notPresentIntervals = null + expect(recordedWaveformContexts(snapshot, start, start + 600000)).toEqual([ + { start_ms: start + 125, end_ms: start + 60000, left_occupied: true, right_occupied: null, active_sides: null, pump_context_known: false }, + ]) + snapshot.sessions[0].presentIntervals = Array(20001).fill([1, 2]) + expect(() => recordedWaveformContexts(snapshot, start, start + 600000)).toThrow('budget') +}) diff --git a/src/biometrics/backtest/trace-contracts.ts b/src/biometrics/backtest/trace-contracts.ts new file mode 100644 index 000000000..06fd06bc7 --- /dev/null +++ b/src/biometrics/backtest/trace-contracts.ts @@ -0,0 +1,12 @@ +import { z } from 'zod' +export const traceRequestSchema = z.object({ id: z.string().uuid(), expectedContentHash: z.string().regex(/^[a-f0-9]{64}$/), + configuration: z.enum(['default', 'current', 'candidate']), cohortId: z.string().min(1).max(200).optional(), + offset: z.number().int().min(0).max(1000000).default(0), limit: z.number().int().min(1).max(500).default(200), +}).strict() +export const tracePageSchema = z.object({ jobId: z.string().uuid(), contentHash: z.string(), configuration: z.enum(['default', 'current', 'candidate']), + provenance: z.literal('versioned_stage_replay'), timestampBasis: z.literal('original_trace_before_reference_clock_offset'), clockOffsetMs: z.number(), + totalItems: z.number().int(), offset: z.number().int(), limit: z.number().int(), downsampled: z.literal(false), + items: z.array(z.object({ cohortId: z.string(), configurationHash: z.string(), revisionId: z.string().uuid().nullable(), + windowStartMs: z.number(), windowEndMs: z.number(), startMs: z.number(), endMs: z.number(), stage: z.enum(['wake', 'light', 'deep', 'rem']) })), + nextCursor: z.object({ offset: z.number().int(), contentHash: z.string() }).nullable(), +}) diff --git a/src/biometrics/backtest/trace-page.ts b/src/biometrics/backtest/trace-page.ts new file mode 100644 index 000000000..ba8825b7b --- /dev/null +++ b/src/biometrics/backtest/trace-page.ts @@ -0,0 +1,47 @@ +import { z } from 'zod' +import { ReferenceInputError } from '../references/time' +import { traceRequestSchema, tracePageSchema } from './trace-contracts' + +const reportSchema = z.object({ version: z.enum(['processed-backtest-report-v1', 'waveform-backtest-report-v1']), + protocol: z.object({ mode: z.enum(['comparison', 'stage_replay', 'signal_replay']), waveformStages: z.boolean().optional(), clockOffsetMs: z.number().int() }), + nights: z.array(z.object({ cohortId: z.string(), evaluations: z.array(z.object({ label: z.enum(['default', 'current', 'candidate']), + configHash: z.string(), revisionId: z.string().uuid().nullable(), windows: z.array(z.object({ startMs: z.number().int(), endMs: z.number().int(), + blocks: z.array(z.object({ start: z.number().int(), end: z.number().int(), stage: z.enum(['wake', 'light', 'deep', 'rem']) })), + })), + })).max(3) })).max(60), +}) + +/** Selection changes only returned detail, never the retained evaluation or + * its scoring. Offset cursors are pinned to an immutable verified report hash. */ +export function pageStageTraces(report: unknown, raw: z.input) { + const input = traceRequestSchema.parse(raw) + const parsed = reportSchema.safeParse(report) + if (!parsed.success) throw new ReferenceInputError('RESULT_CORRUPT', 'Report trace format is unavailable') + const data = parsed.data + if (data.protocol.mode !== 'stage_replay' && !(data.protocol.mode === 'signal_replay' && data.protocol.waveformStages)) throw new ReferenceInputError('INCOMPATIBLE_METRIC', 'This comparison did not produce stage replay traces') + const nights = input.cohortId === undefined ? data.nights : data.nights.filter(night => night.cohortId === input.cohortId) + if (!nights.length || !nights.some(night => night.evaluations.some(evaluation => evaluation.label === input.configuration))) { + throw new ReferenceInputError('NOT_FOUND', 'Requested cohort or configuration was not evaluated') + } + const items: z.infer['items'] = [] + let totalItems = 0 + for (const night of nights) { + for (const evaluation of night.evaluations.filter(row => row.label === input.configuration)) { + for (const window of evaluation.windows) { + for (const block of window.blocks) { + if (totalItems >= input.offset && items.length < input.limit) items.push({ cohortId: night.cohortId, configurationHash: evaluation.configHash, + revisionId: evaluation.revisionId, windowStartMs: window.startMs, windowEndMs: window.endMs, + startMs: block.start, endMs: block.end, stage: block.stage }) + totalItems++ + } + } + } + } + if (input.offset > totalItems) throw new ReferenceInputError('INVALID_CURSOR', 'Trace cursor exceeds the selected report') + const next = input.offset + items.length + return tracePageSchema.parse({ jobId: input.id, contentHash: input.expectedContentHash, configuration: input.configuration, + provenance: 'versioned_stage_replay', timestampBasis: 'original_trace_before_reference_clock_offset', clockOffsetMs: data.protocol.clockOffsetMs, + items, totalItems, offset: input.offset, limit: input.limit, downsampled: false, + nextCursor: next < totalItems ? { offset: next, contentHash: input.expectedContentHash } : null }) +} + diff --git a/src/biometrics/backtest/traces.ts b/src/biometrics/backtest/traces.ts new file mode 100644 index 000000000..0b5be7b0f --- /dev/null +++ b/src/biometrics/backtest/traces.ts @@ -0,0 +1,40 @@ +import { DatasetStore } from './datasets' +import { pageReferenceTraces, referenceTraceIdentity, referenceTraceRequestSchema } from './reference-trace-page' +import { pageNumericTraces } from './numeric-trace-page' +import type { z } from 'zod' +import { pageStageTraces } from './trace-page' +export { pageStageTraces } from './trace-page' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import type * as schema from '@/src/db/biometrics-schema' +import { ReferenceInputError } from '../references/time' +import { BacktestJobStore } from './jobs' +import { traceRequestSchema } from './trace-contracts' + +export function readStageTraces(db: BetterSQLite3Database, raw: z.input) { + const input = traceRequestSchema.parse(raw) + const result = new BacktestJobStore(db).result(input.id) + if (!result) throw new ReferenceInputError('INVALID_STATE', 'Wait for a successful replay before reading traces') + if (result.contentHash !== input.expectedContentHash) throw new ReferenceInputError('REVISION_CONFLICT', 'Trace report hash changed') + if (!result.available) throw new ReferenceInputError('RESULT_RELEASED', 'Replay report was released') + return pageStageTraces(result.report, input) +} + +export function readNumericTraces(db: BetterSQLite3Database, raw: z.input) { + const input = traceRequestSchema.parse(raw) + const result = new BacktestJobStore(db).result(input.id) + if (!result) throw new ReferenceInputError('INVALID_STATE', 'Wait for a successful replay before reading traces') + if (result.contentHash !== input.expectedContentHash) throw new ReferenceInputError('REVISION_CONFLICT', 'Trace report hash changed') + if (!result.available) throw new ReferenceInputError('RESULT_RELEASED', 'Replay report was released') + return pageNumericTraces(result.report, input) +} + +export function readReferenceTraces(db: BetterSQLite3Database, raw: z.input) { + const input = referenceTraceRequestSchema.parse(raw) + const result = new BacktestJobStore(db).result(input.id) + if (!result) throw new ReferenceInputError('INVALID_STATE', 'Wait for a successful replay before reading references') + if (result.contentHash !== input.expectedContentHash) throw new ReferenceInputError('REVISION_CONFLICT', 'Reference report hash changed') + if (!result.available) throw new ReferenceInputError('RESULT_RELEASED', 'Replay report was released') + const identity = referenceTraceIdentity(result.report) + const dataset = new DatasetStore(db).read(identity.datasetId) + return pageReferenceTraces(result.report, dataset, input) +} diff --git a/src/biometrics/backtest/uncertainty-core.ts b/src/biometrics/backtest/uncertainty-core.ts new file mode 100644 index 000000000..122b8b148 --- /dev/null +++ b/src/biometrics/backtest/uncertainty-core.ts @@ -0,0 +1,57 @@ +import { nightBootstrapSchema, type NightBootstrapPolicy } from './uncertainty-policy' +export type { NightBootstrapPolicy } from './uncertainty-policy' + +/** Versioned, platform-independent SHA-256 counter stream. Rejection sampling + * avoids modulo bias. This seed is for reproducibility, not security. */ +function indexStream(seed: string, size: number, digest: (text: string) => Uint8Array) { + let counter = 0, position = 32, bytes: Uint8Array = new Uint8Array(32) + const ceiling = 4294967296 - (4294967296 % size) + return () => { + for (;;) { + if (position === 32) { + bytes = digest(JSON.stringify(['paired-night-percentile-v1', seed, counter++])) + position = 0 + } + const value = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength).getUint32(position, false) + position += 4 + if (value < ceiling) return value % size + } + } +} +const average = (values: number[]) => values.reduce((sum, value) => sum + value, 0) / values.length +function quantile(sorted: number[], probability: number) { + const position = (sorted.length - 1) * probability + const lower = Math.floor(position), fraction = position - lower + return sorted[lower] + fraction * (sorted[Math.min(lower + 1, sorted.length - 1)] - sorted[lower]) +} + +/** Resample paired whole-night differences, never correlated epochs. An + * exploratory interval alone cannot award improvement or validate activation. */ +export function createNightUncertainty(digest: (text: string) => Uint8Array, statistic: 'equal_night_mean_common_agreement_difference' | 'equal_night_mean_common_mae_difference' = 'equal_night_mean_common_agreement_difference') { + return function nightUncertainty(input: Array<{ cohortId: string, difference: number | null }>, rawPolicy?: NightBootstrapPolicy) { + if (input.length > 60 || new Set(input.map(row => row.cohortId)).size !== input.length + || input.some(row => row.difference !== null && (!Number.isFinite(row.difference) || Math.abs(row.difference) > (statistic === 'equal_night_mean_common_agreement_difference' ? 1 : 2 * Number.MAX_SAFE_INTEGER)))) throw new Error('Invalid paired-night uncertainty input') + const policy = rawPolicy === undefined ? null : nightBootstrapSchema.parse(rawPolicy) + const ordered = [...input].sort((a, b) => a.cohortId < b.cohortId ? -1 : a.cohortId > b.cohortId ? 1 : 0) + const values = ordered.flatMap(row => row.difference === null ? [] : [row.difference]) + const metadata = { policy, confidenceLevel: 0.95, samplingUnit: 'whole_night_cohort' as const, + statistic, scoredNights: values.length, omittedNights: input.length - values.length, + estimate: values.length ? average(values) : null, + limitations: ['Exploratory interval assumes exchangeable nights; serial dependence may understate uncertainty.', + 'The requested minimum night count is a reporting threshold, not proof of sufficient evidence.', + 'Missing-reference nights are omitted and remain counted; the interval does not establish whole-history accuracy.', + 'This interval is not a promotion decision or clinical validation.'] } + if (!policy) return { ...metadata, status: 'not_requested' as const, interval: null } + if (values.length < policy.minimumNights) return { ...metadata, status: 'insufficient_nights' as const, interval: null } + if (values.every(value => value === values[0])) return { ...metadata, status: 'degenerate' as const, interval: null } + const index = indexStream(policy.seed, values.length, digest) + const samples: number[] = [] + for (let replicate = 0; replicate < policy.resamples; replicate++) { + let sum = 0 + for (let draw = 0; draw < values.length; draw++) sum += values[index()] + samples.push(sum / values.length) + } + samples.sort((a, b) => a - b) + return { ...metadata, status: 'estimated' as const, interval: { lower: quantile(samples, 0.025), upper: quantile(samples, 0.975) } } + } +} diff --git a/src/biometrics/backtest/uncertainty-policy.ts b/src/biometrics/backtest/uncertainty-policy.ts new file mode 100644 index 000000000..99102e78e --- /dev/null +++ b/src/biometrics/backtest/uncertainty-policy.ts @@ -0,0 +1,8 @@ +import { z } from 'zod' + +export const nightBootstrapSchema = z.object({ + version: z.literal('paired-night-percentile-v1'), seed: z.string().min(1).max(64), + resamples: z.number().int().min(200).max(5000), minimumNights: z.number().int().min(3).max(60), +}).strict() +export type NightBootstrapPolicy = z.infer + diff --git a/src/biometrics/backtest/uncertainty.ts b/src/biometrics/backtest/uncertainty.ts new file mode 100644 index 000000000..48182d826 --- /dev/null +++ b/src/biometrics/backtest/uncertainty.ts @@ -0,0 +1,4 @@ +import { createHash } from 'node:crypto' +import { createNightUncertainty } from './uncertainty-core' +export type { NightBootstrapPolicy } from './uncertainty-policy' +export const nightUncertainty = createNightUncertainty(text => createHash('sha256').update(text).digest()) diff --git a/src/biometrics/backtest/waveform-build.ts b/src/biometrics/backtest/waveform-build.ts new file mode 100644 index 000000000..d64852937 --- /dev/null +++ b/src/biometrics/backtest/waveform-build.ts @@ -0,0 +1,44 @@ +import { execFile } from 'node:child_process' +import { tmpdir } from 'node:os' +import { promisify } from 'node:util' +import { z } from 'zod' +import { configHash } from '../algorithms/canonical' +import { workerBuildHash } from './child' +import type { WaveformWorkerLocation } from './waveform-child' + +const hash = z.string().regex(/^[a-f0-9]{64}$/) +const label = z.string().min(1).max(1000) +const dependency = z.object({ version: label, record_sha256: hash, wheel_sha256: hash }).strict() +export const waveformPythonBuildSchema = z.object({ hash, identity: z.object({ + version: z.literal('waveform-python-build-v1'), + sources: z.record(z.string().regex(/^(common|piezo-processor)\/[A-Za-z0-9_./-]+$/), hash) + .refine(value => Object.keys(value).length > 0 && Object.keys(value).length <= 64), + dependencies: z.object({ numpy: dependency, scipy: dependency }).strict(), + runtime: z.object({ implementation: label, version: label, cache_tag: label.nullable(), soabi: label.nullable(), + machine: label, system: label, byteorder: z.enum(['little', 'big']) }).strict(), +}).strict() }).strict() + +/** Read-only bounded probe of the exact interpreter used for replay. Locations + * are trusted deployment configuration; no command or source comes from RPC. */ +export async function probeWaveformBuild(location: Pick, signal?: AbortSignal) { + const { stdout } = await promisify(execFile)(location.pythonPath, ['-B', '-m', 'common.biometrics.replay_worker', '--describe-build'], { + cwd: tmpdir(), env: { NODE_ENV: 'production', PYTHONPATH: location.modulesPath, LANG: 'C.UTF-8', TZ: 'UTC', + OPENBLAS_NUM_THREADS: '1', OMP_NUM_THREADS: '1', MKL_NUM_THREADS: '1', NUMEXPR_NUM_THREADS: '1' }, + encoding: 'utf8', maxBuffer: 65536, timeout: 10000, killSignal: 'SIGKILL', signal, + }).catch(() => { throw new Error('Waveform build identity is unavailable') }) + const result = waveformPythonBuildSchema.parse(JSON.parse(stdout)) + if (configHash(result.identity) !== result.hash) throw new Error('Waveform build identity integrity mismatch') + return result +} + +/** Pin orchestration, scoring, Python code and installed numerical wheel + * metadata together. Architecture/runtime changes deliberately invalidate it. */ +export async function waveformEvaluatorBuildIdentity(coordinatorPath: string, workerPath: string, + location: Pick, signal?: AbortSignal) { + const [coordinatorHash, evaluatorHash, python] = await Promise.all([ + workerBuildHash(coordinatorPath), workerBuildHash(workerPath), probeWaveformBuild(location, signal), + ]) + const identity = { version: 'waveform-evaluator-build-v1', coordinatorHash, evaluatorHash, pythonHash: python.hash, + node: { version: process.version, platform: process.platform, arch: process.arch, icu: process.versions.icu ?? null } } + return { hash: configHash(identity), identity, python } +} diff --git a/src/biometrics/backtest/waveform-child.ts b/src/biometrics/backtest/waveform-child.ts new file mode 100644 index 000000000..d0c901783 --- /dev/null +++ b/src/biometrics/backtest/waveform-child.ts @@ -0,0 +1,99 @@ +import { spawn, type ChildProcessWithoutNullStreams } from 'node:child_process' +import { tmpdir } from 'node:os' +import { StringDecoder } from 'node:string_decoder' +import { z } from 'zod' + +import { waveformReadingSchema, waveformRunSchema } from './waveform-output' +export { waveformReadingSchema, waveformRunSchema } from './waveform-output' +const hash = z.string().regex(/^[a-f0-9]{64}$/) +const eventSchema = z.discriminatedUnion('type', [ + z.object({ type: z.literal('reading'), reading: waveformReadingSchema }).strict(), + z.object({ type: z.literal('result'), result: waveformRunSchema }).strict(), +]) +export interface WaveformWorkerLocation { + pythonPath: string + modulesPath: string + retainedDirectory: string +} + +/** Locations are trusted server configuration, never fields accepted by RPC. */ +export function runWaveformChild(location: WaveformWorkerLocation, work: unknown, options: { + signal?: AbortSignal + timeoutMs?: number + expectedBuildHash?: string + cpuSeconds?: number +} = {}): Promise<{ readings: z.infer[], run: z.infer }> { + if (options.cpuSeconds !== undefined && (!Number.isSafeInteger(options.cpuSeconds) || options.cpuSeconds < 1 || options.cpuSeconds > 3600)) return Promise.reject(new Error('Invalid waveform CPU budget')) + if (options.expectedBuildHash !== undefined) hash.parse(options.expectedBuildHash) + const serialized = JSON.stringify(options.expectedBuildHash === undefined ? work : { ...z.record(z.string(), z.unknown()).parse(work), expected_build_hash: options.expectedBuildHash }) + if (!serialized || Buffer.byteLength(serialized) > 8 * 1024 * 1024) return Promise.reject(new Error('Waveform input exceeds 8 MiB')) + const timeoutMs = options.timeoutMs ?? 120000 + if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 10 || timeoutMs > 3600000) return Promise.reject(new Error('Invalid waveform time budget')) + if (options.signal?.aborted) return Promise.reject(new Error('Waveform replay cancelled')) + return new Promise((resolve, reject) => { + const child: ChildProcessWithoutNullStreams = spawn(location.pythonPath, ['-B', '-m', 'common.biometrics.replay_worker', '--retained-directory', location.retainedDirectory, ...(options.cpuSeconds === undefined ? [] : ['--cpu-seconds', String(options.cpuSeconds)])], { + cwd: tmpdir(), env: { NODE_ENV: 'production', PYTHONPATH: location.modulesPath, LANG: 'C.UTF-8', TZ: 'UTC', + OPENBLAS_NUM_THREADS: '1', OMP_NUM_THREADS: '1', MKL_NUM_THREADS: '1', NUMEXPR_NUM_THREADS: '1' }, + stdio: ['pipe', 'pipe', 'pipe'], + }) + const readings: z.infer[] = [] + let result: z.infer | undefined + const decoder = new StringDecoder('utf8') + let pending = '', outputBytes = 0, diagnosticBytes = 0 + let failure: Error | undefined, killTimer: ReturnType | undefined + const stop = (error: Error) => { + failure ??= error + child.kill('SIGTERM') + killTimer ??= setTimeout(() => child.kill('SIGKILL'), 500) + } + const timeout = setTimeout(() => stop(new Error('Waveform replay exceeded its time budget')), timeoutMs) + const abort = () => stop(new Error('Waveform replay cancelled')) + options.signal?.addEventListener('abort', abort, { once: true }) + if (options.signal?.aborted) abort() + child.on('error', () => stop(new Error('Waveform worker could not start'))) + child.stdin.on('error', () => stop(new Error('Waveform worker input pipe closed'))) + child.stderr.on('data', (chunk: Buffer) => { + diagnosticBytes += chunk.length + if (diagnosticBytes > 16384) stop(new Error('Waveform diagnostic output exceeded its bound')) + }) + child.stdout.on('data', (chunk: Buffer) => { + if (failure) return + outputBytes += chunk.length + if (outputBytes > 16 * 1024 * 1024) return stop(new Error('Waveform output exceeds 16 MiB')) + pending += decoder.write(chunk) + let newline: number + while ((newline = pending.indexOf('\n')) >= 0) { + const line = pending.slice(0, newline) + pending = pending.slice(newline + 1) + try { + if (result || Buffer.byteLength(line) > 65536) throw new Error('Invalid event order or length') + const event = eventSchema.parse(JSON.parse(line)) + if (event.type === 'reading') { + if (readings.length >= 20000) throw new Error('Too many waveform readings') + readings.push(event.reading) + } + else result = event.result + } + catch { + stop(new Error('Waveform worker emitted an invalid event')) + return + } + } + if (Buffer.byteLength(pending) > 65536) stop(new Error('Waveform event exceeds its byte bound')) + }) + child.on('close', (code) => { + pending += decoder.end() + clearTimeout(timeout) + if (killTimer) clearTimeout(killTimer) + options.signal?.removeEventListener('abort', abort) + if (failure) reject(failure) + else if (code !== 0 || !result || (options.expectedBuildHash !== undefined && result.build_hash !== options.expectedBuildHash) || pending.length || result.counts.scored_outputs !== readings.length + || result.raw_records < result.decoded_records || result.decoded_records < result.counts.records + || readings.some(reading => reading.configHash !== result?.config_hashes[reading.side])) { + reject(new Error('Waveform worker failed or returned an incomplete result')) + } + else resolve({ readings, run: result }) + }) + child.stdin.end(serialized) + }) +} diff --git a/src/biometrics/backtest/waveform-contracts.ts b/src/biometrics/backtest/waveform-contracts.ts new file mode 100644 index 000000000..2f1e16cc8 --- /dev/null +++ b/src/biometrics/backtest/waveform-contracts.ts @@ -0,0 +1,57 @@ +import { z } from 'zod' + +const integer = z.number().int().min(0).max(Number.MAX_SAFE_INTEGER) +const hash = z.string().regex(/^[a-f0-9]{64}$/) +export const rawArtifactDescriptorSchema = z.object({ + name: z.string(), sha256: hash, compressed_bytes: integer, expanded_bytes: integer, +}).strict().refine(value => value.name === `${value.sha256}.RAW.gz`, 'Artifact name must match its content hash') +const context = z.object({ + start_ms: integer, end_ms: integer, + active_sides: z.array(z.enum(['left', 'right'])).max(2).nullable().refine(value => value === null + || value.length < 2 || (value[0] === 'left' && value[1] === 'right'), 'Invalid recorded sleeper mode'), + left_occupied: z.boolean().nullable(), right_occupied: z.boolean().nullable(), pump_context_known: z.boolean(), +}).strict() + +/** The same immutable input envelope consumed by the shipped Python driver. */ +export const retainedWaveformInputSchema = z.object({ + version: z.literal('retained-waveform-input-v1'), + retained_start_ms: integer, score_start_ms: integer, score_end_ms: integer, + artifacts: z.array(z.object({ artifact: rawArtifactDescriptorSchema, first_event_ms: integer, last_event_ms: integer }).strict()).min(1).max(64), + contexts: z.array(context).max(20000), +}).strict().superRefine((value, ctx) => { + const invalid = (message: string) => ctx.addIssue({ code: 'custom', message }) + if (!(value.retained_start_ms <= value.score_start_ms && value.score_start_ms < value.score_end_ms)) invalid('Invalid retained replay window') + const hashes = new Set() + let previous = -1, compressed = 0 + for (const source of value.artifacts) { + if (source.first_event_ms > source.last_event_ms || source.last_event_ms < value.retained_start_ms || source.first_event_ms >= value.score_end_ms) invalid('Artifact has no events in the retained window') + if (hashes.has(source.artifact.sha256) || source.first_event_ms <= previous) invalid('Retained artifacts must be distinct with nonoverlapping ordered event bounds') + hashes.add(source.artifact.sha256) + previous = source.last_event_ms + compressed += source.artifact.compressed_bytes + } + if (compressed > 1024 * 1024 * 1024) invalid('Retained replay compressed input budget exceeded') + previous = value.retained_start_ms + for (const interval of value.contexts) { + if (interval.start_ms < previous || interval.start_ms >= interval.end_ms || interval.end_ms > value.score_end_ms) invalid('Context intervals must be ordered and nonoverlapping inside the retained window') + previous = interval.end_ms + } +}) + +export const waveformDatasetInputsSchema = z.array(z.object({ + cohortId: z.string().min(1).max(200), input: retainedWaveformInputSchema, +}).strict()).min(1).max(60).superRefine((value, ctx) => { + if (new Set(value.map(item => item.cohortId)).size !== value.length) ctx.addIssue({ code: 'custom', message: 'Select each waveform cohort once' }) + const artifacts = new Map() + let compressed = 0 + for (const cohort of value) { + for (const source of cohort.input.artifacts) { + const previous = artifacts.get(source.artifact.sha256) + const encoded = JSON.stringify(source) + if (previous && previous !== encoded) ctx.addIssue({ code: 'custom', message: 'Conflicting evidence for shared waveform artifact' }) + if (!previous) compressed += source.artifact.compressed_bytes + artifacts.set(source.artifact.sha256, encoded) + } + } + if (artifacts.size > 1024 || compressed > 1024 * 1024 * 1024) ctx.addIssue({ code: 'custom', message: 'Waveform dataset RAW storage budget exceeded' }) +}) diff --git a/src/biometrics/backtest/waveform-dataset.ts b/src/biometrics/backtest/waveform-dataset.ts new file mode 100644 index 000000000..e0bf924e3 --- /dev/null +++ b/src/biometrics/backtest/waveform-dataset.ts @@ -0,0 +1,45 @@ +import { randomUUID } from 'node:crypto' +import { z } from 'zod' +import { configHash, dataHash } from '../algorithms/canonical' +import { datasetManifestSchema, datasetSnapshotSchema } from './contracts' +import { waveformDatasetInputsSchema } from './waveform-contracts' + +const sourceSchema = z.object({ id: z.string().uuid(), contentHash: z.string().regex(/^[a-f0-9]{64}$/), + manifest: datasetManifestSchema, snapshot: datasetSnapshotSchema }) + +/** Server-internal preparation. Sources and context must come from retained evidence. */ +export function prepareWaveformDataset(source: z.input, rawInputs: unknown, idempotencyKey: string, now = Date.now()) { + const base = sourceSchema.parse(structuredClone(source)) + if (base.snapshot.version !== 'processed-dataset-v1' || base.manifest.version !== 'processed-dataset-manifest-v1') throw new Error('Select a processed source dataset') + if (dataHash({ manifest: base.manifest, snapshot: base.snapshot }) !== base.contentHash) throw new Error('Source dataset integrity mismatch') + z.string().min(1).max(200).parse(idempotencyKey) + z.number().int().min(0).max(Number.MAX_SAFE_INTEGER).parse(now) + const inputs = waveformDatasetInputsSchema.parse(rawInputs) + const cohorts = new Map(base.manifest.splitProtocol.assignments.map(item => [item.cohortId, item])) + if (cohorts.size !== base.manifest.splitProtocol.assignments.length || inputs.length !== cohorts.size) throw new Error('Retain waveform inputs for every whole-night cohort') + for (const item of inputs) { + const cohort = cohorts.get(item.cohortId) + if (!cohort || item.input.score_start_ms !== cohort.startMs || item.input.score_end_ms !== cohort.endMs) throw new Error('Waveform input must match its whole-night cohort') + } + // Follow the immutable split assignment order, independent of selection UI order. + const order = [...cohorts.keys()] + inputs.sort((a, b) => order.indexOf(a.cohortId) - order.indexOf(b.cohortId)) + const snapshot = datasetSnapshotSchema.parse({ ...base.snapshot, version: 'waveform-dataset-v1', waveformInputs: inputs }) + const manifest = datasetManifestSchema.parse({ ...base.manifest, version: 'waveform-dataset-manifest-v1', + parentDataset: { id: base.id, contentHash: base.contentHash }, + capabilities: { ...base.manifest.capabilities, signalReplay: true }, + limitations: ['Waveform files and supplied context intervals are frozen; execution requires configured deployment admission and current resource health.', + 'Event bounds do not establish continuous waveform coverage; gaps in historical context remain unknown.', + 'Replay uses event time and a cold start with retained pre-roll, not serialized historical processor state.', + 'Measured calibration is not reconstructed from current calibration settings.', + 'Historical stage replay must declare its calibration quality assumption when original calibration is unavailable.', + 'Reference deletion does not erase this separately retained dataset.', + 'Historical occupant identity is asserted by the dataset creator, not measured by the pod.'], + }) + const contentHash = dataHash({ manifest, snapshot }) + const prepared = { id: randomUUID(), side: snapshot.side, subjectId: snapshot.subjectId, + idempotencyKey, requestHash: configHash({ version: 'waveform-dataset-request-v1', parentId: base.id, + parentHash: base.contentHash, inputsHash: dataHash(inputs), idempotencyKey }), contentHash, manifest, snapshot, createdAtMs: now } + if (Buffer.byteLength(JSON.stringify(prepared)) > 8 * 1024 * 1024) throw new Error('Waveform dataset exceeds its byte budget') + return prepared +} diff --git a/src/biometrics/backtest/waveform-execution.ts b/src/biometrics/backtest/waveform-execution.ts new file mode 100644 index 000000000..9cf09ea6d --- /dev/null +++ b/src/biometrics/backtest/waveform-execution.ts @@ -0,0 +1,63 @@ +import type { ReplayBudget } from './execution-budget' +import { sql } from 'drizzle-orm' +import { dirname, join } from 'node:path' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import type { z } from 'zod' +import type * as schema from '@/src/db/biometrics-schema' +import { dataHash } from '../algorithms/canonical' +import type { backtestWorkSchema } from './evaluator' +import { waveformEvaluationSchema } from './evaluator' +import { datasetManifestSchema } from './contracts' +import { podRawInspectionLocation, type RawInspectionLocation } from './raw-inspection-child' +import { waveformEvaluatorBuildIdentity } from './waveform-build' +import { runWaveformChild } from './waveform-child' + +export async function verifyWaveformJobBuild(work: z.infer, options: { + workerPath: string + rawLocation?: RawInspectionLocation + signal: AbortSignal +}) { + const build = await waveformEvaluatorBuildIdentity(join(dirname(options.workerPath), 'coordinator.cjs'), options.workerPath, + options.rawLocation ?? podRawInspectionLocation, options.signal) + if (build.hash !== work.job.evaluatorBuildHash || build.python.hash !== work.job.waveform?.pythonBuildHash) throw new Error('Waveform worker build changed since enqueue') +} + +/** Run one cohort/configuration at a time. Raw bytes stay in Python; only + * bounded derived rows reach the isolated Node scorer and retained report. */ +export async function executeWaveformInputs(db: BetterSQLite3Database, work: z.infer, options: { + workerPath: string + rawLocation?: RawInspectionLocation + signal: AbortSignal + timeoutMs?: number + replayBudget?: ReplayBudget +}) { + const { job, dataset } = work + if (job.protocol.mode !== 'signal_replay' || !job.waveform || dataset.snapshot.version !== 'waveform-dataset-v1') throw new Error('Missing retained waveform job inputs') + const databasePath = db.all<{ name: string, file: string }>(sql`PRAGMA database_list`).find(row => row.name === 'main')?.file + if (!databasePath) throw new Error('Waveform execution requires a persistent database') + const location = { ...(options.rawLocation ?? podRawInspectionLocation), retainedDirectory: join(dirname(databasePath), 'biometric-raw') } + await verifyWaveformJobBuild(work, options) + const assignments = datasetManifestSchema.parse(dataset.manifest).splitProtocol.assignments.filter(row => row.split === job.split) + const configurations = [...new Map(job.configs.map(config => [config.resolvedHash, config])).values()] + const outputs: z.infer['outputs'] = [] + let count = 0 + for (const cohort of assignments) { + const input = dataset.snapshot.waveformInputs.find(row => row.cohortId === cohort.cohortId)?.input + if (!input) throw new Error('Missing retained waveform cohort') + for (const config of configurations) { + if (options.signal.aborted) throw new Error('Waveform job cancelled') + const configs = dataset.snapshot.side === 'left' + ? { left: config.resolved, right: job.waveform.otherSide.resolved } + : { left: job.waveform.otherSide.resolved, right: config.resolved } + const output = await runWaveformChild(location, { version: 'retained-waveform-work-v1', input, configs }, { + signal: options.signal, timeoutMs: options.replayBudget?.waveformWallMs ?? options.timeoutMs, + cpuSeconds: options.replayBudget?.waveformCpuSeconds, expectedBuildHash: job.waveform.pythonBuildHash, + }) + count += output.readings.length + if (count > 20000) throw new Error('Waveform job derived reading budget exceeded') + outputs.push({ cohortId: cohort.cohortId, configHash: config.resolvedHash, ...output }) + } + } + await verifyWaveformJobBuild(work, options) + return waveformEvaluationSchema.parse({ hash: dataHash(outputs), outputs }) +} diff --git a/src/biometrics/backtest/waveform-output.ts b/src/biometrics/backtest/waveform-output.ts new file mode 100644 index 000000000..a00eb7f93 --- /dev/null +++ b/src/biometrics/backtest/waveform-output.ts @@ -0,0 +1,33 @@ +import { z } from 'zod' + +const count = z.number().int().min(0).max(Number.MAX_SAFE_INTEGER) +const hash = z.string().regex(/^[a-f0-9]{64}$/) +const side = z.enum(['left', 'right']) +const value = z.number().finite().nullable() +const limitations = z.array(z.string().max(200)).max(40) +export const waveformReadingSchema = z.object({ + side, timestampMs: count, heartRate: value, hrvIndex: value, + hrvMetric: z.literal('window_ibi_variability_v1'), breathingRate: value, + qualityScore: z.number().finite(), flags: z.array(z.string().max(200)).max(40).nullable(), hrRaw: value, + sourceSide: side, configHash: hash, limitations, +}).strict() +/** Optional only for retained reports produced before resource collection. + * These are one Python process's costs, not whole-job or live-ingestion costs. */ +export const waveformProcessResourcesSchema = z.object({ + version: z.literal('waveform-process-resources-v1'), + wallMs: z.number().finite().nonnegative(), cpuMs: z.number().finite().nonnegative(), + peakRssBytes: z.number().int().positive().max(Number.MAX_SAFE_INTEGER).nullable(), + scope: z.literal('python_process_until_result'), memoryScope: z.literal('process_lifetime_high_water_mark'), +}).strict() +export const waveformRunSchema = z.object({ + resources: waveformProcessResourcesSchema.optional(), + version: z.literal('retained-waveform-run-v1'), build_hash: hash, decoded_records: count, raw_records: count, + counts: z.object({ records: count, accepted_frames: count, empty_channel_frames: count, + pump_gated_frames: count, preroll_outputs: count, scored_outputs: count }).strict(), + limitations, config_hashes: z.object({ left: hash, right: hash }).strict(), artifact_hashes: z.array(hash).min(1).max(64), +}).strict() + +export const waveformEvaluationSchema = z.object({ hash: z.string().regex(/^[a-f0-9]{64}$/), outputs: z.array(z.object({ + cohortId: z.string().min(1).max(200), configHash: z.string().regex(/^[a-f0-9]{64}$/), + readings: z.array(waveformReadingSchema).max(20000), run: waveformRunSchema, +}).strict()).max(180) }).strict().refine(value => value.outputs.reduce((sum, output) => sum + output.readings.length, 0) <= 20000, 'Waveform reading budget exceeded') diff --git a/src/biometrics/backtest/waveform-publication.ts b/src/biometrics/backtest/waveform-publication.ts new file mode 100644 index 000000000..82b4a7ebe --- /dev/null +++ b/src/biometrics/backtest/waveform-publication.ts @@ -0,0 +1,62 @@ +import { sql } from 'drizzle-orm' +import { dirname, join } from 'node:path' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import type { z } from 'zod' +import type * as schema from '@/src/db/biometrics-schema' +import { ReferenceInputError } from '../references/time' +import { createWaveformDatasetSchema, prepareSelectedWaveformDataset, selectedWaveformRetry } from './waveform-selection' +import { podRawInspectionLocation, type RawInspectionLocation } from './raw-inspection-child' +import { runRawPublicationChild } from './raw-publication-child' + +/** Copy outside the live DB transaction. Python atomically publishes all refs; + * a lost response or cancellation after commit is recovered by request identity. */ +export async function publishSelectedWaveformDataset(db: BetterSQLite3Database, raw: z.input, options: { + signal: AbortSignal + admit: () => Promise + rawLocation?: RawInspectionLocation +}) { + const input = createWaveformDatasetSchema.parse(raw) + const existing = selectedWaveformRetry(db, input) + if (existing) return { status: 'published' as const, dataset: existing } + const deferred = { status: 'deferred' as const } + const admitted = () => Promise.resolve().then(options.admit).catch(() => false) + if (options.signal.aborted || !await admitted() || options.signal.aborted) return deferred + const databasePath = db.all<{ name: string, file: string }>(sql`PRAGMA database_list`).find(row => row.name === 'main')?.file + if (!databasePath) throw new ReferenceInputError('WORKER_UNAVAILABLE', 'Waveform publication requires a persistent biometrics database') + const work = prepareSelectedWaveformDataset(db, input) + if (!await admitted() || options.signal.aborted) return deferred + const controller = new AbortController() + const abort = () => controller.abort() + options.signal.addEventListener('abort', abort, { once: true }) + if (options.signal.aborted) abort() + let checking: Promise | undefined, denied = false + const timer = setInterval(() => { + if (checking) return + checking = admitted().then((allowed) => { + if (!allowed) { + denied = true + controller.abort() + } + }).finally(() => { checking = undefined }) + }, 5000) + try { + const result = await runRawPublicationChild({ ...(options.rawLocation ?? podRawInspectionLocation), databasePath, + retainedDirectory: join(dirname(databasePath), 'biometric-raw') }, work, { signal: controller.signal }) + const published = selectedWaveformRetry(db, input) + if (!published || published.id !== result.id || published.contentHash !== result.contentHash) throw new Error('Published waveform identity mismatch') + return { status: 'published' as const, dataset: published } + } + catch (error) { + // A fully committed dataset remains published even if IPC was lost or a + // stop raced with commit. Returning it never resurrects a released row. + const published = selectedWaveformRetry(db, input) + if (published) return { status: 'published' as const, dataset: published } + if (denied) return deferred + throw error + } + finally { + clearInterval(timer) + await checking + options.signal.removeEventListener('abort', abort) + } +} diff --git a/src/biometrics/backtest/waveform-resources.ts b/src/biometrics/backtest/waveform-resources.ts new file mode 100644 index 000000000..d227505d5 --- /dev/null +++ b/src/biometrics/backtest/waveform-resources.ts @@ -0,0 +1,28 @@ +import { z } from 'zod' +import { waveformProcessResourcesSchema } from './waveform-output' + +export const waveformResourceSummarySchema = z.object({ + version: z.literal('waveform-resource-summary-v1'), + scope: z.literal('sequential_python_processes_excluding_coordinator_and_scorer'), + expectedProcesses: z.number().int().min(0).max(180), measuredProcesses: z.number().int().min(0).max(180), + totalWallMs: z.number().finite().nonnegative().nullable(), totalCpuMs: z.number().finite().nonnegative().nullable(), + largestProcessPeakRssBytes: z.number().int().positive().max(Number.MAX_SAFE_INTEGER).nullable(), + ingestionHealthMeasured: z.literal(false), +}).strict() + +/** Sequential workers: sum durations and retain the largest process high water + * mark. This deliberately does not claim simultaneous job RSS or live health. */ +export function summarizeWaveformResources(outputs: Array<{ run: { resources?: z.infer } }>) { + const measured = outputs.flatMap(output => output.run.resources ? [waveformProcessResourcesSchema.parse(output.run.resources)] : []) + const complete = outputs.length > 0 && measured.length === outputs.length + const memoryComplete = complete && measured.every(row => row.peakRssBytes !== null) + return { + version: 'waveform-resource-summary-v1' as const, + scope: 'sequential_python_processes_excluding_coordinator_and_scorer' as const, + expectedProcesses: outputs.length, measuredProcesses: measured.length, + totalWallMs: complete ? measured.reduce((sum, row) => sum + row.wallMs, 0) : null, + totalCpuMs: complete ? measured.reduce((sum, row) => sum + row.cpuMs, 0) : null, + largestProcessPeakRssBytes: memoryComplete ? Math.max(...measured.map(row => row.peakRssBytes ?? 0)) : null, + ingestionHealthMeasured: false as const, + } +} diff --git a/src/biometrics/backtest/waveform-selection.ts b/src/biometrics/backtest/waveform-selection.ts new file mode 100644 index 000000000..133d20d7b --- /dev/null +++ b/src/biometrics/backtest/waveform-selection.ts @@ -0,0 +1,131 @@ +import { z } from 'zod' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import { eq } from 'drizzle-orm' +import { configHash } from '../algorithms/canonical' +import * as schema from '@/src/db/biometrics-schema' +import { ReferenceInputError } from '../references/time' +import { DatasetStore } from './datasets' +import { datasetManifestSchema, type DatasetSnapshot } from './contracts' +import { RawSourceCatalog } from './raw-catalog' +import { prepareWaveformDataset } from './waveform-dataset' + +const hash = z.string().regex(/^[a-f0-9]{64}$/) +export const createWaveformDatasetSchema = z.object({ + parentDatasetId: z.string().uuid(), expectedContentHash: hash, + cohorts: z.array(z.object({ cohortId: z.string().min(1).max(200), + sourceIds: z.array(hash).min(1).max(64), + }).strict()).min(1).max(60), + idempotencyKey: z.string().min(1).max(200), +}).strict() + +/** Retry identity depends on the requested immutable IDs, not mutable catalog availability. */ +export function waveformSelectionRequestHash(raw: z.input) { + const input = createWaveformDatasetSchema.parse(raw) + const cohorts = input.cohorts.map(item => ({ ...item, sourceIds: [...item.sourceIds].sort() })) + .sort((a, b) => a.cohortId < b.cohortId ? -1 : a.cohortId > b.cohortId ? 1 : 0) + return configHash({ version: 'selected-waveform-dataset-request-v1', ...input, cohorts }) +} + +export function selectedWaveformRetry(db: BetterSQLite3Database, raw: z.input) { + const input = createWaveformDatasetSchema.parse(raw) + const row = db.select().from(schema.biometricDatasets).where(eq(schema.biometricDatasets.idempotencyKey, input.idempotencyKey)).get() + if (!row) return null + if (row.requestHash !== waveformSelectionRequestHash(input)) throw new ReferenceInputError('REVISION_CONFLICT', 'Dataset idempotency key was already used for a different selection') + return new DatasetStore(db).descriptor(row.id) +} + +/** Only intervals actually recorded in the frozen parent are evidence. Session + * boundaries are not evidence of continuous occupancy, nor of sleeper mode. */ +export function recordedWaveformContexts(snapshot: DatasetSnapshot, startMs: number, endMs: number) { + const events = new Map() + let count = 0 + for (const session of snapshot.sessions) { + for (const [raw, state] of [[session.presentIntervals, 0], [session.notPresentIntervals, 1]] as const) { + if (!Array.isArray(raw)) continue + for (const item of raw) { + if (++count > 20000) throw new ReferenceInputError('RESOURCE_LIMIT', 'Recorded occupancy interval budget exceeded') + // The sleep detector stores epoch seconds (including fractional seconds). + if (!Array.isArray(item) || item.length !== 2 || item.some(value => typeof value !== 'number' || !Number.isFinite(value) || value < 0)) continue + const first = Math.round(item[0] * 1000), last = Math.round(item[1] * 1000) + if (!Number.isSafeInteger(first) || !Number.isSafeInteger(last) || first >= last) continue + const start = Math.max(first, startMs, session.startMs), end = Math.min(last, endMs, session.endMs) + if (start >= end) continue + for (const [time, delta] of [[start, 1], [end, -1]]) { + const event = events.get(time) ?? [0, 0] + event[state] += delta + events.set(time, event) + } + } + } + } + const times = [...events.keys()].sort((a, b) => a - b) + let present = 0, absent = 0 + const contexts: Array<{ + start_ms: number + end_ms: number + active_sides: null + left_occupied: boolean | null + right_occupied: boolean | null + pump_context_known: false + }> = [] + for (let index = 0; index < times.length - 1; index++) { + const [yes, no] = events.get(times[index]) ?? [0, 0] + present += yes + absent += no + // Conflicting evidence and unrecorded gaps both remain unknown. + if ((present > 0) === (absent > 0)) continue + const occupied = present > 0 + const last = contexts.at(-1) + if (last && last.end_ms === times[index] && last[`${snapshot.side}_occupied`] === occupied) last.end_ms = times[index + 1] + else contexts.push({ start_ms: times[index], end_ms: times[index + 1], active_sides: null, + left_occupied: snapshot.side === 'left' ? occupied : null, + right_occupied: snapshot.side === 'right' ? occupied : null, pump_context_known: false }) + } + if (contexts.length > 20000) throw new ReferenceInputError('RESOURCE_LIMIT', 'Recorded occupancy context budget exceeded') + return contexts +} + +/** Resolve IDs to verified catalog evidence and frozen historical context. No + * archive reads or live configuration queries occur while the DB snapshot is held. */ +export function prepareSelectedWaveformDataset(db: BetterSQLite3Database, raw: z.input, now = Date.now()) { + const input = createWaveformDatasetSchema.parse(raw) + if (new Set(input.cohorts.map(item => item.cohortId)).size !== input.cohorts.length) throw new ReferenceInputError('DUPLICATE_SELECTION', 'Select each whole-night cohort once') + return db.transaction(() => { + const parent = new DatasetStore(db).read(input.parentDatasetId) + if (parent.contentHash !== input.expectedContentHash) throw new ReferenceInputError('DATASET_CHANGED', 'Select the expected immutable parent dataset') + const manifest = datasetManifestSchema.parse(parent.manifest) + if (input.cohorts.length !== manifest.splitProtocol.assignments.length) throw new ReferenceInputError('INSUFFICIENT_RAW_DATA', 'Select RAW inputs for every whole-night cohort') + const catalog = new RawSourceCatalog(db) + const selections = new Map['inspection']['artifact'] }>() + const inputs = manifest.splitProtocol.assignments.map((cohort) => { + const selection = input.cohorts.find(item => item.cohortId === cohort.cohortId) + if (!selection) throw new ReferenceInputError('INSUFFICIENT_RAW_DATA', 'Missing whole-night RAW selection') + if (new Set(selection.sourceIds).size !== selection.sourceIds.length) throw new ReferenceInputError('DUPLICATE_SELECTION', 'Select each RAW source once per cohort') + const artifacts = selection.sourceIds.map((id) => { + const source = catalog.read(id) + const { inventory, artifact, source_name } = source.inspection + const cached = source.unavailableAtMs === null + ? undefined + : db.select().from(schema.biometricRawArtifacts) + .where(eq(schema.biometricRawArtifacts.contentHash, artifact.sha256)).get() + const mayReuse = cached?.name === artifact.name && cached.compressedBytes === artifact.compressed_bytes + && cached.expandedBytes === artifact.expanded_bytes + // Metadata only permits attempting reuse; Python still verifies the complete retained file. + if ((source.unavailableAtMs !== null && !mayReuse) || !inventory.ordered_waveform_available + || inventory.first_event_ms === null || inventory.last_event_ms === null) { + throw new ReferenceInputError('INSUFFICIENT_RAW_DATA', 'Selected RAW source is unavailable or cannot provide ordered waveforms') + } + const previous = selections.get(artifact.sha256) + if (!previous || source_name < previous.source_name) selections.set(artifact.sha256, { source_name, artifact }) + return { artifact, first_event_ms: inventory.first_event_ms, last_event_ms: inventory.last_event_ms } + }).sort((a, b) => a.first_event_ms - b.first_event_ms || a.artifact.sha256.localeCompare(b.artifact.sha256)) + const start = Math.max(0, cohort.startMs - 300000) + return { cohortId: cohort.cohortId, input: { version: 'retained-waveform-input-v1' as const, + retained_start_ms: start, score_start_ms: cohort.startMs, score_end_ms: cohort.endMs, + artifacts, contexts: recordedWaveformContexts(parent.snapshot, start, cohort.endMs) } } + }) + const prepared = prepareWaveformDataset({ ...parent, manifest }, inputs, input.idempotencyKey, now) + prepared.requestHash = waveformSelectionRequestHash(input) + return { prepared, selections: [...selections.values()].sort((a, b) => a.artifact.sha256.localeCompare(b.artifact.sha256)) } + }) +} diff --git a/src/biometrics/backtest/worker-entry.ts b/src/biometrics/backtest/worker-entry.ts new file mode 100644 index 000000000..930a4979f --- /dev/null +++ b/src/biometrics/backtest/worker-entry.ts @@ -0,0 +1,30 @@ +import { evaluateBacktest } from './evaluator' + +// Shipped evaluator only. This process is not an untrusted-code sandbox. +const MAX_INPUT = 10 * 1024 * 1024 +let received = 0 +const chunks: Buffer[] = [] +process.stdin.on('data', (chunk: Buffer) => { + received += chunk.length + if (received > MAX_INPUT) { + process.stderr.write('Replay input exceeds the worker byte limit\n') + process.exit(2) + } + chunks.push(chunk) +}) +process.stdin.on('end', () => { + try { + const input: unknown = JSON.parse(Buffer.concat(chunks).toString('utf8')) + // Parsing at the boundary is performed again by the shared contract. + const started = performance.now() + const report = evaluateBacktest(input as Parameters[0], (completedNights, totalNights) => { + process.stdout.write(`${JSON.stringify({ type: 'progress', completedNights, totalNights })}\n`) + }) + process.stdout.write(`${JSON.stringify({ type: 'result', report: { ...report, operational: { runtimeMs: performance.now() - started, peakRssKiB: process.resourceUsage().maxRSS, heapLimitMiB: 128 } } })}\n`) + } + catch { + // Input exports and annotations are health data, not diagnostic log text. + process.stderr.write('Replay failed input validation or evaluation\n') + process.exitCode = 1 + } +}) diff --git a/src/biometrics/packages/README.md b/src/biometrics/packages/README.md new file mode 100644 index 000000000..e387e9148 --- /dev/null +++ b/src/biometrics/packages/README.md @@ -0,0 +1,71 @@ +# Algorithm package proposals + +`manifest.ts` validates inert proposals for the algorithm-development workflow. +It does not import, execute, install, or approve their code. No API currently +accepts these proposals. The [off-pod experiment workflow](../../../scripts/algorithm-experiments/README.md) +provides Docker execution, deterministic/noise checks, managed whole-night +comparisons, resource assessments and combined external qualification evidence. +That evidence cannot install code. Promotion requires representative-fixture +review, matching target-pod and prospective evidence, and a reviewed release +into the shipped registry. + +The versioned manifest declares engine identity, frame schema identifiers, +metric definitions and units, supported hardware/sample rates, causal or +retrospective behavior, deterministic initialization and reset, JSON state +serialization, resource requests, an entrypoint, and a dependency lock. Every +file is listed with its exact byte length and SHA-256 digest. Package identity +is SHA-256 over the complete manifest using `finite-binary64-json-v1` canonical +JSON. Artifact array order is part of that identity. + +`verifyAlgorithmPackage` takes a map of bytes, avoiding filesystem resolution +or archive extraction. It rejects unexpected files, missing files, mismatched +sizes/hashes, traversal paths, duplicate identities, and oversized proposals. +The 32 MiB intake ceiling is a storage bound, not a qualified runtime budget. +A future transport must enforce size limits before buffering. Intake verifies +only the dependency-lock artifact identity. The external +executor additionally requires an exact Python environment lock and independently +probes its reviewed runtime image; see the executor README for that format. + +Execution must use an external isolation boundary without pod credentials, +network access by default, writable input, or access to live sinks. Resource +requests must be checked against server-owned limits and enforced outside the +candidate process. Deterministic replay, artifact/noise fixtures, resource +measurements, held-out comparisons, and compatibility checks remain required. +An experiment result never grants installation rights: promotion goes through +code review and a release into the shipped registry. + +## Streaming frame contract + +`protocol.ts` defines the host-side exchange. A run names the exact package and +input hashes, hardware, sample rate, side, seed, and finite JSON configuration. +Its canonical hash binds all those choices. Create a fresh validator and reset +candidate state for each independent run; never carry state between nights or +qualification repetitions unless a future protocol explicitly declares it. + +Frames carry increasing sequence numbers from zero, explicit sample-start +timestamps, one or both channels of at most 4096 finite samples, and explicit +nullable occupancy/pump context. Missing context is unknown, not false. +Channels must agree in length. Sample duration determines the next permissible +frame time; gaps are permitted, overlaps are rejected. The executor must call +`validateFrame` before exposing each frame to the candidate. + +A response echoes the run hash, canonical input-frame hash, and sequence. It +contains bounded numeric/null metric readings and JSON state. The host checks +metric definitions, side, ranges, strictly increasing timestamps per metric, +observed time bounds, declared retrospective lookahead, state bytes, and the +cumulative canonical response-byte budget. Validation failure does not advance +the run. No response can authorize database or hardware writes. + +The external executor must bound transport bytes before JSON decoding (including +whitespace), enforce process resources independently, and expose only the current +frame and previously returned state. The validator cannot prove that code obeys +causality, initialization, reset, or determinism; those need the executor and +repeated qualification runs. This protocol is not itself an isolation boundary. + +A metric can optionally declare a structured `reference` contract for external +comparison: `metric` is one of `heart_rate`, `breathing_rate`, +`window_ibi_variability_v1`, `hrv_rmssd`, or `hrv_sdnn`; `windowMs` is required and +may be null only for non-HRV metrics. The declaration requires canonical units +and participates in package identity. Metrics without this declaration remain +valid experimental outputs but cannot enter reference comparison based on their +name or prose definition alone. diff --git a/src/biometrics/packages/manifest.ts b/src/biometrics/packages/manifest.ts new file mode 100644 index 000000000..50c173b8e --- /dev/null +++ b/src/biometrics/packages/manifest.ts @@ -0,0 +1,75 @@ +import { createHash } from 'node:crypto' +import { z } from 'zod' +import { configHash } from '../algorithms/canonical' + +const hash = z.string().regex(/^[a-f0-9]{64}$/) +const identifier = z.string().regex(/^[a-z][a-z0-9_.-]{0,99}$/) +const positiveInteger = z.number().int().positive().max(Number.MAX_SAFE_INTEGER) +const relativePath = z.string().min(1).max(240).refine(path => + path.split('/').every(part => /^[A-Za-z0-9_][A-Za-z0-9_.-]*$/.test(part)), 'Expected a relative package file path') +const artifactSchema = z.object({ path: relativePath, sha256: hash, bytes: positiveInteger }).strict() +const referenceContractSchema = z.object({ + metric: z.enum(['heart_rate', 'breathing_rate', 'window_ibi_variability_v1', 'hrv_rmssd', 'hrv_sdnn']), + windowMs: z.number().int().positive().max(86400000).nullable(), +}).strict().refine(value => !['window_ibi_variability_v1', 'hrv_rmssd', 'hrv_sdnn'].includes(value.metric) || value.windowMs !== null, 'HRV comparison requires an explicit window') +const metricSchema = z.object({ + id: identifier, definition: z.string().min(1).max(2000), unit: z.string().min(1).max(80), + minimum: z.number().finite(), maximum: z.number().finite(), + reference: referenceContractSchema.optional(), +}).strict().refine(metric => metric.minimum < metric.maximum, 'Metric bounds must be ordered') + .refine(metric => !metric.reference || metric.unit === (metric.reference.metric === 'heart_rate' ? 'bpm' : metric.reference.metric === 'breathing_rate' ? 'breaths/min' : 'ms'), 'Reference metric requires canonical units') + +/** Inert proposal metadata. Passing this schema grants no execution or installation rights. */ +export const algorithmPackageManifestSchema = z.object({ + schema: z.literal('biometric-algorithm-package-v1'), + canonicalization: z.literal('finite-binary64-json-v1'), + engine: z.object({ id: identifier, version: z.string().regex(/^\d+\.\d+\.\d+$/) }).strict(), + runtime: z.literal('python-frame-v1'), entrypoint: relativePath, + inputSchema: z.literal('biometric-signal-frame-v1'), outputSchema: z.literal('biometric-metric-frame-v1'), + metrics: z.array(metricSchema).min(1).max(16), + hardware: z.array(z.object({ id: identifier, sampleRatesHz: z.array(positiveInteger).min(1).max(16) }).strict()).min(1).max(16), + temporal: z.discriminatedUnion('mode', [ + z.object({ mode: z.literal('causal'), lookaheadMs: z.literal(0) }).strict(), + z.object({ mode: z.literal('retrospective'), lookaheadMs: positiveInteger }).strict(), + ]), + state: z.object({ + schemaId: identifier, initialization: z.literal('explicit_seed_and_config'), + reset: z.literal('reset_before_each_independent_run'), serialization: z.literal('json'), maximumBytes: positiveInteger, + }).strict(), + resources: z.object({ + cpuMs: positiveInteger, wallMs: positiveInteger, memoryBytes: positiveInteger, + processes: positiveInteger, outputBytes: positiveInteger, + }).strict(), + dependencyLock: relativePath, + artifacts: z.array(artifactSchema).min(2).max(128), +}).strict().superRefine((value, context) => { + const unique = (values: (string | number)[], label: string) => { + if (new Set(values).size !== values.length) context.addIssue({ code: 'custom', message: `Duplicate ${label}` }) + } + unique(value.metrics.map(metric => metric.id), 'metric') + unique(value.hardware.map(hardware => hardware.id), 'hardware') + for (const hardware of value.hardware) unique(hardware.sampleRatesHz, 'sample rate') + unique(value.artifacts.map(artifact => artifact.path), 'artifact path') + for (const path of [value.entrypoint, value.dependencyLock]) { + if (!value.artifacts.some(artifact => artifact.path === path)) context.addIssue({ code: 'custom', message: `Unhashed package file: ${path}` }) + } + if (value.entrypoint === value.dependencyLock) context.addIssue({ code: 'custom', message: 'Entrypoint and dependency lock must be separate files' }) +}) +export type AlgorithmPackageManifest = z.infer + +/** Bounded byte inputs, deliberately with no filesystem paths, imports, or subprocesses. + * Callers must bound transport size before buffering. These limits are proposal intake + * limits, not evidence that an algorithm fits the pod's runtime budget. */ +export const maximumPackageBytes = 32 * 1024 * 1024 +export function verifyAlgorithmPackage(manifestInput: unknown, files: ReadonlyMap) { + const manifest = algorithmPackageManifestSchema.parse(manifestInput) + if (files.size !== manifest.artifacts.length) throw new Error('Package file set differs from manifest') + const totalBytes = manifest.artifacts.reduce((sum, artifact) => sum + artifact.bytes, 0) + if (!Number.isSafeInteger(totalBytes) || totalBytes > maximumPackageBytes) throw new Error('Package exceeds intake byte limit') + for (const artifact of manifest.artifacts) { + const bytes = files.get(artifact.path) + if (!bytes || bytes.byteLength !== artifact.bytes) throw new Error(`Package file size mismatch: ${artifact.path}`) + if (createHash('sha256').update(bytes).digest('hex') !== artifact.sha256) throw new Error(`Package file hash mismatch: ${artifact.path}`) + } + return { manifest, packageHash: configHash(manifest), totalBytes, status: 'inert_proposal' as const } +} diff --git a/src/biometrics/packages/protocol.ts b/src/biometrics/packages/protocol.ts new file mode 100644 index 000000000..13ac10e6b --- /dev/null +++ b/src/biometrics/packages/protocol.ts @@ -0,0 +1,92 @@ +import { z } from 'zod' +import { canonicalJson, configHash, dataHash } from '../algorithms/canonical' +import { algorithmPackageManifestSchema } from './manifest' + +const hash = z.string().regex(/^[a-f0-9]{64}$/) +const time = z.number().finite().nonnegative().max(Number.MAX_SAFE_INTEGER) +const integer = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) +const side = z.enum(['left', 'right']) +const samples = z.array(z.number().finite()).min(1).max(4096) +export const packageRunRequestSchema = z.object({ + schema: z.literal('biometric-package-run-v1'), packageHash: hash, inputHash: hash, + hardwareId: z.string().min(1).max(100), sampleRateHz: z.number().int().positive(), side, + seed: integer, config: z.record(z.string(), z.unknown()), +}).strict() +export const signalFrameSchema = z.object({ + schema: z.literal('biometric-signal-frame-v1'), sequence: integer, timestampMs: time, + samples: z.object({ left: samples.nullable(), right: samples.nullable() }).strict(), + context: z.object({ occupied: z.boolean().nullable(), pumpActive: z.boolean().nullable() }).strict(), +}).strict().refine(frame => frame.samples.left !== null || frame.samples.right !== null, 'Frame must contain samples') + .refine(frame => !frame.samples.left || !frame.samples.right || frame.samples.left.length === frame.samples.right.length, 'Channel lengths differ') +export const metricFrameSchema = z.object({ + schema: z.literal('biometric-metric-frame-v1'), runHash: hash, frameHash: hash, sequence: integer, + readings: z.array(z.object({ + timestampMs: time, side, metric: z.string().min(1).max(100), value: z.number().finite().nullable(), + quality: z.number().min(0).max(1), flags: z.array(z.string().min(1).max(100)).max(32), + }).strict()).max(256), + state: z.unknown(), +}).strict() + +/** Host-side validation of a streaming exchange. This class never runs a package. + * One validator belongs to one independent run and starts at sequence zero. + * The executor must send only the current frame, after resetting package state. */ +export class PackageFrameValidator { + readonly manifest + readonly request + readonly runHash: string + private nextSequence = 0 + private outputBytes = 0 + private nextInputTime = 0 + private firstInputTime: number | null = null + private lastReadingTimes = new Map() + + constructor(manifestInput: unknown, requestInput: unknown) { + this.manifest = algorithmPackageManifestSchema.parse(manifestInput) + this.request = packageRunRequestSchema.parse(requestInput) + if (this.request.packageHash !== configHash(this.manifest)) throw new Error('Package identity mismatch') + const hardware = this.manifest.hardware.find(item => item.id === this.request.hardwareId) + if (!hardware?.sampleRatesHz.includes(this.request.sampleRateHz)) throw new Error('Unsupported hardware or sample rate') + // Canonical hashing also rejects non-JSON config values and bounds complexity. + this.runHash = configHash(this.request) + } + + validateFrame(frameInput: unknown) { + const frame = signalFrameSchema.parse(frameInput) + if (frame.sequence !== this.nextSequence || frame.timestampMs < this.nextInputTime) throw new Error('Out-of-order or overlapping frame') + const channel = frame.samples[this.request.side] + if (!channel) throw new Error('Requested side has no samples') + const endMs = frame.timestampMs + channel.length * 1000 / this.request.sampleRateHz + if (!Number.isFinite(endMs) || endMs > Number.MAX_SAFE_INTEGER) throw new Error('Frame end exceeds timestamp range') + return { frame, endMs } + } + + validateStep(frameInput: unknown, resultInput: unknown) { + const { frame, endMs } = this.validateFrame(frameInput) + const result = metricFrameSchema.parse(resultInput) + if (result.runHash !== this.runHash || result.frameHash !== dataHash(frame) || result.sequence !== frame.sequence) throw new Error('Output identity mismatch') + // A transport must additionally cap stdout before buffering/parsing it. + const outputBytes = this.outputBytes + Buffer.byteLength(canonicalJson(result)) + if (outputBytes > this.manifest.resources.outputBytes) throw new Error('Output byte limit exceeded') + if (Buffer.byteLength(canonicalJson(result.state)) > this.manifest.state.maximumBytes) throw new Error('State byte limit exceeded') + const firstInputTime = this.firstInputTime ?? frame.timestampMs + const lastTimes = new Map(this.lastReadingTimes) + for (const reading of result.readings) { + const metric = this.manifest.metrics.find(item => item.id === reading.metric) + if (!metric || reading.side !== this.request.side) throw new Error('Undeclared metric or side') + if (reading.value !== null && (reading.value < metric.minimum || reading.value > metric.maximum)) throw new Error('Metric outside declared range') + // A retrospective reading may lag the latest available sample by its + // declared lookahead. No output may refer to unavailable future samples. + const latestSampleMs = endMs - 1000 / this.request.sampleRateHz + if (reading.timestampMs < firstInputTime || reading.timestampMs + this.manifest.temporal.lookaheadMs > latestSampleMs) throw new Error('Reading outside observed time range') + if (reading.timestampMs <= (lastTimes.get(reading.metric) ?? -1)) throw new Error('Duplicate or out-of-order reading') + lastTimes.set(reading.metric, reading.timestampMs) + } + // A rejected response leaves the sequence and watermarks unchanged. + this.outputBytes = outputBytes + this.nextSequence++ + this.nextInputTime = endMs + this.firstInputTime = firstInputTime + this.lastReadingTimes = lastTimes + return { frame, result, endMs } + } +} diff --git a/src/biometrics/packages/tests/fixture.ts b/src/biometrics/packages/tests/fixture.ts new file mode 100644 index 000000000..41affcbee --- /dev/null +++ b/src/biometrics/packages/tests/fixture.ts @@ -0,0 +1,17 @@ +import { createHash } from 'node:crypto' + +export function packageFixture() { + const files = new Map([['engine.py', Buffer.from('raise RuntimeError("must never execute during intake")')], ['uv.lock', Buffer.from('version = 1')]]) + const manifest = { + schema: 'biometric-algorithm-package-v1', canonicalization: 'finite-binary64-json-v1', engine: { id: 'test.hr', version: '1.0.0' }, + runtime: 'python-frame-v1', entrypoint: 'engine.py', dependencyLock: 'uv.lock', + inputSchema: 'biometric-signal-frame-v1', outputSchema: 'biometric-metric-frame-v1', + metrics: [{ id: 'heart_rate', definition: 'Heart beats per minute', unit: 'bpm', minimum: 20, maximum: 240 }], + hardware: [{ id: 'pod4', sampleRatesHz: [500] }], temporal: { mode: 'causal', lookaheadMs: 0 }, + state: { schemaId: 'test.hr.state.v1', initialization: 'explicit_seed_and_config', reset: 'reset_before_each_independent_run', serialization: 'json', maximumBytes: 10000 }, + resources: { cpuMs: 1000, wallMs: 2000, memoryBytes: 100000000, processes: 1, outputBytes: 10000 }, + artifacts: [...files].map(([path, bytes]) => ({ path, bytes: bytes.byteLength, sha256: createHash('sha256').update(bytes).digest('hex') })), + } + return { files, manifest } +} + diff --git a/src/biometrics/packages/tests/manifest.test.ts b/src/biometrics/packages/tests/manifest.test.ts new file mode 100644 index 000000000..221eb9104 --- /dev/null +++ b/src/biometrics/packages/tests/manifest.test.ts @@ -0,0 +1,64 @@ +import { packageFixture as fixture } from './fixture' +import { describe, expect, it } from 'vitest' +import { algorithmPackageManifestSchema, maximumPackageBytes, verifyAlgorithmPackage } from '../manifest' + +describe('inert algorithm package intake', () => { + it('verifies executable bytes without executing them and binds all declarations to the package identity', () => { + const { files, manifest } = fixture() + const original = verifyAlgorithmPackage(manifest, files) + expect(original.status).toBe('inert_proposal') + expect(verifyAlgorithmPackage(JSON.parse(JSON.stringify(manifest)), files).packageHash).toBe(original.packageHash) + manifest.resources.cpuMs++ + expect(verifyAlgorithmPackage(manifest, files).packageHash).not.toBe(original.packageHash) + }) + it('rejects substituted code and locks even when the file size is unchanged', () => { + for (const path of ['engine.py', 'uv.lock']) { + const { files, manifest } = fixture() + const original = files.get(path) + if (!original) throw new Error('Missing fixture file') + files.set(path, Buffer.alloc(original.length, 65)) + expect(() => verifyAlgorithmPackage(manifest, files)).toThrow('hash mismatch') + } + }) + it('rejects missing, extra, truncated and oversized files', () => { + const { files, manifest } = fixture() + files.set('extra.py', Buffer.from('pass')) + expect(() => verifyAlgorithmPackage(manifest, files)).toThrow('file set') + files.delete('extra.py') + files.set('engine.py', Buffer.from('pass')) + expect(() => verifyAlgorithmPackage(manifest, files)).toThrow('size mismatch') + files.delete('engine.py') + expect(() => verifyAlgorithmPackage(manifest, files)).toThrow('file set') + manifest.artifacts[0].bytes = maximumPackageBytes + expect(() => verifyAlgorithmPackage(manifest, fixture().files)).toThrow('byte limit') + }) + it.each(['../engine.py', '/engine.py', 'a/../../engine.py', 'a\\engine.py', 'a//engine.py', './engine.py', 'a/engine.py\u0000'])('rejects unsafe artifact path %s', (path) => { + const { manifest } = fixture() + manifest.artifacts[0].path = path + manifest.entrypoint = path + expect(algorithmPackageManifestSchema.safeParse(manifest).success).toBe(false) + }) + it('requires canonical units and explicit HRV windows for comparable metrics', () => { + const { manifest } = fixture() + const metric = manifest.metrics[0] + const parse = (unit: string, reference: unknown) => algorithmPackageManifestSchema.safeParse({ ...manifest, metrics: [{ ...metric, unit, reference }] }).success + expect(parse('bpm', { metric: 'heart_rate', windowMs: null })).toBe(true) + expect(parse('ms', { metric: 'hrv_rmssd', windowMs: 300000 })).toBe(true) + expect(parse('bpm', { metric: 'hrv_rmssd', windowMs: 300000 })).toBe(false) + expect(parse('ms', { metric: 'hrv_rmssd', windowMs: null })).toBe(false) + expect(parse('ms', { metric: 'hrv_unknown', windowMs: 300000 })).toBe(false) + }) + it('rejects ambiguous identity, unhashed entrypoints, missing locks and unknown privileges', () => { + const { manifest } = fixture() + for (const altered of [ + { ...manifest, metrics: [...manifest.metrics, manifest.metrics[0]] }, + { ...manifest, artifacts: [...manifest.artifacts, manifest.artifacts[0]] }, + { ...manifest, hardware: [...manifest.hardware, manifest.hardware[0]] }, + { ...manifest, hardware: [{ id: 'pod4', sampleRatesHz: [500, 500] }] }, + { ...manifest, entrypoint: 'other.py' }, { ...manifest, dependencyLock: 'absent.lock' }, + { ...manifest, dependencyLock: manifest.entrypoint }, { ...manifest, network: true }, + { ...manifest, resources: { ...manifest.resources, processes: 0 } }, + { ...manifest, temporal: { mode: 'causal', lookaheadMs: 1000 } }, + ]) expect(algorithmPackageManifestSchema.safeParse(altered).success).toBe(false) + }) +}) diff --git a/src/biometrics/packages/tests/protocol.test.ts b/src/biometrics/packages/tests/protocol.test.ts new file mode 100644 index 000000000..2fdb5a88e --- /dev/null +++ b/src/biometrics/packages/tests/protocol.test.ts @@ -0,0 +1,82 @@ +import { describe, expect, it } from 'vitest' +import { canonicalJson, configHash, dataHash } from '../../algorithms/canonical' +import { PackageFrameValidator } from '../protocol' +import { packageFixture } from './fixture' + +function fixture() { + const { manifest } = packageFixture() + const request = { schema: 'biometric-package-run-v1', packageHash: configHash(manifest), inputHash: 'a'.repeat(64), hardwareId: 'pod4', sampleRateHz: 500, side: 'left', seed: 123, config: {} } + const frame = { schema: 'biometric-signal-frame-v1', sequence: 0, timestampMs: 1000, samples: { left: [1, 2, 3], right: null }, context: { occupied: null, pumpActive: null } } + const validator = new PackageFrameValidator(manifest, request) + const result = { schema: 'biometric-metric-frame-v1', runHash: validator.runHash, frameHash: dataHash(frame), sequence: 0, + readings: [{ timestampMs: 1004, side: 'left', metric: 'heart_rate', value: 60, quality: 0.9, flags: [] }], state: { count: 3 } } + return { manifest, request, frame, result, validator } +} + +describe('algorithm frame exchange', () => { + it('accepts deterministic identities and ordered frames with explicit unknown context', () => { + const { frame, result, validator } = fixture() + expect(validator.validateStep(frame, result).endMs).toBe(1006) + const next = { ...frame, sequence: 1, timestampMs: 1006 } + expect(validator.validateStep(next, { ...result, frameHash: dataHash(next), sequence: 1, readings: [] }).endMs).toBe(1012) + }) + it('binds the run to package, input, configuration and deterministic seed', () => { + const { manifest, request, validator } = fixture() + for (const changed of [{ ...request, seed: 124 }, { ...request, config: { threshold: 10 } }, { ...request, inputHash: 'b'.repeat(64) }]) { + expect(new PackageFrameValidator(manifest, changed).runHash).not.toBe(validator.runHash) + } + for (const changed of [{ ...request, packageHash: 'b'.repeat(64) }, { ...request, hardwareId: 'other' }, { ...request, sampleRateHz: 100 }]) { + expect(() => new PackageFrameValidator(manifest, changed)).toThrow() + } + }) + it('rejects output substitution and leaves the run retryable', () => { + const { frame, result, validator } = fixture() + for (const changed of [{ ...result, runHash: 'b'.repeat(64) }, { ...result, frameHash: 'b'.repeat(64) }, { ...result, sequence: 1 }]) { + expect(() => validator.validateStep(frame, changed)).toThrow('identity') + } + expect(validator.validateStep(frame, result).endMs).toBe(1006) + }) + it('rejects future, undeclared, out-of-range and duplicated readings', () => { + const { frame, result } = fixture() + const reading = result.readings[0] + for (const readings of [[{ ...reading, timestampMs: 1006 }], [{ ...reading, timestampMs: 999 }], [{ ...reading, side: 'right' }], + [{ ...reading, metric: 'hrv' }], [{ ...reading, value: 300 }], [reading, reading]]) { + expect(() => fixture().validator.validateStep(frame, { ...result, readings })).toThrow() + } + }) + it('rejects overlapping frames, missing channels and inconsistent channel lengths', () => { + const { frame, result, validator } = fixture() + for (const changed of [{ ...frame, sequence: 1 }, { ...frame, samples: { left: null, right: [1] } }, { ...frame, samples: { left: [1], right: [1, 2] } }]) { + expect(() => validator.validateStep(changed, result)).toThrow() + } + validator.validateStep(frame, result) + expect(() => validator.validateStep({ ...frame, sequence: 1, timestampMs: 1005 }, result)).toThrow('overlapping') + const next = { ...frame, sequence: 1, timestampMs: 1006 } + expect(() => validator.validateStep(next, { ...result, sequence: 1, frameHash: dataHash(next) })).toThrow('Duplicate') + }) + it('enforces declared lookahead without accepting future readings', () => { + const { manifest, request, frame, result } = fixture() + const retrospective = { ...manifest, temporal: { mode: 'retrospective', lookaheadMs: 4 } } + const validator = new PackageFrameValidator(retrospective, { ...request, packageHash: configHash(retrospective) }) + const output = { ...result, runHash: validator.runHash } + expect(() => validator.validateStep(frame, output)).toThrow('time range') + expect(validator.validateStep(frame, { ...output, readings: [{ ...output.readings[0], timestampMs: 1000 }] })).toBeDefined() + }) + it('enforces the cumulative output budget across individually small responses', () => { + const { manifest, request, frame, result } = fixture() + manifest.resources.outputBytes = Buffer.byteLength(canonicalJson(result)) + 10 + const validator = new PackageFrameValidator(manifest, { ...request, packageHash: configHash(manifest) }) + validator.validateStep(frame, { ...result, runHash: validator.runHash }) + const next = { ...frame, sequence: 1, timestampMs: 1006 } + expect(() => validator.validateStep(next, { ...result, runHash: validator.runHash, frameHash: dataHash(next), sequence: 1, readings: [] })).toThrow('Output byte limit') + }) + it('bounds state and response sizes and rejects non-JSON state', () => { + for (const kind of ['state', 'output', 'non-json']) { + const { manifest, request, frame, result } = fixture() + if (kind === 'state') manifest.state.maximumBytes = 2 + if (kind === 'output') manifest.resources.outputBytes = 2 + const validator = new PackageFrameValidator(manifest, { ...request, packageHash: configHash(manifest) }) + expect(() => validator.validateStep(frame, { ...result, runHash: validator.runHash, state: kind === 'non-json' ? new Date() : result.state })).toThrow() + } + }) +}) diff --git a/src/biometrics/profiles/activation.ts b/src/biometrics/profiles/activation.ts new file mode 100644 index 000000000..fbb2152cc --- /dev/null +++ b/src/biometrics/profiles/activation.ts @@ -0,0 +1,130 @@ +import { randomUUID } from 'node:crypto' +import { and, eq } from 'drizzle-orm' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import { z } from 'zod' +import type * as schema from '@/src/db/schema' +import type * as bioSchema from '@/src/db/biometrics-schema' +import { biometricActivationEvents as events, biometricConfigOutbox as outbox, biometricProfileBindings as bindings, sideSettings } from '@/src/db/schema' +import { defaultConfig, manifestHash } from '../algorithms/registry' +import { configHash } from '../algorithms/canonical' +import { ReferenceInputError } from '../references/time' +import { stageClassifierSchema } from '@/src/lib/sleep-stage-rules' +import { ProfileStore } from './store' +import { prepareVerifiedEvaluation, type PromotionBuild } from './evaluation-store' + +export const activationIntentSchema = z.object({ + side: z.enum(['left', 'right']), action: z.enum(['activate', 'reset', 'rollback']), + revisionId: z.string().uuid().nullable(), expectedGeneration: z.number().int().nonnegative(), + expectedResolvedHash: z.string().regex(/^[a-f0-9]{64}$/), boundary: z.enum(['immediate', 'next_session']), + requester: z.string().min(1).max(200), idempotencyKey: z.string().min(1).max(200), + allowUnvalidated: z.literal(true), +}).strict() + +export const validatedActivationIntentSchema = activationIntentSchema.omit({ allowUnvalidated: true }).extend({ + action: z.literal('activate'), revisionId: z.string().uuid(), evaluationId: z.string().uuid(), + expectedReportHash: z.string().regex(/^[a-f0-9]{64}$/), +}).strict() + +export const validatedActivationReviewSchema = validatedActivationIntentSchema.omit({ action: true, boundary: true, requester: true, idempotencyKey: true }) + +/** A point-in-time review only; application repeats every verification. */ +export async function reviewValidatedActivation(db: BetterSQLite3Database, bioDb: BetterSQLite3Database, + raw: z.input, resolveBuild: () => Promise) { + const input = validatedActivationReviewSchema.parse(raw) + const verified = prepareVerifiedEvaluation(db, bioDb, input, await resolveBuild()) + verified.recheck() + return { eligible: true as const } +} + +function retryActivation(db: BetterSQLite3Database, idempotencyKey: string, requestHash: string) { + const existing = db.select().from(events).where(eq(events.idempotencyKey, idempotencyKey)).get() + if (existing && existing.requestHash !== requestHash) throw new ReferenceInputError('REVISION_CONFLICT', 'Activation idempotency key changed') + return existing +} + +export function requestManualActivation(db: BetterSQLite3Database, raw: z.input, now = Date.now()) { + const input = activationIntentSchema.parse(raw) + return commitActivationIntent(db, input, configHash(input), now) +} + +/** Configuration is the only database written here. The short biometrics write + * reservation prevents exposure/release/runtime changes across the config + * commit; it is not a transaction committing writes to two databases. */ +export function requestValidatedActivation(db: BetterSQLite3Database, bioDb: BetterSQLite3Database, + raw: z.input, build: PromotionBuild, now = Date.now()) { + const input = validatedActivationIntentSchema.parse(raw) + const requestHash = configHash(input) + const existing = retryActivation(db, input.idempotencyKey, requestHash) + if (existing) return existing + const verified = prepareVerifiedEvaluation(db, bioDb, input, build) + const intent = activationIntentSchema.strip().parse({ ...input, allowUnvalidated: true }) + return bioDb.transaction(() => commitActivationIntent(db, intent, requestHash, now, + { ...verified, evaluationId: input.evaluationId }), { behavior: 'immediate' }) +} + +/** Retry before artifact/build access. Recheck again after the async probe so + * concurrent identical requests still create only one activation event. */ +export async function requestValidatedActivationWithBuild(db: BetterSQLite3Database, bioDb: BetterSQLite3Database, + raw: z.input, resolveBuild: () => Promise, now = Date.now()) { + const input = validatedActivationIntentSchema.parse(raw) + const existing = retryActivation(db, input.idempotencyKey, configHash(input)) + if (existing) return existing + const build = await resolveBuild() + return requestValidatedActivation(db, bioDb, input, build, now) +} + +function commitActivationIntent(db: BetterSQLite3Database, input: z.infer, requestHash: string, now: number, + verified?: ReturnType & { evaluationId: string }) { + return db.transaction((tx) => { + const existing = retryActivation(db, input.idempotencyKey, requestHash) + if (existing) return existing + const profiles = new ProfileStore(db) + const storedBinding = tx.select().from(bindings).where(eq(bindings.side, input.side)).get() + const binding = profiles.binding(input.side) + let invalidLegacy: unknown + if (!storedBinding) { + const legacy = tx.select({ classifier: sideSettings.stageClassifier }).from(sideSettings).where(eq(sideSettings.side, input.side)).get() + if (legacy?.classifier != null) { + // Valid customizations must still migrate losslessly. Malformed legacy + // data cannot become a revision; an explicit reset preserves it as audit + // evidence and leaves the original legacy row untouched. + if (input.action !== 'reset' || stageClassifierSchema.safeParse(legacy.classifier).success) { + throw new ReferenceInputError('LEGACY_MIGRATION_REQUIRED', 'Preserve the existing stage profile before activation') + } + invalidLegacy = legacy.classifier + } + } + if (binding.generation !== input.expectedGeneration) throw new ReferenceInputError('STALE_BINDING', 'Binding changed; inspect the side before applying this revision') + if (input.action === 'reset' && input.revisionId !== null) throw new ReferenceInputError('INVALID_SELECTION', 'Reset must select system defaults') + if (input.action === 'activate' && input.revisionId === null) throw new ReferenceInputError('INVALID_SELECTION', 'Activation must select an exact revision; use reset for defaults') + if (input.action === 'rollback' && (binding.generation === 0 || input.revisionId !== binding.previousRevisionId)) throw new ReferenceInputError('INVALID_SELECTION', 'Rollback must select the previous binding') + const selected = input.revisionId === null + ? { resolved: defaultConfig, hash: configHash(defaultConfig) } + : profiles.validateRevision(input.revisionId, input.side) + if (selected.hash !== input.expectedResolvedHash) throw new ReferenceInputError('REVISION_CONFLICT', 'Resolved configuration changed; inspect its exact hash before applying') + const previous = profiles.effective(input.side) + verified?.recheck() + const generation = binding.generation + 1 + if (!storedBinding) { + tx.insert(bindings).values({ side: input.side, generation, desiredRevisionId: input.revisionId, + previousRevisionId: binding.desiredRevisionId, updatedAtMs: now }).run() + } + else { + const changed = tx.update(bindings).set({ generation, desiredRevisionId: input.revisionId, + previousRevisionId: binding.desiredRevisionId, updatedAtMs: now }) + .where(and(eq(bindings.side, input.side), eq(bindings.generation, input.expectedGeneration))).run() + if (changed.changes !== 1) throw new ReferenceInputError('STALE_BINDING', 'Binding changed during activation') + } + const event = tx.insert(events).values({ id: randomUUID(), side: input.side, generation, action: input.action, + previousRevisionId: binding.desiredRevisionId, nextRevisionId: input.revisionId, previousHash: previous.hash, nextHash: selected.hash, + evaluationId: verified?.evaluationId ?? null, + evidence: verified?.evidence ?? { status: 'manual_unvalidated', manifestHash, resolvedHash: selected.hash, + ...(invalidLegacy === undefined ? {} : { recovery: 'invalid_legacy_reset', preservedLegacyClassifier: invalidLegacy }) }, requester: input.requester, + state: 'pending_delivery', idempotencyKey: input.idempotencyKey, requestHash, createdAtMs: now, updatedAtMs: now }).returning().get() + tx.insert(outbox).values({ id: randomUUID(), side: input.side, generation, + payload: { schemaVersion: 1, side: input.side, generation, revisionId: input.revisionId, manifestHash, + resolved: selected.resolved, resolvedHash: selected.hash, boundary: input.boundary }, + resolvedHash: selected.hash, state: 'pending', nextAttemptAtMs: now, createdAtMs: now }).run() + return event + }, { behavior: 'immediate' }) +} diff --git a/src/biometrics/profiles/contracts.ts b/src/biometrics/profiles/contracts.ts new file mode 100644 index 000000000..cb04673e0 --- /dev/null +++ b/src/biometrics/profiles/contracts.ts @@ -0,0 +1,40 @@ +import { z } from 'zod' +import { stageViewSchema } from './stage-view' +import { runtimeHealthSchema } from './health' +import { stageClassifierSchema } from '@/src/lib/sleep-stage-rules' +import { biometricManifest, validateConfig, type BiometricConfig } from '../algorithms/registry' + +// The canonical manifest remains the source of parameter bounds. The typed +// adapter supplies UI inference while the shared validator checks interactions. +export const biometricConfigSchema = (z.fromJSONSchema(biometricManifest.parameters as unknown as z.core.JSONSchema.JSONSchema) as z.ZodType) + .superRefine((value, context) => { + try { + validateConfig(value) + } + catch { + context.addIssue({ code: 'custom', message: 'Invalid biometric parameter relationship' }) + } + }) +export const overrideSchema = z.object({ path: z.string(), value: z.unknown() }) +export const revisionSchema = z.object({ + id: z.string().uuid(), familyId: z.string().uuid(), revision: z.number().int(), side: z.enum(['left', 'right']), + baseManifestHash: z.string(), baseManifest: z.unknown(), overrides: z.array(overrideSchema), + resolved: z.unknown(), resolvedHash: z.string(), dependencyHashes: z.record(z.string(), z.string()), + parentRevisionId: z.string().uuid().nullable(), origin: z.enum(['manual', 'migration', 'agent', 'system']), + requester: z.string(), idempotencyKey: z.string(), requestHash: z.string(), createdAtMs: z.number(), +}) +export const bindingSchema = z.object({ side: z.enum(['left', 'right']), generation: z.number().int(), + desiredRevisionId: z.string().uuid().nullable(), previousRevisionId: z.string().uuid().nullable(), updatedAtMs: z.number() }) +export const profileOverviewSchema = z.object({ + stageView: stageViewSchema, + desired: z.object({ binding: bindingSchema, resolved: biometricConfigSchema, hash: z.string(), isDefault: z.boolean(), warning: z.string().nullable() }), + legacyStageView: z.object({ classifier: stageClassifierSchema, personalized: z.boolean(), migrationPending: z.boolean() }), + runtimeHealth: runtimeHealthSchema, + runtimeStatus: z.enum(['unreported', 'receipts_available']), + receipts: z.array(z.object({ component: z.string(), generation: z.number(), appliedHash: z.string(), effectiveSampleTimeMs: z.number().nullable(), + state: z.enum(['pending_boundary', 'applied', 'failed']), error: z.string().nullable(), heartbeatAtMs: z.number() })), + sensorCalibration: z.array(z.object({ id: z.number(), sensorType: z.string(), status: z.string(), qualityScore: z.number().nullable(), + createdAtMs: z.number(), expiresAtMs: z.number().nullable(), freshness: z.enum(['expired', 'current', 'unknown']), + sourceWindowStartMs: z.number().nullable(), sourceWindowEndMs: z.number().nullable() })), + activationAvailable: z.boolean(), limitations: z.array(z.string()), +}) diff --git a/src/biometrics/profiles/diff.ts b/src/biometrics/profiles/diff.ts new file mode 100644 index 000000000..2545f99ed --- /dev/null +++ b/src/biometrics/profiles/diff.ts @@ -0,0 +1,26 @@ +import { configHash } from '../algorithms/canonical' +import { defaultConfig, type BiometricConfig, type Override } from '../algorithms/registry' +import type { ProfileStore } from './store' + +export function diffProfiles(store: ProfileStore, side: 'left' | 'right', revisionId: string | null, againstRevisionId: string | null) { + const load = (id: string | null) => { + if (id === null) return { resolved: defaultConfig, hash: configHash(defaultConfig), overrides: [] as Override[] } + const validated = store.validateRevision(id, side) + return { resolved: validated.resolved, hash: validated.hash, overrides: validated.revision.overrides as Override[] } + } + const before = load(againstRevisionId), after = load(revisionId) + const changes: Array<{ path: string, before: unknown, after: unknown }> = [] + const walk = (left: unknown, right: unknown, path: string) => { + if (configHash(left) === configHash(right)) return + if (left !== null && right !== null && typeof left === 'object' && typeof right === 'object' && !Array.isArray(left) && !Array.isArray(right)) { + for (const key of Object.keys(left)) walk((left as Record)[key], (right as Record)[key], path ? `${path}.${key}` : key) + } + else changes.push({ path, before: left, after: right }) + } + walk(before.resolved as BiometricConfig, after.resolved as BiometricConfig, '') + const oldIntent = new Map(before.overrides.map(row => [row.path, row.value])) + const newIntent = new Map(after.overrides.map(row => [row.path, row.value])) + const intentChangedPaths = [...new Set([...oldIntent.keys(), ...newIntent.keys()])].filter(path => + !oldIntent.has(path) || !newIntent.has(path) || configHash(oldIntent.get(path)) !== configHash(newIntent.get(path))) + return { revisionId, againstRevisionId, beforeHash: before.hash, afterHash: after.hash, changes, intentChangedPaths } +} diff --git a/src/biometrics/profiles/evaluation-assessment.ts b/src/biometrics/profiles/evaluation-assessment.ts new file mode 100644 index 000000000..2de6e5c07 --- /dev/null +++ b/src/biometrics/profiles/evaluation-assessment.ts @@ -0,0 +1,103 @@ +import { verifyReplayContext, type ReplayContextQualification } from './replay-context' +import { numericObjectivePolicies } from '../backtest/contracts' +import { z } from 'zod' +import { configHash } from '../algorithms/canonical' +import { validateConfig } from '../algorithms/registry' +import { finalizeObjectiveReport, finalizeStageObjectiveReport } from '../backtest/objective-finalization' +import type { ObjectivePolicy } from '../backtest/objective-policy' +import type { StageObjectivePolicy } from '../backtest/stage-objective' +import { ReferenceInputError } from '../references/time' +import { verifyEvaluationIdentity } from './evaluation-identity' + +const assessmentSchema = z.object({ + baseline: z.enum(['default', 'current']), candidate: z.literal('candidate'), baselineHash: z.string(), candidateHash: z.string(), + status: z.enum(['improved', 'no_material_change', 'regressed', 'insufficient_evidence', 'incompatible']), + designated: z.boolean(), missingEvidence: z.array(z.string()), failedThresholds: z.array(z.string()), + evidence: z.object({ split: z.literal('holdout'), holdoutUse: z.literal(1), nights: z.array(z.object({ cohortId: z.string() })) }), +}) + +/** Internal qualified-build dependency, never populated from experiment JSON. */ +export interface DesignatedEvaluationPolicies { + /** Full frozen protocol, including alignment/calibration/uncertainty settings. */ + protocolHash: string | null + waveformContext: ReplayContextQualification | null + numeric: ObjectivePolicy | readonly ObjectivePolicy[] | null + stages: StageObjectivePolicy | null +} + +/** Reassess retained evidence; claimed report statuses are never authority. + * The caller still must commit activation under a current binding generation + * and recheck holdout exposure immediately before that commit. */ +export function assessEvaluation( + rawJob: Parameters[0], + result: Parameters[1], + expected: Parameters[2], + designations: DesignatedEvaluationPolicies, +) { + const identity = verifyEvaluationIdentity(rawJob, result, expected) + const reject = (message: string): never => { + throw new ReferenceInputError('EVALUATION_NOT_VALIDATED', message) + } + const { job } = identity + const configuration = (label: 'candidate' | 'current' | 'default') => job.configs.find(row => row.label === label) ?? reject('Missing evaluated configuration') + const candidate = configuration('candidate') + const current = configuration('current') + const after = validateConfig(candidate.resolved), before = validateConfig(current.resolved) + const domains = (Object.keys(after) as Array).filter(domain => configHash(after[domain]) !== configHash(before[domain])) + if (!domains.length) reject('Candidate does not change the current configuration') + const numericMetrics = { heart_rate: 'heart_rate', breathing_rate: 'breathing_rate', hrv_index: 'window_ibi_variability_v1' } as const + const requiredMetrics = domains.filter((domain): domain is keyof typeof numericMetrics => domain !== 'sleep_stages').map(domain => numericMetrics[domain]) + const policies = numericObjectivePolicies(job.protocol) + if (requiredMetrics.some(metric => !policies.some(policy => policy.metric === metric))) reject('Every changed numeric component requires its own objective evidence') + if (!job.protocol.stageObjectivePolicy) reject('Stage agreement and downstream effects require stage objective evidence') + if (requiredMetrics.length && (job.protocol.mode !== 'signal_replay' || job.protocol.waveformStages !== true)) reject('Numeric changes require waveform replay with downstream stages') + const raw = z.record(z.string(), z.unknown()).parse(result.report) + const numeric = finalizeObjectiveReport(raw, policies, raw.jobOperations, designations.numeric) + const finalized = finalizeStageObjectiveReport(numeric, job.protocol.stageObjectivePolicy, raw.jobOperations, designations.stages) + const outcomes: Array<{ status: string, designated: boolean, missingEvidence: string[], failedThresholds: string[], improvementRequired: boolean }> = [] + const check = (rawAssessments: unknown, improvementRequired: boolean) => { + const assessments = z.array(assessmentSchema).length(2).parse(rawAssessments) + for (const assessment of assessments) { + const baseline = configuration(assessment.baseline) + if (assessment.baselineHash !== baseline.resolvedHash || assessment.candidateHash !== candidate.resolvedHash) reject('Objective evidence belongs to another configuration') + const cohorts = assessment.evidence.nights.map(row => row.cohortId) + if (new Set(cohorts).size !== cohorts.length || cohorts.length !== identity.cohortIds.length + || cohorts.some(id => !identity.cohortIds.includes(id))) reject('Objective evidence omits frozen holdout nights') + outcomes.push({ ...assessment, improvementRequired }) + } + return assessments.map(row => ({ baseline: row.baseline, baselineHash: row.baselineHash, candidateHash: row.candidateHash, status: row.status })) + } + const numericRows = z.array(z.object({ policy: z.object({ metric: z.string() }) }).passthrough()).parse(finalized.objectiveAssessments ?? []) + const numericAssessments = policies.flatMap(policy => check(numericRows.filter(row => row.policy.metric === policy.metric), true) + .map(row => ({ ...row, metric: policy.metric }))) + const stageAssessments = check(finalized.stageObjectiveAssessments, domains.includes('sleep_stages')) + if (designations.protocolHash !== job.protocolHash) reject('Evaluation protocol is outside the qualified context') + const replayContext = expected.waveform + ? verifyReplayContext(raw.waveform, { ...expected.waveform, side: expected.side, + cohortIds: identity.cohortIds, configHashes: job.configs.map(config => config.resolvedHash) }, designations.waveformContext) + : null + const status = outcomes.some(row => row.status === 'incompatible') + ? 'incompatible' as const + : outcomes.some(row => !row.designated || row.missingEvidence.length || row.status === 'insufficient_evidence') + ? 'insufficient_evidence' as const + : outcomes.some(row => row.failedThresholds.length || row.status === 'regressed') + ? 'regressed' as const + : outcomes.every(row => row.status === 'improved' || (!row.improvementRequired && row.status === 'no_material_change')) + ? 'improved' as const + : 'no_material_change' as const + return { status, audit: { version: 'verified-evaluation-v1' as const, interpretation: 'agreement_with_named_reference_not_clinical_validation' as const, + reportHash: result.contentHash, revisionId: expected.revisionId, resolvedHash: expected.resolvedHash, + currentResolvedHash: expected.currentResolvedHash, datasetId: job.datasetId, datasetHash: job.datasetHash, + evaluatorBuildHash: job.evaluatorBuildHash, manifestHash: identity.report.manifestHash, protocolHash: job.protocolHash, + changedDomains: domains, numericAssessments, stageAssessments, replayContext, + policies: { numeric: job.protocol.objectivePolicies ?? job.protocol.objectivePolicy ?? null, stages: job.protocol.stageObjectivePolicy }, + ...(expected.waveform ? { waveform: expected.waveform } : {}), + } } +} + +/** Activation requires the independently reassessed whole candidate to improve. */ +export function verifyEvaluationAssessments(...args: Parameters) { + const result = assessEvaluation(...args) + if (result.status !== 'improved') throw new ReferenceInputError('EVALUATION_NOT_VALIDATED', 'Objective evidence does not support this activation') + return result.audit +} diff --git a/src/biometrics/profiles/evaluation-identity.ts b/src/biometrics/profiles/evaluation-identity.ts new file mode 100644 index 000000000..b8a8fe715 --- /dev/null +++ b/src/biometrics/profiles/evaluation-identity.ts @@ -0,0 +1,73 @@ +import { z } from 'zod' +import { configHash, dataHash } from '../algorithms/canonical' +import { defaultConfig, manifestHash, validateConfig } from '../algorithms/registry' +import { backtestJobSnapshotSchema, experimentProtocolSchema } from '../backtest/contracts' +import { ReferenceInputError } from '../references/time' + +const hash = z.string().regex(/^[a-f0-9]{64}$/) +const reportIdentitySchema = z.object({ + version: z.enum(['waveform-backtest-report-v1', 'processed-backtest-report-v1']), + side: z.enum(['left', 'right']), datasetId: z.string().uuid(), datasetHash: hash, + protocol: experimentProtocolSchema, protocolHash: hash, evaluatorBuildHash: hash, manifestHash: hash, + split: z.literal('holdout'), holdoutUse: z.literal(1), + support: z.enum(['identical_retained_waveform_inputs', 'identical_retained_epoch_intervals']), + nights: z.array(z.object({ cohortId: z.string() })).min(1).max(60), + waveform: z.object({ pythonBuildHash: hash, otherSideConfigHash: hash }).optional(), +}) + +/** These expectations are loaded by the service from the profile binding, + * immutable dataset and installed build. They must never come from a client. */ +export interface EvaluationIdentityExpectation { + side: 'left' | 'right' + revisionId: string + resolvedHash: string + currentResolvedHash: string + evaluatorBuildHash: string + dataset: { id: string, contentHash: string, side: 'left' | 'right', available: boolean, holdoutCohortIds: string[] } + waveform?: { pythonBuildHash: string, otherSideConfigHash: string } +} + +/** Necessary identity checks only: this does not designate a policy, award + * improvement, or authorize activation. Assessments must be verified next. */ +export function verifyEvaluationIdentity(rawJob: unknown, result: { + report: unknown + contentHash: string + available: boolean + currentHoldoutUseCount: number +}, expected: EvaluationIdentityExpectation) { + const reject = (message: string): never => { + throw new ReferenceInputError('EVALUATION_MISMATCH', message) + } + if (!result.available || !expected.dataset.available) reject('Retained evaluation artifacts are unavailable') + if (dataHash(result.report) !== result.contentHash) reject('Evaluation content hash does not match') + const job = backtestJobSnapshotSchema.parse(rawJob) + const report = reportIdentitySchema.parse(result.report) + if (job.split !== 'holdout' || job.holdoutUse !== 1 || result.currentHoldoutUseCount !== 1 || job.evidence === 'reused_holdout_is_validation') reject('Independent holdout has been reused or was not evaluated') + if (report.side !== expected.side || expected.dataset.side !== expected.side + || job.datasetId !== expected.dataset.id || report.datasetId !== job.datasetId + || job.datasetHash !== expected.dataset.contentHash || report.datasetHash !== job.datasetHash) reject('Evaluation dataset or side changed') + if (job.evaluatorBuildHash !== expected.evaluatorBuildHash || report.evaluatorBuildHash !== job.evaluatorBuildHash + || report.manifestHash !== manifestHash) reject('Evaluation build is incompatible') + if (configHash(job.protocol) !== job.protocolHash || report.protocolHash !== job.protocolHash + || configHash(report.protocol) !== job.protocolHash) reject('Evaluation protocol identity changed') + if (job.configs.length !== 3 || new Set(job.configs.map(row => row.label)).size !== 3) reject('Evaluation must include default, current and candidate') + for (const config of job.configs) { + if (config.manifestHash !== manifestHash || configHash(validateConfig(config.resolved)) !== config.resolvedHash) reject('Evaluated configuration identity changed') + if (config.label === 'default' && (config.revisionId !== null || config.resolvedHash !== configHash(defaultConfig))) reject('Evaluated default does not match the shipped default') + if (config.label === 'current' && config.resolvedHash !== expected.currentResolvedHash) reject('Current baseline changed since evaluation') + if (config.label === 'candidate' && (config.revisionId !== expected.revisionId || config.resolvedHash !== expected.resolvedHash)) reject('Evaluation belongs to another candidate revision') + } + const cohorts = report.nights.map(row => row.cohortId) + if (new Set(cohorts).size !== cohorts.length || new Set(expected.dataset.holdoutCohortIds).size !== expected.dataset.holdoutCohortIds.length + || cohorts.length !== expected.dataset.holdoutCohortIds.length || cohorts.some(id => !expected.dataset.holdoutCohortIds.includes(id))) reject('Evaluation does not cover the exact frozen holdout nights') + if (job.protocol.mode === 'signal_replay') { + if (!expected.waveform || !job.waveform || !report.waveform + || job.waveform.pythonBuildHash !== expected.waveform.pythonBuildHash || report.waveform.pythonBuildHash !== job.waveform.pythonBuildHash + || job.waveform.otherSide.resolvedHash !== expected.waveform.otherSideConfigHash || report.waveform.otherSideConfigHash !== job.waveform.otherSide.resolvedHash + || job.waveform.otherSide.manifestHash !== manifestHash || configHash(validateConfig(job.waveform.otherSide.resolved)) !== job.waveform.otherSide.resolvedHash + || report.version !== 'waveform-backtest-report-v1' || report.support !== 'identical_retained_waveform_inputs') reject('Waveform dependencies changed since evaluation') + } + else if (job.protocol.mode !== 'stage_replay' || job.waveform || report.waveform + || report.version !== 'processed-backtest-report-v1' || report.support !== 'identical_retained_epoch_intervals') reject('Evaluation mode cannot validate the selected replay') + return { job, report, contentHash: result.contentHash, cohortIds: cohorts } +} diff --git a/src/biometrics/profiles/evaluation-store.ts b/src/biometrics/profiles/evaluation-store.ts new file mode 100644 index 000000000..9b8ba9a0e --- /dev/null +++ b/src/biometrics/profiles/evaluation-store.ts @@ -0,0 +1,86 @@ +import { eq } from 'drizzle-orm' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import type * as configSchema from '@/src/db/schema' +import * as bioSchema from '@/src/db/biometrics-schema' +import { configHash } from '../algorithms/canonical' +import { BacktestJobStore } from '../backtest/jobs' +import { DatasetStore } from '../backtest/datasets' +import { backtestJobSnapshotSchema, datasetManifestSchema } from '../backtest/contracts' +import { ReferenceInputError } from '../references/time' +import { verifyEvaluationAssessments, type DesignatedEvaluationPolicies } from './evaluation-assessment' +import { ProfileStore } from './store' +import { resolveReplayBaseline } from './replay-baseline' + +type ConfigDB = BetterSQLite3Database +type BioDB = BetterSQLite3Database +export interface EvaluationSelection { + side: 'left' | 'right' + revisionId: string + expectedResolvedHash: string + expectedGeneration: number + evaluationId: string + expectedReportHash: string +} +/** Supplied by the installed build, never an RPC request. */ +export interface PromotionBuild { + evaluatorBuildHash: string + pythonBuildHash?: string + designations: DesignatedEvaluationPolicies +} + +function baseline(configDb: ConfigDB, bioDb: BioDB, side: 'left' | 'right') { + const desired = new ProfileStore(configDb).effective(side) + const actual = resolveReplayBaseline(configDb, bioDb, side) + if (desired.warning || (actual.provenance.warning && desired.binding.generation > 0) + || desired.hash !== actual.hash || (desired.binding.generation > 0 && actual.provenance.generation !== desired.binding.generation)) { + throw new ReferenceInputError('BASELINE_PENDING', 'Resolve pending or incompatible runtime configuration before validated activation') + } + return actual +} + +/** Full retained artifact parsing/hashing runs before the short commit locks. + * The returned guard must run while both databases are held by the caller. */ +export function prepareVerifiedEvaluation(configDb: ConfigDB, bioDb: BioDB, input: EvaluationSelection, build: PromotionBuild) { + const profiles = new ProfileStore(configDb), jobs = new BacktestJobStore(bioDb) + const revision = profiles.validateRevision(input.revisionId, input.side) + if (revision.hash !== input.expectedResolvedHash) throw new ReferenceInputError('REVISION_CONFLICT', 'Candidate hash changed') + if (profiles.binding(input.side).generation !== input.expectedGeneration) throw new ReferenceInputError('STALE_BINDING', 'Binding changed before verification') + const storedJob = jobs.get(input.evaluationId) + const job = backtestJobSnapshotSchema.parse(storedJob.input) + const experiment = jobs.getExperiment(storedJob.experimentId) + if (job.experimentId !== experiment.id || experiment.datasetId !== job.datasetId || experiment.side !== input.side + || experiment.protocolHash !== job.protocolHash || configHash(experiment.protocol) !== job.protocolHash) throw new ReferenceInputError('EVALUATION_MISMATCH', 'Stored experiment and job disagree') + const result = jobs.result(input.evaluationId) + if (!result || result.contentHash !== input.expectedReportHash) throw new ReferenceInputError('EVALUATION_MISMATCH', 'Completed evaluation hash does not match') + const dataset = new DatasetStore(bioDb).read(job.datasetId) + const manifest = datasetManifestSchema.parse(dataset.manifest) + const current = baseline(configDb, bioDb, input.side) + const otherSide = input.side === 'left' ? 'right' : 'left' + const other = job.protocol.mode === 'signal_replay' ? baseline(configDb, bioDb, otherSide) : null + const evidence = verifyEvaluationAssessments(job, result, { + side: input.side, revisionId: input.revisionId, resolvedHash: revision.hash, currentResolvedHash: current.hash, + evaluatorBuildHash: build.evaluatorBuildHash, + dataset: { id: dataset.id, contentHash: dataset.contentHash, side: dataset.side, available: dataset.available, + holdoutCohortIds: manifest.splitProtocol.assignments.filter(row => row.split === 'holdout').map(row => row.cohortId) }, + ...(other && build.pythonBuildHash ? { waveform: { pythonBuildHash: build.pythonBuildHash, otherSideConfigHash: other.hash } } : {}), + }, build.designations) + return { + evidence: { status: 'validated_reference_agreement' as const, evaluationId: input.evaluationId, ...evidence }, + recheck() { + const latestJob = bioDb.select({ state: bioSchema.biometricBacktestJobs.state, fencingToken: bioSchema.biometricBacktestJobs.fencingToken }) + .from(bioSchema.biometricBacktestJobs).where(eq(bioSchema.biometricBacktestJobs.id, input.evaluationId)).get() + const latestResult = bioDb.select({ fencingToken: bioSchema.biometricBacktestResults.fencingToken, + contentHash: bioSchema.biometricBacktestResults.contentHash, releasedAtMs: bioSchema.biometricBacktestResults.releasedAtMs }) + .from(bioSchema.biometricBacktestResults).where(eq(bioSchema.biometricBacktestResults.jobId, input.evaluationId)).get() + const latestDataset = bioDb.select({ contentHash: bioSchema.biometricDatasets.contentHash, releasedAtMs: bioSchema.biometricDatasets.releasedAtMs }) + .from(bioSchema.biometricDatasets).where(eq(bioSchema.biometricDatasets.id, dataset.id)).get() + if (!latestJob || latestJob.state !== 'succeeded' || latestJob.fencingToken !== storedJob.fencingToken || !latestResult + || latestResult.fencingToken !== storedJob.fencingToken || latestResult.contentHash !== result.contentHash || latestResult.releasedAtMs !== null + || !latestDataset || latestDataset.contentHash !== dataset.contentHash || latestDataset.releasedAtMs !== null + || jobs.holdoutUseCount(dataset.id) !== 1) throw new ReferenceInputError('EVALUATION_MISMATCH', 'Evaluation was released, replaced or reused before activation') + const actual = baseline(configDb, bioDb, input.side) + if (actual.hash !== current.hash || actual.provenance.generation !== current.provenance.generation + || (other && baseline(configDb, bioDb, otherSide).hash !== other.hash)) throw new ReferenceInputError('EVALUATION_MISMATCH', 'Runtime baseline changed before activation') + }, + } +} diff --git a/src/biometrics/profiles/health.ts b/src/biometrics/profiles/health.ts new file mode 100644 index 000000000..52e1ed476 --- /dev/null +++ b/src/biometrics/profiles/health.ts @@ -0,0 +1,42 @@ +import { z } from 'zod' + +// Numerical consumers heartbeat every 30 seconds after accepted frames. +// A stage view is retrospective and consumed on demand, so its receipt age +// cannot establish whether the continuous processor is healthy. +export const RUNTIME_HEARTBEAT_STALE_MS = 120000 +export const runtimeHealthSchema = z.object({ + state: z.enum(['unreported', 'current', 'stale', 'failed', 'mixed_generations', 'clock_skew']), + staleAfterMs: z.number(), + components: z.array(z.object({ component: z.string(), generation: z.number().nullable(), + state: z.enum(['unreported', 'current', 'stale', 'failed', 'pending_boundary', 'clock_skew']), ageMs: z.number().nullable() })), +}) +interface Receipt { component: string, generation: number, appliedHash: string, state: string, heartbeatAtMs: number, effectiveSampleTimeMs: number | null } +export function runtimeHealth(receipts: Receipt[], now = Date.now()): z.infer { + const names = ['heart_rate', 'breathing_rate', 'hrv_index'] + const selected = names.map(component => receipts.filter(row => row.component === component) + .sort((a, b) => b.generation - a.generation || b.heartbeatAtMs - a.heartbeatAtMs)[0]) + const components = names.map((component, index) => { + const receipt = selected[index] + if (!receipt) return { component, generation: null, state: 'unreported' as const, ageMs: null } + const ageMs = now - receipt.heartbeatAtMs + const state = receipt.state === 'failed' + ? 'failed' as const + : ageMs < 0 + ? 'clock_skew' as const + : ageMs > RUNTIME_HEARTBEAT_STALE_MS + ? 'stale' as const + : receipt.state !== 'applied' || receipt.effectiveSampleTimeMs === null ? 'pending_boundary' as const : 'current' as const + return { component, generation: receipt.generation, state, ageMs } + }) + const identities = new Set(selected.filter(row => row !== undefined).map(row => `${row.generation}:${row.appliedHash}`)) + const state = components.some(row => row.state === 'failed') + ? 'failed' + : components.some(row => row.state === 'clock_skew') + ? 'clock_skew' + : components.some(row => row.state === 'unreported' || row.state === 'pending_boundary') + ? 'unreported' + : components.some(row => row.state === 'stale') + ? 'stale' + : identities.size > 1 ? 'mixed_generations' : 'current' + return { state, staleAfterMs: RUNTIME_HEARTBEAT_STALE_MS, components } +} diff --git a/src/biometrics/profiles/history.ts b/src/biometrics/profiles/history.ts new file mode 100644 index 000000000..f816a1146 --- /dev/null +++ b/src/biometrics/profiles/history.ts @@ -0,0 +1,23 @@ +import { and, desc, eq, lt } from 'drizzle-orm' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import { z } from 'zod' +import * as schema from '@/src/db/schema' + +export const activationEventSchema = z.object({ + id: z.string().uuid(), side: z.enum(['left', 'right']), generation: z.number().int(), + action: z.enum(['activate', 'reset', 'rollback', 'migration', 'fallback']), + previousRevisionId: z.string().nullable(), nextRevisionId: z.string().nullable(), + previousHash: z.string(), nextHash: z.string(), evaluationId: z.string().nullable(), evidence: z.unknown(), + requester: z.string(), state: z.enum(['saved', 'pending_delivery', 'pending_boundary', 'applied', 'failed']), + reason: z.string().nullable(), createdAtMs: z.number(), updatedAtMs: z.number(), +}) +export const activationHistorySchema = z.object({ items: z.array(activationEventSchema), nextGeneration: z.number().int().nullable() }) + +export function listActivationHistory(db: BetterSQLite3Database, side: 'left' | 'right', beforeGeneration?: number) { + const events = schema.biometricActivationEvents + const rows = db.select().from(events).where(and(eq(events.side, side), + beforeGeneration === undefined ? undefined : lt(events.generation, beforeGeneration))) + .orderBy(desc(events.generation)).limit(51).all() + const items = rows.slice(0, 50).map(row => activationEventSchema.parse(row)) + return { items, nextGeneration: rows.length > 50 ? items[items.length - 1].generation : null } +} diff --git a/src/biometrics/profiles/legacy-stage.ts b/src/biometrics/profiles/legacy-stage.ts new file mode 100644 index 000000000..bf12d0410 --- /dev/null +++ b/src/biometrics/profiles/legacy-stage.ts @@ -0,0 +1,41 @@ +import { randomUUID } from 'node:crypto' +import { eq } from 'drizzle-orm' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import * as schema from '@/src/db/schema' +import { stageClassifierSchema, type StageClassifier } from '@/src/lib/sleep-stage-rules' +import { defaultConfig, legacyStageOverrides, type Override } from '../algorithms/registry' +import { configHash } from '../algorithms/canonical' +import { ReferenceInputError } from '../references/time' +import { requestManualActivation } from './activation' +import { ProfileStore } from './store' + +/** Compatibility writes retain other metric overrides and use the same atomic + * intent/outbox protocol. The legacy column is a migration baseline, not a + * second writable authority. Null clears only sleep-stage personalization. */ +export function writeLegacyStageClassifier(db: BetterSQLite3Database, side: 'left' | 'right', classifier: StageClassifier | null, now = Date.now()) { + const parsed = classifier === null ? null : stageClassifierSchema.parse(classifier) + return db.transaction((tx) => { + const settings = tx.select({ side: schema.sideSettings.side, classifier: schema.sideSettings.stageClassifier }).from(schema.sideSettings).where(eq(schema.sideSettings.side, side)).get() + if (!settings) throw new ReferenceInputError('NOT_FOUND', `Side settings for ${side} not found`) + const profiles = new ProfileStore(db, () => now) + // An invalid legacy value already resolves to defaults. An explicit new + // save can replace it without trying to create an invalid pinned revision. + if (settings.classifier === null || stageClassifierSchema.safeParse(settings.classifier).success) profiles.migrateLegacy(side) + const binding = profiles.binding(side) + const current = binding.desiredRevisionId ? profiles.validateRevision(binding.desiredRevisionId, side) : null + const overrides: Override[] = current + ? (current.revision.overrides as Override[]).filter(override => !override.path.startsWith('sleep_stages.')) + : [] + if (parsed) overrides.push(...legacyStageOverrides(parsed)) + const key = `legacy-stage-write:${randomUUID()}` + const revision = overrides.length + ? profiles.createRevision({ side, ...(current ? { parentRevisionId: current.revision.id } : {}), + overrides, origin: 'manual', requester: 'legacy-stage-settings', idempotencyKey: key }) + : null + const event = requestManualActivation(db, { side, action: revision ? 'activate' : 'reset', revisionId: revision?.id ?? null, + expectedGeneration: binding.generation, expectedResolvedHash: revision?.resolvedHash ?? configHash(defaultConfig), + boundary: 'immediate', requester: 'legacy-stage-settings', idempotencyKey: key, allowUnvalidated: true }, now) + return { classifier: parsed ?? defaultConfig.sleep_stages, isDefault: parsed === null, + activation: { generation: event.generation, revisionId: revision?.id ?? null, state: event.state } } + }, { behavior: 'immediate' }) +} diff --git a/src/biometrics/profiles/overview.ts b/src/biometrics/profiles/overview.ts new file mode 100644 index 000000000..8a228b27b --- /dev/null +++ b/src/biometrics/profiles/overview.ts @@ -0,0 +1,34 @@ +import { desc, eq } from 'drizzle-orm' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import * as configSchema from '@/src/db/schema' +import * as bioSchema from '@/src/db/biometrics-schema' +import { resolveStageClassifier } from '@/src/lib/sleep-stage-rules' +import { ProfileStore } from './store' +import { resolveStageView } from './stage-view' +import { runtimeHealth } from './health' + +/** Read-only inspection keeps desired intent separate from runtime evidence. */ +export function profileOverview(configDb: BetterSQLite3Database, bioDb: BetterSQLite3Database, side: 'left' | 'right', now = Date.now()) { + const profiles = new ProfileStore(configDb) + const desired = profiles.effective(side) + const legacy = configDb.select({ classifier: configSchema.sideSettings.stageClassifier }).from(configSchema.sideSettings).where(eq(configSchema.sideSettings.side, side)).get() + const receipts = bioDb.select().from(bioSchema.biometricRuntimeReceipts).where(eq(bioSchema.biometricRuntimeReceipts.side, side)) + .orderBy(desc(bioSchema.biometricRuntimeReceipts.generation), desc(bioSchema.biometricRuntimeReceipts.heartbeatAtMs)).limit(20).all() + const latest = receipts.filter((row, index) => receipts.findIndex(other => other.component === row.component) === index) + const sensorCalibration = bioDb.select().from(bioSchema.calibrationProfiles).where(eq(bioSchema.calibrationProfiles.side, side)).all() + .map(row => ({ id: row.id, sensorType: row.sensorType, status: row.status, qualityScore: row.qualityScore, + createdAtMs: row.createdAt.getTime(), expiresAtMs: row.expiresAt?.getTime() ?? null, + freshness: row.status !== 'completed' || row.expiresAt === null ? 'unknown' as const : row.expiresAt.getTime() <= now ? 'expired' as const : 'current' as const, + sourceWindowStartMs: row.sourceWindowStart === null ? null : row.sourceWindowStart * 1000, + sourceWindowEndMs: row.sourceWindowEnd === null ? null : row.sourceWindowEnd * 1000, + })) + return { desired, stageView: resolveStageView(configDb, bioDb, side), + legacyStageView: { classifier: resolveStageClassifier(legacy?.classifier), personalized: legacy?.classifier != null, + migrationPending: legacy?.classifier != null && desired.binding.generation === 0 }, + runtimeHealth: runtimeHealth(latest, now), + runtimeStatus: latest.length ? 'receipts_available' as const : 'unreported' as const, receipts: latest, sensorCalibration, + activationAvailable: true, + limitations: ['Desired configuration is not proof of runtime adoption.', 'Measured sensor calibration is independent of algorithm personalization.', + 'Manual changes are unvalidated; applied status requires matching component receipts.'], + } +} diff --git a/src/biometrics/profiles/promotion-build.ts b/src/biometrics/profiles/promotion-build.ts new file mode 100644 index 000000000..dbf882562 --- /dev/null +++ b/src/biometrics/profiles/promotion-build.ts @@ -0,0 +1,17 @@ +import { installedPromotionPolicies } from './qualification-registry' +import { packagedWorkerPath } from '../backtest/location' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import type * as bioSchema from '@/src/db/biometrics-schema' +import { BacktestJobStore } from '../backtest/jobs' +import { backtestJobSnapshotSchema } from '../backtest/contracts' +import { resolveInstalledReplayBuild } from '../backtest/submission' +import type { PromotionBuild } from './evaluation-store' + +/** This registry is server owned. Frozen client thresholds cannot designate a + * policy; qualified production policies must be installed explicitly. */ +export async function resolvePromotionBuild(bioDb: BetterSQLite3Database, evaluationId: string): Promise { + const job = new BacktestJobStore(bioDb).get(evaluationId) + const snapshot = backtestJobSnapshotSchema.parse(job.input) + const identity = await resolveInstalledReplayBuild(snapshot.protocol.mode) + return { ...identity, designations: await installedPromotionPolicies(packagedWorkerPath(), { ...identity, protocolHash: snapshot.protocolHash }) } +} diff --git a/src/biometrics/profiles/qualification-registry.ts b/src/biometrics/profiles/qualification-registry.ts new file mode 100644 index 000000000..0d05fbc61 --- /dev/null +++ b/src/biometrics/profiles/qualification-registry.ts @@ -0,0 +1,59 @@ +import { open } from 'node:fs/promises' +import { dirname, join } from 'node:path' +import { z } from 'zod' +import { configHash } from '../algorithms/canonical' +import { manifestHash } from '../algorithms/registry' +import { experimentProtocolSchema, numericObjectivePolicies } from '../backtest/contracts' +import { replayContextQualificationSchema } from './replay-context' +import type { DesignatedEvaluationPolicies } from './evaluation-assessment' + +const hash = z.string().regex(/^[a-f0-9]{64}$/) +export const qualificationRegistrySchema = z.object({ + version: z.literal('biometric-promotion-qualifications-v1'), + entries: z.array(z.object({ + qualificationId: z.string().min(1).max(200), manifestHash: hash, evaluatorBuildHash: hash, + pythonBuildHash: hash.optional(), protocol: experimentProtocolSchema, + fixtureEvidenceHash: hash, podEvidenceHash: hash, + waveformContext: replayContextQualificationSchema.nullable(), + }).strict()).max(32), +}).strict() +export const unqualifiedPolicies = (): DesignatedEvaluationPolicies => ({ protocolHash: null, waveformContext: null, numeric: null, stages: null }) + +/** Release-installed authority only. No RPC can write this file. Evidence hashes + * identify the reviewed qualification artifacts; they are not client assertions. */ +export async function installedPromotionPolicies(workerPath: string, identity: { + evaluatorBuildHash: string + pythonBuildHash?: string + protocolHash: string +}): Promise { + try { + const file = await open(join(dirname(workerPath), 'promotion-qualifications.json'), 'r') + let raw: Buffer + try { + if (!(await file.stat()).isFile()) return unqualifiedPolicies() + const buffer = Buffer.alloc(256 * 1024 + 1) + let size = 0 + while (size < buffer.length) { + const read = await file.read(buffer, size, buffer.length - size, size) + if (!read.bytesRead) break + size += read.bytesRead + } + if (size === buffer.length) return unqualifiedPolicies() + raw = buffer.subarray(0, size) + } + finally { await file.close() } + const registry = qualificationRegistrySchema.parse(JSON.parse(raw.toString('utf8'))) + const matches = registry.entries.filter(entry => entry.manifestHash === manifestHash + && entry.evaluatorBuildHash === identity.evaluatorBuildHash && entry.pythonBuildHash === identity.pythonBuildHash + && configHash(entry.protocol) === identity.protocolHash) + if (matches.length !== 1) return unqualifiedPolicies() + const entry = matches[0] + const numeric = numericObjectivePolicies(entry.protocol) + const stages = entry.protocol.stageObjectivePolicy + if (!stages || [stages, ...numeric].some(policy => policy.qualificationId !== entry.qualificationId) + || (entry.protocol.mode === 'signal_replay' && (!entry.pythonBuildHash || !entry.waveformContext)) + || (entry.protocol.mode !== 'signal_replay' && (entry.pythonBuildHash || entry.waveformContext))) return unqualifiedPolicies() + return { protocolHash: identity.protocolHash, numeric, stages, waveformContext: entry.waveformContext } + } + catch { return unqualifiedPolicies() } +} diff --git a/src/biometrics/profiles/recovery.ts b/src/biometrics/profiles/recovery.ts new file mode 100644 index 000000000..849493ccb --- /dev/null +++ b/src/biometrics/profiles/recovery.ts @@ -0,0 +1,72 @@ +import { randomUUID } from 'node:crypto' +import { and, eq } from 'drizzle-orm' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import * as schema from '@/src/db/schema' +import { configHash } from '../algorithms/canonical' +import { manifestHash, validateConfig } from '../algorithms/registry' +import { ProfileStore } from './store' +import { runtimePayloadSchema } from './runtime' + +/** Publish a new immutable generation when the shipped engine changes or a + * projection is unavailable. The desired pinned revision and rollback target + * survive fallback. This records intent, never processor adoption. */ +export function recoverRuntimeIntent(db: BetterSQLite3Database, side: 'left' | 'right', now = Date.now()) { + return db.transaction((tx) => { + const binding = tx.select().from(schema.biometricProfileBindings).where(eq(schema.biometricProfileBindings.side, side)).get() + if (!binding) return null // An unpersonalized side follows the daemon's shipped default. + const desired = new ProfileStore(db).effective(side) + const actualRevisionId = desired.warning ? null : binding.desiredRevisionId + const previous = tx.select().from(schema.biometricConfigOutbox).where(and(eq(schema.biometricConfigOutbox.side, side), + eq(schema.biometricConfigOutbox.generation, binding.generation))).get() + const decoded = runtimePayloadSchema.safeParse(previous?.payload) + let matching = false + if (previous && decoded.success) { + const payload = decoded.data + try { + matching = previous.state !== 'failed' && payload.side === side && payload.generation === binding.generation + && payload.resolvedHash === previous.resolvedHash && payload.manifestHash === manifestHash + && payload.revisionId === actualRevisionId && payload.resolvedHash === desired.hash + && configHash(validateConfig(payload.resolved)) === desired.hash + } + catch { /* A corrupt immutable projection requires a new audited generation. */ } + } + if (matching) return null + + const defaultUpdate = binding.desiredRevisionId === null && decoded.success && decoded.data.manifestHash !== manifestHash + const restoringProfile = binding.desiredRevisionId !== null && decoded.success && decoded.data.revisionId === null + const status = desired.warning + ? 'profile_incompatible' + : defaultUpdate ? 'system_default_update' : restoringProfile ? 'profile_restored' : 'runtime_repair' + const reason = status === 'profile_incompatible' + ? 'Pinned profile is unavailable in this build; using shipped defaults' + : status === 'system_default_update' + ? 'Refresh shipped system defaults' + : status === 'profile_restored' + ? 'Pinned profile is compatible again; restore at the next session' + : 'Repair missing or incompatible runtime intent' + const boundary = status === 'profile_restored' + ? 'next_session' + : status === 'runtime_repair' + ? decoded.success ? decoded.data.boundary : binding.desiredRevisionId ? 'next_session' : 'immediate' + : 'immediate' + const generation = binding.generation + 1 + const key = `runtime-recovery:${side}:${generation}:${manifestHash}` + const evidence = { status, requestedRevisionId: binding.desiredRevisionId, actualRevisionId, + manifestHash, resolvedHash: desired.hash, warning: desired.warning } + tx.update(schema.biometricProfileBindings).set({ generation, updatedAtMs: now }) + .where(and(eq(schema.biometricProfileBindings.side, side), eq(schema.biometricProfileBindings.generation, binding.generation))).run() + // Prevent an undelivered obsolete payload from racing the recovery payload. + if (previous?.state === 'pending') tx.update(schema.biometricConfigOutbox).set({ state: 'failed', lastError: 'Superseded by runtime recovery' }) + .where(eq(schema.biometricConfigOutbox.id, previous.id)).run() + const event = tx.insert(schema.biometricActivationEvents).values({ id: randomUUID(), side, generation, action: 'fallback', + previousRevisionId: binding.desiredRevisionId, nextRevisionId: actualRevisionId, + previousHash: previous?.resolvedHash ?? desired.hash, nextHash: desired.hash, evidence, + requester: 'runtime-coordinator', state: 'pending_delivery', reason, + idempotencyKey: key, requestHash: configHash(evidence), createdAtMs: now, updatedAtMs: now }).returning().get() + tx.insert(schema.biometricConfigOutbox).values({ id: randomUUID(), side, generation, + resolvedHash: desired.hash, payload: { schemaVersion: 1, side, generation, revisionId: actualRevisionId, + manifestHash, resolved: desired.resolved, resolvedHash: desired.hash, boundary }, + state: 'pending', nextAttemptAtMs: now, createdAtMs: now }).run() + return event + }, { behavior: 'immediate' }) +} diff --git a/src/biometrics/profiles/replay-baseline.ts b/src/biometrics/profiles/replay-baseline.ts new file mode 100644 index 000000000..bcd1ae2e3 --- /dev/null +++ b/src/biometrics/profiles/replay-baseline.ts @@ -0,0 +1,50 @@ +import { and, desc, eq, ne } from 'drizzle-orm' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import type * as configSchema from '@/src/db/schema' +import * as bioSchema from '@/src/db/biometrics-schema' +import { configHash } from '../algorithms/canonical' +import { defaultConfig, manifestHash, validateConfig } from '../algorithms/registry' +import { ReferenceInputError } from '../references/time' +import { runtimePayloadSchema } from './runtime' +import { resolveStageView } from './stage-view' + +/** Freeze a coherent numerical baseline, never desired-but-unadopted intent. + * Call inside the caller's read snapshot when capturing both logical sides. */ +export function resolveReplayBaseline(configDb: BetterSQLite3Database, bioDb: BetterSQLite3Database, side: 'left' | 'right') { + const receipts = bioSchema.biometricRuntimeReceipts + const components = ['heart_rate', 'breathing_rate', 'hrv_index'] + const latest = components.map(component => bioDb.select().from(receipts) + .where(and(eq(receipts.side, side), eq(receipts.component, component), ne(receipts.state, 'pending_boundary'))).orderBy(desc(receipts.generation)).limit(1).get()) + if (latest.some(row => row?.state === 'failed')) throw new ReferenceInputError('MISSING_CONTEXT', 'Numerical adoption failed; inspect runtime health before waveform evaluation') + // Pending boundary receipts do not replace the last acknowledged generation. + const applied = components.map(component => bioDb.select().from(receipts) + .where(and(eq(receipts.side, side), eq(receipts.component, component), eq(receipts.state, 'applied'))) + .orderBy(desc(receipts.generation)).limit(1).get()) + if (applied.every(row => !row)) { + const view = resolveStageView(configDb, bioDb, side) + if (view.provenance.warning) throw new ReferenceInputError('MISSING_CONTEXT', 'Current fallback configuration is uncertain; inspect profile compatibility') + const resolved = validateConfig({ ...structuredClone(defaultConfig), sleep_stages: view.classifier }) + return { resolved, hash: configHash(resolved), revisionId: null, + provenance: { source: 'unacknowledged_defaults' as const, generation: null, effectiveSampleTimeMs: null, + warning: 'No numerical configuration has been acknowledged; this baseline uses shipped numerical defaults and the current stage view.' } } + } + const first = applied[0] + if (!first || first.effectiveSampleTimeMs === null || applied.some(row => !row || row.effectiveSampleTimeMs === null + || row.generation !== first.generation || row.appliedHash !== first.appliedHash)) { + throw new ReferenceInputError('MISSING_CONTEXT', 'Numerical components have not acknowledged one consistent configuration') + } + const projection = bioDb.select().from(bioSchema.biometricRuntimeConfigs).where(and( + eq(bioSchema.biometricRuntimeConfigs.side, side), eq(bioSchema.biometricRuntimeConfigs.generation, first.generation))).get() + try { + if (!projection) throw new Error('Missing projection') + const payload = runtimePayloadSchema.parse(projection.payload) + const resolved = validateConfig(payload.resolved) + if (payload.side !== side || payload.generation !== first.generation || payload.boundary !== projection.boundary + || payload.manifestHash !== manifestHash || payload.resolvedHash !== first.appliedHash + || projection.resolvedHash !== first.appliedHash || configHash(resolved) !== first.appliedHash) throw new Error('Inconsistent projection') + return { resolved, hash: first.appliedHash, revisionId: payload.revisionId, + provenance: { source: 'acknowledged_runtime' as const, generation: first.generation, + effectiveSampleTimeMs: Math.max(...applied.map(row => row?.effectiveSampleTimeMs ?? 0)), warning: null } } + } + catch { throw new ReferenceInputError('MISSING_CONTEXT', 'Acknowledged numerical configuration is unavailable or incompatible') } +} diff --git a/src/biometrics/profiles/replay-context.ts b/src/biometrics/profiles/replay-context.ts new file mode 100644 index 000000000..a42ad2f8c --- /dev/null +++ b/src/biometrics/profiles/replay-context.ts @@ -0,0 +1,48 @@ +import { z } from 'zod' +import { waveformEvaluationSchema } from '../backtest/waveform-output' +import { dataHash } from '../algorithms/canonical' +import { ReferenceInputError } from '../references/time' + +const limitation = z.enum(['event_time_replaces_arrival_time', 'cold_start_without_serialized_state', + 'less_than_five_minutes_preroll', 'missing_sleeper_mode', 'missing_occupancy_context', 'missing_pump_context']) +export const replayContextQualificationSchema = z.object({ allowedLimitations: z.array(limitation).max(6).refine(codes => new Set(codes).size === codes.length, 'Duplicate qualified limitation') }).strict() +export type ReplayContextQualification = z.infer + +/** Build-pinned driver diagnostics from every frozen cohort/configuration. A + * threshold policy cannot implicitly approve replay approximations. */ +export function verifyReplayContext(raw: unknown, expected: { + cohortIds: string[] + configHashes: string[] + side: 'left' | 'right' + pythonBuildHash: string + otherSideConfigHash: string +}, qualification: ReplayContextQualification | null) { + const reject = (message: string): never => { + throw new ReferenceInputError('EVALUATION_NOT_VALIDATED', message) + } + if (!qualification) return reject('Waveform context has not been qualified') + const allowed = replayContextQualificationSchema.parse(qualification).allowedLimitations + const envelope = z.object({ hash: z.unknown(), outputs: z.unknown() }).parse(raw) + const evaluation = waveformEvaluationSchema.parse(envelope) + if (dataHash(evaluation.outputs) !== evaluation.hash) reject('Waveform diagnostic content hash changed') + const required = new Set(expected.cohortIds.flatMap(cohort => [...new Set(expected.configHashes)].map(hash => `${cohort}:${hash}`))) + const observed = new Set() + const otherSide = expected.side === 'left' ? 'right' : 'left' + for (const output of evaluation.outputs) { + if (!required.delete(`${output.cohortId}:${output.configHash}`)) reject('Unexpected or duplicated waveform context output') + if (output.run.build_hash !== expected.pythonBuildHash || output.run.config_hashes[expected.side] !== output.configHash + || output.run.config_hashes[otherSide] !== expected.otherSideConfigHash + || output.run.counts.scored_outputs !== output.readings.length) reject('Waveform context identity changed') + for (const reading of output.readings) { + if (reading.configHash !== output.run.config_hashes[reading.side]) reject('Waveform reading identity changed') + } + const codes = [...output.run.limitations, ...output.readings.flatMap(reading => reading.limitations)] + for (const code of codes) { + if (!allowed.some(item => item === code)) reject(`Unqualified replay limitation: ${code}`) + observed.add(code) + } + } + if (required.size) reject('Missing waveform context output') + return { version: 'qualified-replay-context-v1' as const, observedLimitations: [...observed].sort(), + allowedLimitations: [...allowed].sort(), outputs: evaluation.outputs.length } +} diff --git a/src/biometrics/profiles/runtime-host.ts b/src/biometrics/profiles/runtime-host.ts new file mode 100644 index 000000000..a533e5c27 --- /dev/null +++ b/src/biometrics/profiles/runtime-host.ts @@ -0,0 +1,51 @@ +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import type * as configSchema from '@/src/db/schema' +import type * as bioSchema from '@/src/db/biometrics-schema' +import { ProfileStore } from './store' +import { recoverRuntimeIntent } from './recovery' +import { deliverRuntimeOutbox, reconcileActivation, repairCurrentRuntimeProjections } from './runtime' + +type ConfigDB = BetterSQLite3Database +type BioDB = BetterSQLite3Database + +/** Bounded startup/recovery pass. One failure must not suppress the other side + * or a later retry. No hardware commands, numerical work or fabricated receipts. */ +export function tickBiometricRuntime(configDb: ConfigDB, bioDb: BioDB, now = Date.now()) { + const errors: string[] = [] + const attempt = (name: string, operation: () => unknown) => { + try { + operation() + } + catch { + errors.push(name) + } + } + const profiles = new ProfileStore(configDb, () => now) + for (const side of ['left', 'right'] as const) attempt(`migration:${side}`, () => profiles.migrateLegacy(side)) + for (const side of ['left', 'right'] as const) attempt(`intent-recovery:${side}`, () => recoverRuntimeIntent(configDb, side, now)) + attempt('projection-repair', () => repairCurrentRuntimeProjections(configDb, bioDb, now)) + attempt('outbox-delivery', () => deliverRuntimeOutbox(configDb, bioDb, now)) + for (const side of ['left', 'right'] as const) attempt(`reconcile:${side}`, () => reconcileActivation(configDb, bioDb, side, now)) + return { errors } +} + +interface Host { timer: ReturnType, tick: () => void } +const shared = globalThis as typeof globalThis & { __biometricRuntimeHost?: Host } +export function startBiometricRuntime(configDb: ConfigDB, bioDb: BioDB) { + if (shared.__biometricRuntimeHost) return + let previousError = '' + const tick = () => { + const result = tickBiometricRuntime(configDb, bioDb) + const signature = result.errors.join(', ') + if (signature && signature !== previousError) console.warn(`[biometrics] Runtime coordination will retry: ${signature}`) + previousError = signature + } + const timer = setInterval(tick, 5000) + timer.unref() + shared.__biometricRuntimeHost = { timer, tick } + tick() +} +export function stopBiometricRuntime() { + if (shared.__biometricRuntimeHost) clearInterval(shared.__biometricRuntimeHost.timer) + delete shared.__biometricRuntimeHost +} diff --git a/src/biometrics/profiles/runtime.ts b/src/biometrics/profiles/runtime.ts new file mode 100644 index 000000000..4ee20e269 --- /dev/null +++ b/src/biometrics/profiles/runtime.ts @@ -0,0 +1,111 @@ +import { and, asc, eq, lte } from 'drizzle-orm' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import { z } from 'zod' +import * as configSchema from '@/src/db/schema' +import * as bioSchema from '@/src/db/biometrics-schema' +import { configHash } from '../algorithms/canonical' +import { manifestHash, validateConfig } from '../algorithms/registry' +import { ReferenceInputError } from '../references/time' + +export const runtimePayloadSchema = z.object({ schemaVersion: z.literal(1), side: z.enum(['left', 'right']), generation: z.number().int().positive(), + revisionId: z.string().uuid().nullable(), manifestHash: z.string(), resolved: z.unknown(), resolvedHash: z.string(), + boundary: z.enum(['immediate', 'next_session']) }).strict() +export const RUNTIME_COMPONENTS = ['heart_rate', 'breathing_rate', 'hrv_index', 'sleep_stages'] as const + +/** Idempotent projection only; never writes canonical vitals or runtime receipts. */ +export function projectRuntimeConfig(db: BetterSQLite3Database, raw: unknown, now = Date.now()) { + const payload = runtimePayloadSchema.parse(raw) + const resolved = validateConfig(payload.resolved) + if (payload.manifestHash !== manifestHash || configHash(resolved) !== payload.resolvedHash) throw new ReferenceInputError('PROFILE_INCOMPATIBLE', 'Runtime payload does not match its validated engine and configuration') + return db.transaction((tx) => { + const existing = tx.select().from(bioSchema.biometricRuntimeConfigs).where(and(eq(bioSchema.biometricRuntimeConfigs.side, payload.side), eq(bioSchema.biometricRuntimeConfigs.generation, payload.generation))).get() + if (existing) { + if (existing.resolvedHash !== payload.resolvedHash || configHash(existing.payload) !== configHash(payload)) throw new ReferenceInputError('REVISION_CONFLICT', 'Runtime generation already contains a different immutable payload') + return { side: payload.side, generation: payload.generation, alreadyProjected: true } + } + tx.insert(bioSchema.biometricRuntimeConfigs).values({ side: payload.side, generation: payload.generation, + resolvedHash: payload.resolvedHash, payload, boundary: payload.boundary, createdAtMs: now }).run() + return { side: payload.side, generation: payload.generation, alreadyProjected: false } + }, { behavior: 'immediate' }) +} + +/** Two short independent DB transactions; never assumes cross-database atomicity. */ +export function deliverRuntimeOutbox(configDb: BetterSQLite3Database, bioDb: BetterSQLite3Database, now = Date.now()) { + const outbox = configSchema.biometricConfigOutbox + const pending = configDb.select().from(outbox).where(and(eq(outbox.state, 'pending'), lte(outbox.nextAttemptAtMs, now))) + .orderBy(asc(outbox.createdAtMs), asc(outbox.generation)).limit(20).all() + let delivered = 0, failed = 0 + for (const row of pending) { + try { + const payload = runtimePayloadSchema.parse(row.payload) + if (payload.side !== row.side || payload.generation !== row.generation || payload.resolvedHash !== row.resolvedHash) throw new ReferenceInputError('PROFILE_INCOMPATIBLE', 'Outbox envelope and payload disagree') + projectRuntimeConfig(bioDb, payload, now) + configDb.update(outbox).set({ state: 'delivered', deliveredAtMs: now, attempts: row.attempts + 1, lastError: null }) + .where(eq(outbox.id, row.id)).run() + delivered++ + } + catch (error) { + const permanent = error instanceof z.ZodError || error instanceof ReferenceInputError + || (error instanceof Error && /^(Invalid |Unknown |Expected finite|JSON resource)/.test(error.message)) + configDb.transaction((tx) => { + tx.update(outbox).set({ state: permanent ? 'failed' : 'pending', attempts: row.attempts + 1, + nextAttemptAtMs: now + Math.min(300000, 1000 * 2 ** Math.min(row.attempts, 8)), + lastError: permanent ? 'Incompatible or conflicting runtime payload' : 'Runtime database delivery failed; retry scheduled', + }).where(eq(outbox.id, row.id)).run() + if (permanent) tx.update(configSchema.biometricActivationEvents).set({ state: 'failed', reason: 'Runtime payload rejected during delivery', updatedAtMs: now }) + .where(and(eq(configSchema.biometricActivationEvents.side, row.side), eq(configSchema.biometricActivationEvents.generation, row.generation))).run() + }, { behavior: 'immediate' }) + failed++ + } + } + return { delivered, failed } +} + +/** Recreate authoritative projections after a biometric-history reset. */ +export function repairCurrentRuntimeProjections(configDb: BetterSQLite3Database, bioDb: BetterSQLite3Database, now = Date.now()) { + let restored = 0 + for (const binding of configDb.select().from(configSchema.biometricProfileBindings).all()) { + const row = configDb.select().from(configSchema.biometricConfigOutbox).where(and(eq(configSchema.biometricConfigOutbox.side, binding.side), + eq(configSchema.biometricConfigOutbox.generation, binding.generation), eq(configSchema.biometricConfigOutbox.state, 'delivered'))).get() + if (row) { + const payload = runtimePayloadSchema.parse(row.payload) + if (payload.side !== row.side || payload.generation !== row.generation || payload.resolvedHash !== row.resolvedHash) throw new ReferenceInputError('PROFILE_INCOMPATIBLE', 'Stored runtime envelope and payload disagree') + try { + if (!projectRuntimeConfig(bioDb, payload, now).alreadyProjected) restored++ + } + catch (error) { + if (!(error instanceof ReferenceInputError) || error.code !== 'REVISION_CONFLICT') throw error + // Preserve the conflicting immutable row for diagnosis. Recovery will + // publish a fresh audited generation on its next pass. + configDb.update(configSchema.biometricConfigOutbox).set({ state: 'failed', lastError: 'Runtime projection conflicts with authoritative intent' }) + .where(eq(configSchema.biometricConfigOutbox.id, row.id)).run() + } + } + } + return restored +} + +/** Only matching receipts for the desired generation can establish adoption. */ +export function reconcileActivation(configDb: BetterSQLite3Database, bioDb: BetterSQLite3Database, side: 'left' | 'right', now = Date.now()) { + const binding = configDb.select().from(configSchema.biometricProfileBindings).where(eq(configSchema.biometricProfileBindings.side, side)).get() + if (!binding) return null + const event = configDb.select().from(configSchema.biometricActivationEvents).where(and(eq(configSchema.biometricActivationEvents.side, side), eq(configSchema.biometricActivationEvents.generation, binding.generation))).get() + if (!event) return null + const delivery = configDb.select().from(configSchema.biometricConfigOutbox).where(and(eq(configSchema.biometricConfigOutbox.side, side), eq(configSchema.biometricConfigOutbox.generation, binding.generation))).get() + const receipts = bioDb.select().from(bioSchema.biometricRuntimeReceipts).where(and(eq(bioSchema.biometricRuntimeReceipts.side, side), eq(bioSchema.biometricRuntimeReceipts.generation, binding.generation))).all() + .filter(row => row.appliedHash === event.nextHash && RUNTIME_COMPONENTS.includes(row.component as typeof RUNTIME_COMPONENTS[number])) + const rejected = receipts.find(row => row.state === 'failed') + const allApplied = RUNTIME_COMPONENTS.every(component => receipts.some(row => row.component === component && row.state === 'applied' && row.effectiveSampleTimeMs !== null)) + const state = delivery?.state === 'failed' || rejected + ? 'failed' + : delivery?.state !== 'delivered' ? 'pending_delivery' : allApplied ? 'applied' : 'pending_boundary' + // Re-check desired generation inside the write transaction. A newer request + // cannot be acknowledged by receipts read for an older request. + return configDb.transaction((tx) => { + const current = tx.select().from(configSchema.biometricProfileBindings).where(eq(configSchema.biometricProfileBindings.side, side)).get() + if (current?.generation !== binding.generation) return null + tx.update(configSchema.biometricActivationEvents).set({ state, reason: rejected ? 'Runtime component rejected configuration' : delivery?.state === 'failed' || event.action === 'fallback' ? event.reason : null, + updatedAtMs: state === event.state ? event.updatedAtMs : now }).where(eq(configSchema.biometricActivationEvents.id, event.id)).run() + return { side, generation: binding.generation, state, resolvedHash: event.nextHash, receipts } + }, { behavior: 'immediate' }) +} diff --git a/src/biometrics/profiles/stage-view.ts b/src/biometrics/profiles/stage-view.ts new file mode 100644 index 000000000..2131d3603 --- /dev/null +++ b/src/biometrics/profiles/stage-view.ts @@ -0,0 +1,100 @@ +import { and, desc, eq, isNotNull } from 'drizzle-orm' +import { alias } from 'drizzle-orm/sqlite-core' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import { z } from 'zod' +import * as configSchema from '@/src/db/schema' +import * as bioSchema from '@/src/db/biometrics-schema' +import { resolveStageClassifier, stageClassifierSchema } from '@/src/lib/sleep-stage-rules' +import { configHash } from '../algorithms/canonical' +import { defaultConfig, manifestHash, validateConfig } from '../algorithms/registry' +import { runtimePayloadSchema } from './runtime' +import { ProfileStore } from './store' + +export const stageViewSchema = z.object({ + classifier: stageClassifierSchema, + isDefault: z.boolean(), + provenance: z.object({ + interpretation: z.literal('current_classifier_view'), source: z.enum(['runtime', 'legacy', 'default', 'demo']), + generation: z.number().int().nullable(), revisionId: z.string().nullable(), + bundleHash: z.string(), stageHash: z.string(), manifestHash: z.string().nullable(), + effectiveSampleTimeMs: z.number().nullable(), warning: z.string().nullable(), + }), +}) +export type StageView = z.infer +type ConfigDB = BetterSQLite3Database +type BioDB = BetterSQLite3Database +type Side = 'left' | 'right' + +/** The stage consumer follows the boundary actually reached by all numerical + * components. Desired intent alone must never rescore history early. */ +export function resolveStageView(configDb: ConfigDB, bioDb: BioDB, side: Side): StageView { + const projection = bioSchema.biometricRuntimeConfigs + const heart = alias(bioSchema.biometricRuntimeReceipts, 'stage_heart') + const breathing = alias(bioSchema.biometricRuntimeReceipts, 'stage_breathing') + const hrv = alias(bioSchema.biometricRuntimeReceipts, 'stage_hrv') + const match = (receipt: typeof heart | typeof breathing | typeof hrv, component: string) => and( + eq(receipt.side, projection.side), eq(receipt.generation, projection.generation), + eq(receipt.appliedHash, projection.resolvedHash), eq(receipt.component, component), + eq(receipt.state, 'applied'), isNotNull(receipt.effectiveSampleTimeMs), + ) + const eligible = bioDb.select({ projection, + heartTime: heart.effectiveSampleTimeMs, breathingTime: breathing.effectiveSampleTimeMs, hrvTime: hrv.effectiveSampleTimeMs, + }).from(projection) + .innerJoin(heart, match(heart, 'heart_rate')) + .innerJoin(breathing, match(breathing, 'breathing_rate')) + .innerJoin(hrv, match(hrv, 'hrv_index')) + .where(eq(projection.side, side)).orderBy(desc(projection.generation)).limit(1).get() + const desiredWarning = new ProfileStore(configDb).effective(side).warning + let warning: string | null = desiredWarning + if (eligible) { + try { + if (eligible.heartTime === null || eligible.breathingTime === null || eligible.hrvTime === null) throw new Error('Missing runtime boundary') + const row = eligible.projection + const payload = runtimePayloadSchema.parse(row.payload) + const resolved = validateConfig(payload.resolved) + if (payload.side !== side || payload.generation !== row.generation || payload.boundary !== row.boundary + || payload.manifestHash !== manifestHash || payload.resolvedHash !== row.resolvedHash || configHash(resolved) !== row.resolvedHash) { + throw new Error('Incompatible runtime projection') + } + return { classifier: resolved.sleep_stages, isDefault: configHash(resolved.sleep_stages) === configHash(defaultConfig.sleep_stages), + provenance: { interpretation: 'current_classifier_view', source: 'runtime', generation: row.generation, + revisionId: payload.revisionId, bundleHash: row.resolvedHash, stageHash: configHash(resolved.sleep_stages), manifestHash, + effectiveSampleTimeMs: Math.max(eligible.heartTime, eligible.breathingTime, eligible.hrvTime), warning: payload.revisionId === null ? desiredWarning : null } } + } + catch { + warning = 'Applied profile is incompatible with this build; stage view uses system defaults' + } + } + // Preserve legacy behavior until its first migration/adoption. Incompatible + // engines must visibly fall back to shipped defaults, never be relabeled. + const legacy = warning + ? null + : configDb.select({ classifier: configSchema.sideSettings.stageClassifier }) + .from(configSchema.sideSettings).where(eq(configSchema.sideSettings.side, side)).get()?.classifier + const classifier = resolveStageClassifier(legacy) + const personalized = legacy != null && stageClassifierSchema.safeParse(legacy).success + if (legacy != null && !personalized) warning = 'Stored legacy stage rules are invalid; using system defaults' + return { classifier, isDefault: !personalized, + provenance: { interpretation: 'current_classifier_view', source: personalized ? 'legacy' : 'default', generation: null, + revisionId: null, bundleHash: configHash({ ...defaultConfig, sleep_stages: classifier }), stageHash: configHash(classifier), + manifestHash: personalized ? null : manifestHash, effectiveSampleTimeMs: null, warning } } +} + +/** Call at the stage-consumer read boundary. This acknowledges rules delivered + * for classification, not canonical historical stage observations. */ +export function consumeStageView(configDb: ConfigDB, bioDb: BioDB, side: Side, now = Date.now()): StageView { + return bioDb.transaction((tx) => { + const view = resolveStageView(configDb, tx, side) + const provenance = view.provenance + if (provenance.source === 'runtime' && provenance.generation !== null) { + const receipt = bioSchema.biometricRuntimeReceipts + tx.insert(receipt).values({ side, component: 'sleep_stages', generation: provenance.generation, + appliedHash: provenance.bundleHash, effectiveSampleTimeMs: provenance.effectiveSampleTimeMs, + state: 'applied', error: null, heartbeatAtMs: now, + }).onConflictDoUpdate({ target: [receipt.side, receipt.component, receipt.generation], + set: { appliedHash: provenance.bundleHash, effectiveSampleTimeMs: provenance.effectiveSampleTimeMs, + state: 'applied', error: null, heartbeatAtMs: now } }).run() + } + return view + }, { behavior: 'immediate' }) +} diff --git a/src/biometrics/profiles/status.ts b/src/biometrics/profiles/status.ts new file mode 100644 index 000000000..1ba91e229 --- /dev/null +++ b/src/biometrics/profiles/status.ts @@ -0,0 +1,46 @@ +import { and, eq } from 'drizzle-orm' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import { z } from 'zod' +import * as configSchema from '@/src/db/schema' +import * as bioSchema from '@/src/db/biometrics-schema' +import { ReferenceInputError } from '../references/time' +import { activationEventSchema } from './history' +import { runtimeHealth, runtimeHealthSchema } from './health' +import { RUNTIME_COMPONENTS } from './runtime' + +export const activationStatusSchema = z.object({ + desiredGeneration: z.number().int(), isCurrent: z.boolean(), event: activationEventSchema.nullable(), + deliveryState: z.enum(['pending', 'delivered', 'failed']).nullable(), + observedState: z.enum(['unreported', 'pending_delivery', 'pending_boundary', 'applied', 'failed']), + receiptHealth: runtimeHealthSchema, + receipts: z.array(z.object({ component: z.string(), generation: z.number().int(), appliedHash: z.string(), + effectiveSampleTimeMs: z.number().nullable(), state: z.enum(['pending_boundary', 'applied', 'failed']), + error: z.string().nullable(), heartbeatAtMs: z.number() })), +}) + +/** Observation only: polling never acknowledges a processor or writes intent. */ +export function activationStatus(configDb: BetterSQLite3Database, bioDb: BetterSQLite3Database, + side: 'left' | 'right', id?: string, now = Date.now()): z.infer { + const binding = configDb.select().from(configSchema.biometricProfileBindings).where(eq(configSchema.biometricProfileBindings.side, side)).get() + const desiredGeneration = binding?.generation ?? 0 + const events = configSchema.biometricActivationEvents + const event = configDb.select().from(events).where(and(eq(events.side, side), id ? eq(events.id, id) : eq(events.generation, desiredGeneration))).get() + if (id && !event) throw new ReferenceInputError('NOT_FOUND', 'Activation event not found for this side') + const delivery = event + ? configDb.select().from(configSchema.biometricConfigOutbox).where(and( + eq(configSchema.biometricConfigOutbox.side, side), eq(configSchema.biometricConfigOutbox.generation, event.generation))).get() + : undefined + const receipts = event + ? bioDb.select().from(bioSchema.biometricRuntimeReceipts).where(and( + eq(bioSchema.biometricRuntimeReceipts.side, side), eq(bioSchema.biometricRuntimeReceipts.generation, event.generation))).all() + .filter(row => row.appliedHash === event.nextHash && RUNTIME_COMPONENTS.includes(row.component as typeof RUNTIME_COMPONENTS[number])) + : [] + const allApplied = RUNTIME_COMPONENTS.every(component => receipts.some(row => row.component === component && row.state === 'applied' && row.effectiveSampleTimeMs !== null)) + const observedState = !event + ? 'unreported' + : delivery?.state === 'failed' || receipts.some(row => row.state === 'failed') + ? 'failed' + : delivery?.state !== 'delivered' ? 'pending_delivery' : allApplied ? 'applied' : 'pending_boundary' + return { desiredGeneration, isCurrent: event?.generation === desiredGeneration, event: event ? activationEventSchema.parse(event) : null, + deliveryState: delivery?.state ?? null, observedState, receiptHealth: runtimeHealth(receipts, now), receipts } +} diff --git a/src/biometrics/profiles/store.ts b/src/biometrics/profiles/store.ts new file mode 100644 index 000000000..6db2e5049 --- /dev/null +++ b/src/biometrics/profiles/store.ts @@ -0,0 +1,151 @@ +import { randomUUID } from 'node:crypto' +import { and, desc, eq, lt, or } from 'drizzle-orm' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import { z } from 'zod' +import * as schema from '@/src/db/schema' +import { biometricActivationEvents as events, biometricConfigOutbox as outbox, biometricProfileBindings as bindings, biometricProfileRevisions as revisions } from '@/src/db/schema' +import { stageClassifierSchema } from '@/src/lib/sleep-stage-rules' +import { biometricManifest, defaultConfig, legacyStageOverrides, manifestHash, resolveConfig, validateConfig, type Override } from '../algorithms/registry' +import { configHash } from '../algorithms/canonical' + +const sideSchema = z.enum(['left', 'right']) +const keySchema = z.string().min(1).max(200) +export const revisionInput = z.object({ + side: sideSchema, + parentRevisionId: z.string().uuid().optional(), + baseManifestHash: z.string().length(64).default(manifestHash), + overrides: z.array(z.object({ path: z.string().min(1).max(200), value: z.unknown() }).strict()).max(64), + origin: z.enum(['manual', 'migration', 'agent', 'system']), + requester: z.string().min(1).max(200), + idempotencyKey: keySchema, +}).strict() + +type Side = z.infer +type ConfigDB = BetterSQLite3Database +type Revision = typeof revisions.$inferSelect + +/** + * Configuration-only repository. No biometrics DB, numerical work or hardware + * access is allowed here. Synchronous SQLite transactions protect revisions, + * compare-and-swap bindings, audit events and outbox publication together. + */ +export class ProfileStore { + constructor(private readonly db: ConfigDB, private readonly now = Date.now) {} + + createRevision(raw: z.input): Revision { + const input = revisionInput.parse(raw) + // tRPC/SuperJSON can preserve an explicitly undefined optional field. + // Missing parent means a new family and has one canonical request identity. + if (input.parentRevisionId === undefined) delete input.parentRevisionId + const requestHash = configHash(input) + return this.db.transaction((tx) => { + const existing = tx.select().from(revisions).where(eq(revisions.idempotencyKey, input.idempotencyKey)).get() + if (existing) { + if (existing.requestHash !== requestHash) throw new Error('IDEMPOTENCY_CONFLICT: revision request changed') + return existing + } + const parent = input.parentRevisionId + ? tx.select().from(revisions).where(eq(revisions.id, input.parentRevisionId)).get() + : undefined + if (input.parentRevisionId && !parent) throw new Error('NOT_FOUND: parent revision') + if (parent && parent.side !== input.side) throw new Error('PROFILE_INCOMPATIBLE: parent belongs to the other side') + const familyId = parent?.familyId ?? randomUUID() + const latest = tx.select().from(revisions).where(eq(revisions.familyId, familyId)).orderBy(desc(revisions.revision)).get() + const resolved = resolveConfig(input.overrides, input.baseManifestHash) + const row: typeof revisions.$inferInsert = { + id: randomUUID(), familyId, revision: (latest?.revision ?? 0) + 1, + side: input.side, parentRevisionId: parent?.id ?? null, + baseManifestHash: manifestHash, baseManifest: biometricManifest, + overrides: resolved.overrides, resolved: resolved.resolved, resolvedHash: resolved.hash, + dependencyHashes: biometricManifest.dependencyHashes, + origin: input.origin, requester: input.requester, + idempotencyKey: input.idempotencyKey, requestHash, createdAtMs: this.now(), + } + return tx.insert(revisions).values(row).returning().get() + }, { behavior: 'immediate' }) + } + + getRevision(id: string): Revision { + const revision = this.db.select().from(revisions).where(eq(revisions.id, id)).get() + if (!revision) throw new Error('NOT_FOUND: profile revision') + return revision + } + + listRevisions(side: Side, cursor?: { createdAtMs: number, id: string }, familyId?: string) { + const rows = this.db.select().from(revisions).where(and( + eq(revisions.side, side), + familyId ? eq(revisions.familyId, familyId) : undefined, + cursor + ? or(lt(revisions.createdAtMs, cursor.createdAtMs), + and(eq(revisions.createdAtMs, cursor.createdAtMs), lt(revisions.id, cursor.id))) + : undefined, + )).orderBy(desc(revisions.createdAtMs), desc(revisions.id)).limit(101).all() + const items = rows.slice(0, 100) + const last = items.at(-1) + return { items, nextCursor: rows.length > 100 && last ? { createdAtMs: last.createdAtMs, id: last.id } : null } + } + + validateRevision(id: string, side: Side) { + const revision = this.getRevision(id) + if (revision.side !== side) throw new Error('PROFILE_INCOMPATIBLE: revision belongs to the other side') + if (revision.baseManifestHash !== manifestHash || configHash(revision.baseManifest) !== manifestHash) { + throw new Error('PROFILE_INCOMPATIBLE: pinned engine is not available in this build') + } + const resolved = validateConfig(revision.resolved) + const rebuilt = resolveConfig(revision.overrides as Override[], revision.baseManifestHash) + if (configHash(resolved) !== revision.resolvedHash || rebuilt.hash !== revision.resolvedHash + || configHash(revision.dependencyHashes) !== configHash(biometricManifest.dependencyHashes)) { + throw new Error('PROFILE_INCOMPATIBLE: immutable snapshot integrity check failed') + } + return { revision, resolved, hash: revision.resolvedHash } + } + + binding(side: Side) { + return this.db.select().from(bindings).where(eq(bindings.side, side)).get() + ?? { side, generation: 0, desiredRevisionId: null, previousRevisionId: null, updatedAtMs: 0 } + } + + /** Desired state only. Actual runtime state must come from component receipts. */ + effective(side: Side) { + const binding = this.binding(side) + if (!binding.desiredRevisionId) return { binding, resolved: defaultConfig, hash: configHash(defaultConfig), isDefault: true, warning: null } + try { + const selected = this.validateRevision(binding.desiredRevisionId, side) + return { binding, resolved: selected.resolved, hash: selected.hash, isDefault: false, warning: null } + } + catch (error) { + return { binding, resolved: defaultConfig, hash: configHash(defaultConfig), isDefault: true, + warning: error instanceof Error ? error.message : 'PROFILE_INCOMPATIBLE' } + } + } + + /** + * Legacy migration is idempotent and never overwrites an existing binding. + * Copies all rule leaves, not only differences, so future defaults cannot + * silently change the meaning of an old manual customization. + */ + migrateLegacy(side: Side) { + return this.db.transaction((tx) => { + const existing = tx.select().from(bindings).where(eq(bindings.side, side)).get() + if (existing) return existing + const stored = tx.select({ classifier: schema.sideSettings.stageClassifier }).from(schema.sideSettings) + .where(eq(schema.sideSettings.side, side)).get()?.classifier + if (stored == null) return null // no binding means follow defaults + const classifier = stageClassifierSchema.parse(stored) + const revision = this.createRevision({ side, overrides: legacyStageOverrides(classifier), origin: 'migration', + requester: 'legacy-stage-settings', idempotencyKey: `legacy-stage:${side}:${configHash(classifier)}` }) + const now = this.now() + tx.insert(bindings).values({ side, desiredRevisionId: revision.id, generation: 1, updatedAtMs: now }).run() + tx.insert(events).values({ id: randomUUID(), side, generation: 1, action: 'migration', + previousHash: configHash(defaultConfig), nextHash: revision.resolvedHash, + nextRevisionId: revision.id, evidence: { status: 'legacy_unvalidated' }, requester: 'legacy-stage-settings', + state: 'pending_delivery', idempotencyKey: `legacy-stage:${side}`, requestHash: configHash(classifier), + createdAtMs: now, updatedAtMs: now }).run() + tx.insert(outbox).values({ id: randomUUID(), side, generation: 1, + payload: { schemaVersion: 1, side, generation: 1, revisionId: revision.id, manifestHash, + resolved: revision.resolved, resolvedHash: revision.resolvedHash, boundary: 'immediate' }, + resolvedHash: revision.resolvedHash, state: 'pending', nextAttemptAtMs: now, createdAtMs: now }).run() + return this.binding(side) + }, { behavior: 'immediate' }) + } +} diff --git a/src/biometrics/profiles/tests/activation.test.ts b/src/biometrics/profiles/tests/activation.test.ts new file mode 100644 index 000000000..ec3f76790 --- /dev/null +++ b/src/biometrics/profiles/tests/activation.test.ts @@ -0,0 +1,110 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import * as schema from '@/src/db/schema' +import { ProfileStore } from '../store' +import { requestManualActivation } from '../activation' +import { configHash } from '../../algorithms/canonical' +import { defaultConfig } from '../../algorithms/registry' +let sqlite: Database.Database +let db: ReturnType> +let profiles: ProfileStore +beforeEach(() => { + sqlite = new Database(':memory:') + db = drizzle(sqlite, { schema }) + migrate(db, { migrationsFolder: 'src/db/migrations' }) + profiles = new ProfileStore(db) +}) +afterEach(() => sqlite.close()) +const candidate = () => profiles.createRevision({ side: 'left', overrides: [{ path: 'heart_rate.trackingToleranceBpm', value: 20 }], origin: 'manual', requester: 'fixture', idempotencyKey: 'candidate' }) +const request = () => { + const revision = candidate() + return { side: 'left' as const, action: 'activate' as const, revisionId: revision.id, expectedGeneration: 0, + expectedResolvedHash: revision.resolvedHash, boundary: 'next_session' as const, requester: 'fixture', idempotencyKey: 'activation', allowUnvalidated: true as const } +} +describe('atomic activation intent', () => { + it('writes one binding, audit event and immutable outbox payload, without claiming adoption', () => { + const input = request() + const event = requestManualActivation(db, input) + expect(event).toMatchObject({ generation: 1, state: 'pending_delivery', nextHash: input.expectedResolvedHash }) + expect(requestManualActivation(db, input)).toEqual(event) + expect(profiles.binding('left')).toMatchObject({ generation: 1, desiredRevisionId: input.revisionId, previousRevisionId: null }) + expect(profiles.binding('right').generation).toBe(0) + expect(db.select().from(schema.biometricConfigOutbox).all()).toHaveLength(1) + expect(() => requestManualActivation(db, { ...input, idempotencyKey: 'stale' })).toThrow('Binding changed') + }) + it('distinguishes rollback from reset and verifies exact target hashes', () => { + const input = request() + requestManualActivation(db, input) + expect(() => requestManualActivation(db, { ...input, action: 'rollback', expectedGeneration: 1, idempotencyKey: 'wrong' })).toThrow('previous binding') + const rollback = requestManualActivation(db, { ...input, action: 'rollback', revisionId: null, expectedGeneration: 1, + expectedResolvedHash: configHash(defaultConfig), idempotencyKey: 'rollback' }) + expect(rollback).toMatchObject({ action: 'rollback', generation: 2, nextRevisionId: null }) + const reset = requestManualActivation(db, { ...input, action: 'reset', revisionId: null, expectedGeneration: 2, + expectedResolvedHash: configHash(defaultConfig), idempotencyKey: 'reset' }) + expect(reset).toMatchObject({ action: 'reset', generation: 3 }) + }) + it('rolls back all intent writes when outbox insertion fails', () => { + const input = request() + sqlite.exec('CREATE TRIGGER reject_outbox BEFORE INSERT ON biometric_config_outbox BEGIN SELECT RAISE(ABORT, \'fixture failure\'); END;') + expect(() => requestManualActivation(db, input)).toThrow('fixture failure') + expect(profiles.binding('left').generation).toBe(0) + expect(db.select().from(schema.biometricActivationEvents).all()).toEqual([]) + }) + it('requires legacy personalization to be preserved before a new binding can replace it', () => { + db.insert(schema.sideSettings).values({ side: 'left', name: 'Left', stageClassifier: defaultConfig.sleep_stages }).run() + expect(() => requestManualActivation(db, request())).toThrow('Preserve the existing stage profile') + expect(profiles.binding('left').generation).toBe(0) + expect(() => requestManualActivation(db, { ...request(), action: 'reset', revisionId: null, expectedResolvedHash: configHash(defaultConfig) })).toThrow('Preserve the existing stage profile') + profiles.migrateLegacy('left') + const input = request() + expect(requestManualActivation(db, { ...input, expectedGeneration: 1 }).generation).toBe(2) + }) + + it('rejects incorrect hashes and changed idempotency payloads', () => { + const input = request() + expect(() => requestManualActivation(db, { ...input, expectedResolvedHash: '0'.repeat(64) })).toThrow('Resolved configuration changed') + requestManualActivation(db, input) + expect(() => requestManualActivation(db, { ...input, boundary: 'immediate' })).toThrow('idempotency key changed') + }) +}) + +it('allows explicit reset of malformed legacy rules while preserving the original as audit evidence', () => { + const malformed = { engine: 'unknown-engine', ruleset: { cannotMigrate: true } } + db.insert(schema.sideSettings).values({ side: 'left', name: 'Left', stageClassifier: malformed as unknown as typeof defaultConfig.sleep_stages }).run() + expect(() => profiles.migrateLegacy('left')).toThrow() + const input = { ...request(), action: 'reset' as const, revisionId: null, expectedResolvedHash: configHash(defaultConfig), idempotencyKey: 'recover-invalid-legacy' } + expect(() => requestManualActivation(db, { ...input, expectedResolvedHash: '0'.repeat(64) })).toThrow('Resolved configuration changed') + expect(profiles.binding('left').generation).toBe(0) + const event = requestManualActivation(db, input) + expect(event).toMatchObject({ action: 'reset', generation: 1, state: 'pending_delivery', nextRevisionId: null, + evidence: { status: 'manual_unvalidated', recovery: 'invalid_legacy_reset', preservedLegacyClassifier: malformed } }) + expect(db.select().from(schema.sideSettings).get()?.stageClassifier).toEqual(malformed) + expect(requestManualActivation(db, input).id).toBe(event.id) + expect(profiles.migrateLegacy('left')).toMatchObject({ generation: 1, desiredRevisionId: null }) +}) + +it('does not use malformed legacy recovery to bypass preservation or atomic publication', () => { + db.insert(schema.sideSettings).values({ side: 'left', name: 'Left', stageClassifier: { broken: true } as unknown as typeof defaultConfig.sleep_stages }).run() + expect(() => requestManualActivation(db, request())).toThrow('Preserve the existing stage profile') + sqlite.exec('CREATE TRIGGER fail_reset_outbox BEFORE INSERT ON biometric_config_outbox BEGIN SELECT RAISE(ABORT, \'outbox failure\'); END;') + expect(() => requestManualActivation(db, { ...request(), action: 'reset', revisionId: null, expectedResolvedHash: configHash(defaultConfig) })).toThrow('outbox failure') + expect(profiles.binding('left').generation).toBe(0) + expect(db.select().from(schema.biometricActivationEvents).all()).toEqual([]) + expect(db.select().from(schema.sideSettings).get()?.stageClassifier).toEqual({ broken: true }) +}) + +it('resets an unavailable pinned engine without deleting or relabeling its previous revision', () => { + const input = request() + requestManualActivation(db, input) + sqlite.prepare('UPDATE biometric_profile_revisions SET base_manifest_hash = ? WHERE id = ?').run('old-engine', input.revisionId) + expect(profiles.effective('left').warning).toContain('PROFILE_INCOMPATIBLE') + const event = requestManualActivation(db, { ...input, action: 'reset', revisionId: null, expectedGeneration: 1, expectedResolvedHash: configHash(defaultConfig), idempotencyKey: 'reset-unavailable' }) + expect(event).toMatchObject({ generation: 2, nextRevisionId: null, previousRevisionId: input.revisionId }) + expect(profiles.binding('left').previousRevisionId).toBe(input.revisionId) + expect(profiles.getRevision(input.revisionId).baseManifestHash).toBe('old-engine') + expect(() => requestManualActivation(db, { ...input, action: 'rollback', expectedGeneration: 2, idempotencyKey: 'rollback-unavailable' })).toThrow('PROFILE_INCOMPATIBLE') + expect(profiles.binding('left').generation).toBe(2) +}) diff --git a/src/biometrics/profiles/tests/evaluation-assessment.test.ts b/src/biometrics/profiles/tests/evaluation-assessment.test.ts new file mode 100644 index 000000000..22d8170b2 --- /dev/null +++ b/src/biometrics/profiles/tests/evaluation-assessment.test.ts @@ -0,0 +1,174 @@ +import { replayContextFixture } from './fixtures/replay-context' +// @vitest-environment node +import { expect, it } from 'vitest' +import { verifyEvaluationAssessments } from '../evaluation-assessment' +import { evaluationFixture as fixture } from './fixtures/evaluation' +import { backtestFixture } from '../../backtest/tests/fixtures/backtest' +import { assessNumericObjective, type ObjectivePolicy, type ObjectiveEvidence } from '../../backtest/objective-policy' +import { configHash, dataHash } from '../../algorithms/canonical' +import { defaultConfig, manifestHash } from '../../algorithms/registry' + +it('reassesses exact retained evidence under a trusted policy and returns a bounded audit snapshot', () => { + const work = fixture() + expect(work.report.stageObjectiveAssessments[0].status).toBe('insufficient_evidence') + const verified = verifyEvaluationAssessments(work.job, work.result(), work.expected, work.designations) + expect(verified).toMatchObject({ version: 'verified-evaluation-v1', reportHash: work.result().contentHash, + changedDomains: ['sleep_stages'], numericAssessments: [], stageAssessments: [{ baseline: 'default', status: 'improved' }, { baseline: 'current', status: 'improved' }] }) + expect(verified).not.toHaveProperty('nights') + expect(verified.policies).toEqual({ numeric: null, stages: work.designations.stages }) +}) + +it('cannot promote a client-selected policy or trust claimed successful assessments', () => { + const work = fixture() + expect(() => verifyEvaluationAssessments(work.job, work.result(), work.expected, { protocolHash: null, waveformContext: null, numeric: null, stages: null })).toThrow() + work.report.stageObjectiveAssessments[0].status = 'improved' + work.report.stageObjectiveAssessments[0].evidence.nights[0].candidateOutputCoverage = 0.5 + expect(() => verifyEvaluationAssessments(work.job, work.result(), work.expected, work.designations)).toThrow() +}) + +it('requires both baseline comparisons and exact cohort membership even when scores pass', () => { + for (const change of [ + (work: ReturnType) => { work.report.stageObjectiveAssessments.pop() }, + (work: ReturnType) => { work.report.stageObjectiveAssessments[0].baselineHash = '0'.repeat(64) }, + (work: ReturnType) => { work.report.stageObjectiveAssessments[0].candidateHash = '0'.repeat(64) }, + (work: ReturnType) => { work.report.stageObjectiveAssessments[0].evidence.nights[0].cohortId = 'other-night' }, + ]) { + const work = fixture() + change(work) + expect(() => verifyEvaluationAssessments(work.job, work.result(), work.expected, work.designations)).toThrow() + } + const incomplete = fixture() + incomplete.report.nights.push({ cohortId: 'd' }) + incomplete.expected.dataset.holdoutCohortIds.push('d') + expect(() => verifyEvaluationAssessments(incomplete.job, incomplete.result(), incomplete.expected, incomplete.designations)).toThrow('omits frozen') +}) + +it('requires an actual configuration change and a frozen stage policy', () => { + const work = fixture() + work.job.configs[2].resolved = defaultConfig + work.job.configs[2].resolvedHash = configHash(defaultConfig) + work.expected.resolvedHash = configHash(defaultConfig) + expect(() => verifyEvaluationAssessments(work.job, work.result(), work.expected, work.designations)).toThrow('does not change') + const missing = fixture() + const protocol = backtestFixture().job.protocol + const job = { ...missing.job, protocol, protocolHash: configHash(protocol) } + const report = { ...missing.report, protocol, protocolHash: job.protocolHash } + expect(() => verifyEvaluationAssessments(job, { ...missing.result(), report, contentHash: dataHash(report) }, missing.expected, missing.designations)).toThrow('require stage objective') +}) + +it('rejects missing supervisor measurements and no material primary improvement', () => { + const work = fixture() + work.report.jobOperations.memory.allSamplesAvailable = false + expect(() => verifyEvaluationAssessments(work.job, work.result(), work.expected, work.designations)).toThrow() + work.report.jobOperations.memory.allSamplesAvailable = true + work.report.stageObjectiveAssessments[1].evidence.intervalLower = 0 + expect(() => verifyEvaluationAssessments(work.job, work.result(), work.expected, work.designations)).toThrow('does not support') +}) + +it('does not let stage agreement validate a simultaneous numeric parameter change', () => { + const work = fixture() + const candidate = work.job.configs[2] + candidate.resolved.heart_rate.trackingToleranceBpm = 20 + candidate.resolvedHash = configHash(candidate.resolved) + work.expected.resolvedHash = candidate.resolvedHash + expect(() => verifyEvaluationAssessments(work.job, work.result(), work.expected, work.designations)).toThrow('numeric component') +}) + +it('requires numeric improvement while allowing unchanged downstream stages, and rejects an unassessed second metric', () => { + const work = fixture() + const policy: ObjectivePolicy = { version: 'numeric-objective-v1', qualificationId: work.designations.stages.qualificationId, + metric: 'heart_rate', minimumNights: 3, minimumPairedReferencesPerNight: 1, maximumLostObservationFraction: 0.1, + minimumMaeImprovement: 2, maximumNightMaeRegression: 1, maximumRegressedNightFraction: 0.34, + maximumSubgroupMaeRegression: 1, requiredSubgroups: [], operational: work.designations.stages.operational } + const candidate = structuredClone(defaultConfig) + candidate.heart_rate.trackingToleranceBpm = 20 + work.job.configs[2].resolved = candidate + work.job.configs[2].resolvedHash = configHash(candidate) + const protocol = { ...work.job.protocol, mode: 'signal_replay' as const, waveformStages: true, objectivePolicy: policy } + const pythonBuildHash = 'a'.repeat(64), otherSideConfigHash = configHash(defaultConfig) + const job = { ...work.job, protocol, protocolHash: configHash(protocol), waveform: { pythonBuildHash, + otherSide: { resolved: defaultConfig, resolvedHash: otherSideConfigHash, manifestHash } } } + const evidence: ObjectiveEvidence = { metricCompatible: true, split: 'holdout', holdoutUse: 1, intervalUpper: -2, subgroups: [], operational: null, + nights: ['a', 'b', 'c'].map(cohortId => ({ cohortId, pairedReferences: 10, maeDifference: -2, baselineObservationCount: 10, lostObservations: 0 })) } + const report = { ...work.report, protocol, protocolHash: job.protocolHash, version: 'waveform-backtest-report-v1', + support: 'identical_retained_waveform_inputs', waveform: { pythonBuildHash, otherSideConfigHash, + ...replayContextFixture(['a', 'b', 'c'], job.configs.map(row => row.resolvedHash), pythonBuildHash, otherSideConfigHash) }, + objectiveAssessments: ['default', 'current'].map(baseline => ({ baseline, candidate: 'candidate', baselineHash: configHash(defaultConfig), + candidateHash: configHash(candidate), ...assessNumericObjective(policy, structuredClone(evidence), null) })) } + for (const row of report.stageObjectiveAssessments) { + row.candidateHash = configHash(candidate) + row.evidence.intervalLower = 0 + for (const night of row.evidence.nights) night.agreementDifference = 0 + } + const result = () => ({ report, contentHash: dataHash(report), available: true, currentHoldoutUseCount: 1 }) + const expected = { ...work.expected, resolvedHash: configHash(candidate), waveform: { pythonBuildHash, otherSideConfigHash } } + const designations = { ...work.designations, protocolHash: job.protocolHash, waveformContext: { allowedLimitations: ['event_time_replaces_arrival_time' as const] }, numeric: policy } + expect(verifyEvaluationAssessments(job, result(), expected, designations)).toMatchObject({ changedDomains: ['heart_rate'], + numericAssessments: [{ status: 'improved' }, { status: 'improved' }], stageAssessments: [{ status: 'no_material_change' }, { status: 'no_material_change' }] }) + expect(verifyEvaluationAssessments(job, result(), expected, designations)).toMatchObject({ policies: { numeric: policy }, waveform: expected.waveform }) + report.objectiveAssessments[1].evidence.intervalUpper = 0 + expect(() => verifyEvaluationAssessments(job, result(), expected, designations)).toThrow('does not support') + candidate.breathing_rate.intervalLowerRatio = 0.6 + job.configs[2].resolvedHash = configHash(candidate) + expected.resolvedHash = configHash(candidate) + expect(() => verifyEvaluationAssessments(job, result(), expected, designations)).toThrow('numeric component') + + const breathingPolicy: ObjectivePolicy = { ...policy, metric: 'breathing_rate' } + const rest = { ...work.job.protocol, mode: protocol.mode, waveformStages: true } + const multiProtocol = { ...rest, objectivePolicies: [policy, breathingPolicy] } + const multiJob = { ...job, protocol: multiProtocol, protocolHash: configHash(multiProtocol) } + const multiReport = { ...report, protocol: multiProtocol, protocolHash: multiJob.protocolHash, + waveform: { pythonBuildHash, otherSideConfigHash, ...replayContextFixture(['a', 'b', 'c'], multiJob.configs.map(row => row.resolvedHash), pythonBuildHash, otherSideConfigHash) }, + stageObjectiveAssessments: report.stageObjectiveAssessments.map(row => ({ ...row, candidateHash: configHash(candidate) })), + objectiveAssessments: multiProtocol.objectivePolicies.flatMap(item => ['default', 'current'].map(baseline => ({ + baseline, candidate: 'candidate', baselineHash: configHash(defaultConfig), candidateHash: configHash(candidate), + ...assessNumericObjective(item, structuredClone(evidence), null), + }))) } + const multiResult = () => ({ ...result(), report: multiReport, contentHash: dataHash(multiReport) }) + const qualified = { ...designations, protocolHash: multiJob.protocolHash, numeric: multiProtocol.objectivePolicies } + expect(verifyEvaluationAssessments(multiJob, multiResult(), expected, qualified)).toMatchObject({ + changedDomains: ['heart_rate', 'breathing_rate'], numericAssessments: [ + { metric: 'heart_rate', status: 'improved' }, { metric: 'heart_rate', status: 'improved' }, + { metric: 'breathing_rate', status: 'improved' }, { metric: 'breathing_rate', status: 'improved' }, + ], + }) + expect(() => verifyEvaluationAssessments(multiJob, multiResult(), expected, designations)).toThrow() + multiReport.objectiveAssessments[3].evidence.nights.pop() + expect(() => verifyEvaluationAssessments(multiJob, multiResult(), expected, qualified)).toThrow() +}) + +it('does not transfer threshold qualification to changed alignment, calibration or uncertainty settings', () => { + for (const patch of [ + { clockOffsetMs: 1000 }, + { pointToleranceMs: 1000 }, + { assumedStageCalibrationQuality: 0.75 }, + { nightUncertainty: { version: 'paired-night-percentile-v1' as const, seed: 'changed-seed', resamples: 200, minimumNights: 3 } }, + ]) { + const work = fixture() + const protocol = { ...work.job.protocol, ...patch } + const job = { ...work.job, protocol, protocolHash: configHash(protocol) } + const report = { ...work.report, protocol, protocolHash: job.protocolHash } + const result = { ...work.result(), report, contentHash: dataHash(report) } + expect(() => verifyEvaluationAssessments(job, result, work.expected, work.designations)).toThrow('qualified context') + expect(verifyEvaluationAssessments(job, result, work.expected, { ...work.designations, protocolHash: job.protocolHash })) + .toMatchObject({ protocolHash: job.protocolHash }) + } + const work = fixture() + expect(() => verifyEvaluationAssessments(work.job, work.result(), work.expected, { ...work.designations, protocolHash: null })) + .toThrow('qualified context') +}) + +it('derives aggregate status from reassessed evidence without accepting a claimed report status', async () => { + const { assessEvaluation } = await import('../evaluation-assessment') + const work = fixture() + expect(assessEvaluation(work.job, work.result(), work.expected, work.designations).status).toBe('improved') + work.report.stageObjectiveAssessments[0].status = 'improved' + work.report.stageObjectiveAssessments[0].evidence.intervalLower = 0 + for (const night of work.report.stageObjectiveAssessments[0].evidence.nights) night.agreementDifference = 0 + expect(assessEvaluation(work.job, work.result(), work.expected, work.designations).status).toBe('no_material_change') + work.report.stageObjectiveAssessments[0].evidence.nights[0].candidateOutputCoverage = 0.1 + expect(assessEvaluation(work.job, work.result(), work.expected, work.designations).status).toBe('regressed') + work.report.jobOperations.memory.allSamplesAvailable = false + expect(assessEvaluation(work.job, work.result(), work.expected, work.designations).status).toBe('insufficient_evidence') + expect(() => verifyEvaluationAssessments(work.job, work.result(), work.expected, work.designations)).toThrow('does not support') +}) diff --git a/src/biometrics/profiles/tests/evaluation-identity.test.ts b/src/biometrics/profiles/tests/evaluation-identity.test.ts new file mode 100644 index 000000000..f3c458c85 --- /dev/null +++ b/src/biometrics/profiles/tests/evaluation-identity.test.ts @@ -0,0 +1,135 @@ +// @vitest-environment node +import { expect, it } from 'vitest' +import { verifyEvaluationIdentity, type EvaluationIdentityExpectation } from '../evaluation-identity' +import { backtestFixture } from '../../backtest/tests/fixtures/backtest' +import { configHash, dataHash } from '../../algorithms/canonical' +import { defaultConfig, manifestHash } from '../../algorithms/registry' + +function fixture() { + const source = backtestFixture() + const candidate = { ...defaultConfig, heart_rate: { ...defaultConfig.heart_rate, trackingToleranceBpm: 20 } } + const revisionId = '33333333-3333-4333-8333-333333333333' + const job = { ...source.job, split: 'holdout' as const, holdoutUse: 1, configs: [...source.job.configs, + { label: 'candidate' as const, revisionId, resolved: candidate, resolvedHash: configHash(candidate), manifestHash }] } + const report = { version: 'processed-backtest-report-v1', side: 'left', datasetId: job.datasetId, datasetHash: job.datasetHash, + protocol: job.protocol, protocolHash: job.protocolHash, evaluatorBuildHash: job.evaluatorBuildHash, manifestHash, + split: 'holdout', holdoutUse: 1, support: 'identical_retained_epoch_intervals', nights: [{ cohortId: 'night' }] } + const expected: EvaluationIdentityExpectation = { side: 'left', revisionId, resolvedHash: configHash(candidate), + currentResolvedHash: configHash(defaultConfig), evaluatorBuildHash: job.evaluatorBuildHash, + dataset: { id: job.datasetId, contentHash: job.datasetHash, side: 'left', available: true, holdoutCohortIds: ['night'] } } + const result = { report, contentHash: dataHash(report), available: true, currentHoldoutUseCount: 1 } + return { job, report, expected, result } +} + +it('binds retained report bytes to all three configurations without awarding activation', () => { + const work = fixture() + expect(verifyEvaluationIdentity(work.job, work.result, work.expected)).toMatchObject({ contentHash: work.result.contentHash, cohortIds: ['night'] }) + expect(verifyEvaluationIdentity(work.job, work.result, work.expected)).not.toHaveProperty('status') +}) + +it('rejects unavailable or changed artifacts and reused holdout evidence', () => { + for (const patch of [{ available: false }, { contentHash: '0'.repeat(64) }, { currentHoldoutUseCount: 2 }]) { + const work = fixture() + expect(() => verifyEvaluationIdentity(work.job, { ...work.result, ...patch }, work.expected)).toThrow() + } + const work = fixture() + expect(() => verifyEvaluationIdentity(work.job, work.result, { ...work.expected, dataset: { ...work.expected.dataset, available: false } })).toThrow('unavailable') + expect(() => verifyEvaluationIdentity({ ...work.job, split: 'validation' }, work.result, work.expected)).toThrow('holdout') + expect(() => verifyEvaluationIdentity({ ...work.job, holdoutUse: 2 }, work.result, work.expected)).toThrow('holdout') +}) + +it('rejects another revision, baseline, build, side, or frozen dataset even with matching report bytes', () => { + const work = fixture() + for (const patch of [{ revisionId: '44444444-4444-4444-8444-444444444444' }, { resolvedHash: '0'.repeat(64) }, + { currentResolvedHash: '0'.repeat(64) }, { evaluatorBuildHash: '0'.repeat(64) }, { side: 'right' as const }, + { dataset: { ...work.expected.dataset, contentHash: '0'.repeat(64) } }]) { + expect(() => verifyEvaluationIdentity(work.job, work.result, { ...work.expected, ...patch })).toThrow() + } +}) + +it('requires all exact frozen cohorts, a replay protocol, and matching protocol contents', () => { + const work = fixture() + for (const nights of [[], [{ cohortId: 'night' }, { cohortId: 'night' }], [{ cohortId: 'other' }]]) { + const report = { ...work.report, nights } + expect(() => verifyEvaluationIdentity(work.job, { ...work.result, report, contentHash: dataHash(report) }, work.expected)).toThrow() + } + const protocol = { ...work.job.protocol, clockOffsetMs: 1000 } + const report = { ...work.report, protocol } + expect(() => verifyEvaluationIdentity(work.job, { ...work.result, report, contentHash: dataHash(report) }, work.expected)).toThrow('protocol') + const comparison = { ...work.job.protocol, mode: 'comparison' as const } + const comparisonReport = { ...work.report, protocol: comparison, protocolHash: configHash(comparison) } + expect(() => verifyEvaluationIdentity({ ...work.job, protocol: comparison, protocolHash: configHash(comparison) }, + { ...work.result, report: comparisonReport, contentHash: dataHash(comparisonReport) }, work.expected)).toThrow('mode') +}) + +it('checks independently altered job and report identities, not only their content hashes', () => { + const work = fixture() + const otherId = '44444444-4444-4444-8444-444444444444' + for (const patch of [{ side: 'right' }, { datasetId: otherId }, { datasetHash: '0'.repeat(64) }, + { evaluatorBuildHash: '0'.repeat(64) }, { manifestHash: '0'.repeat(64) }, { protocolHash: '0'.repeat(64) }, + { support: 'identical_retained_waveform_inputs' }, { version: 'waveform-backtest-report-v1' }]) { + const report = { ...work.report, ...patch } + expect(() => verifyEvaluationIdentity(work.job, { ...work.result, report, contentHash: dataHash(report) }, work.expected)).toThrow() + } + for (const patch of [{ datasetId: otherId }, { datasetHash: '0'.repeat(64) }, { protocolHash: '0'.repeat(64) }, + { evidence: 'reused_holdout_is_validation' }]) { + expect(() => verifyEvaluationIdentity({ ...work.job, ...patch }, work.result, work.expected)).toThrow() + } + expect(() => verifyEvaluationIdentity(work.job, work.result, { ...work.expected, dataset: { ...work.expected.dataset, side: 'right' } })).toThrow('side changed') + for (const holdoutCohortIds of [['night', 'night'], ['night', 'missing']]) { + expect(() => verifyEvaluationIdentity(work.job, work.result, { ...work.expected, dataset: { ...work.expected.dataset, holdoutCohortIds } })).toThrow('exact frozen') + } +}) + +it('does not let mutually consistent replacements bypass the service expectations', () => { + const work = fixture() + const otherId = '44444444-4444-4444-8444-444444444444' + const report = { ...work.report, datasetId: otherId } + expect(() => verifyEvaluationIdentity({ ...work.job, datasetId: otherId }, + { ...work.result, report, contentHash: dataHash(report) }, work.expected)).toThrow('dataset') + expect(() => verifyEvaluationIdentity({ ...work.job, protocol: { ...work.job.protocol, clockOffsetMs: 1 } }, work.result, work.expected)).toThrow('protocol') + const configs = work.job.configs.map(row => row.label === 'current' ? { ...row, resolvedHash: '0'.repeat(64) } : row) + expect(() => verifyEvaluationIdentity({ ...work.job, configs }, work.result, { ...work.expected, currentResolvedHash: '0'.repeat(64) })).toThrow('configuration identity') + const expected = { ...work.expected, dataset: { ...work.expected.dataset, holdoutCohortIds: ['night', 'second'] } } + for (const cohortIds of [['night', 'other'], ['night', 'night']]) { + const altered = { ...work.report, nights: cohortIds.map(cohortId => ({ cohortId })) } + expect(() => verifyEvaluationIdentity(work.job, { ...work.result, report: altered, contentHash: dataHash(altered) }, expected)).toThrow('exact frozen') + } +}) + +it('rejects mislabeled defaults, duplicated baseline labels and changed resolved snapshots', () => { + const work = fixture() + const [baseline, current, candidate] = work.job.configs + for (const configs of [[baseline, candidate], [baseline, baseline, candidate], + [{ ...candidate, label: 'default', revisionId: null }, current, candidate], + [{ ...baseline, revisionId: candidate.revisionId }, current, candidate], + [{ ...baseline, manifestHash: '0'.repeat(64) }, current, candidate], + [{ ...baseline, resolvedHash: '0'.repeat(64) }, current, candidate]]) { + expect(() => verifyEvaluationIdentity({ ...work.job, configs }, work.result, work.expected)).toThrow() + } +}) + +it('pins waveform Python and cross-side dependencies separately from the scoring build', () => { + const work = fixture() + const protocol = { ...work.job.protocol, mode: 'signal_replay' as const } + const pythonBuildHash = 'a'.repeat(64), otherSideConfigHash = configHash(defaultConfig) + const job = { ...work.job, protocol, protocolHash: configHash(protocol), waveform: { pythonBuildHash, + otherSide: { resolved: defaultConfig, resolvedHash: otherSideConfigHash, manifestHash } } } + const report = { ...work.report, version: 'waveform-backtest-report-v1', support: 'identical_retained_waveform_inputs', + protocol, protocolHash: job.protocolHash, waveform: { pythonBuildHash, otherSideConfigHash } } + const result = { ...work.result, report, contentHash: dataHash(report) } + const expected = { ...work.expected, waveform: { pythonBuildHash, otherSideConfigHash } } + expect(verifyEvaluationIdentity(job, result, expected).contentHash).toBe(result.contentHash) + expect(() => verifyEvaluationIdentity(job, result, work.expected)).toThrow('Waveform') + for (const waveform of [{ ...expected.waveform, pythonBuildHash: 'b'.repeat(64) }, { ...expected.waveform, otherSideConfigHash: 'b'.repeat(64) }]) { + expect(() => verifyEvaluationIdentity(job, result, { ...expected, waveform })).toThrow('Waveform') + } + for (const waveform of [{ ...report.waveform, pythonBuildHash: 'b'.repeat(64) }, { ...report.waveform, otherSideConfigHash: 'b'.repeat(64) }]) { + const altered = { ...report, waveform } + expect(() => verifyEvaluationIdentity(job, { ...result, report: altered, contentHash: dataHash(altered) }, expected)).toThrow('Waveform') + } + for (const otherSide of [{ ...job.waveform.otherSide, manifestHash: 'b'.repeat(64) }, + { ...job.waveform.otherSide, resolved: work.job.configs[2].resolved }]) { + expect(() => verifyEvaluationIdentity({ ...job, waveform: { ...job.waveform, otherSide } }, result, expected)).toThrow('Waveform') + } +}) diff --git a/src/biometrics/profiles/tests/fixtures/evaluation.ts b/src/biometrics/profiles/tests/fixtures/evaluation.ts new file mode 100644 index 000000000..0276d872b --- /dev/null +++ b/src/biometrics/profiles/tests/fixtures/evaluation.ts @@ -0,0 +1,39 @@ +import type { EvaluationIdentityExpectation } from '../../evaluation-identity' +import { stageObjectiveTestPolicy } from '../../../backtest/tests/fixtures/stage-objective' +import { backtestFixture } from '../../../backtest/tests/fixtures/backtest' +import { assessStageObjective, type StageObjectiveEvidence } from '../../../backtest/stage-objective' +import { configHash, dataHash } from '../../../algorithms/canonical' +import { defaultConfig, manifestHash } from '../../../algorithms/registry' + +export function evaluationFixture() { + const source = backtestFixture() + const policy = { ...stageObjectiveTestPolicy, operational: { ...stageObjectiveTestPolicy.operational, + observations: { version: 'sampled-process-group-v1' as const, maximumMemoryGapMs: 1000, maximumIngestionGapMs: 5000 } } } + const candidate = structuredClone(defaultConfig) + candidate.sleep_stages.ruleset.smoothIsolatedEpochs = false + const revisionId = '33333333-3333-4333-8333-333333333333' + const protocol = { ...source.job.protocol, stageObjectivePolicy: policy } + const job = { ...source.job, protocol, protocolHash: configHash(protocol), split: 'holdout' as const, holdoutUse: 1, + configs: [...source.job.configs, { label: 'candidate' as const, revisionId, resolved: candidate, resolvedHash: configHash(candidate), manifestHash }] } + const evidence: StageObjectiveEvidence = { split: 'holdout', holdoutUse: 1, intervalLower: 0.125, operational: null, + nights: ['a', 'b', 'c'].map(cohortId => ({ cohortId, pairedDurationMs: 800, agreementDifference: 0.125, macroF1Difference: 0.125, + candidateOutputCoverage: 0.9, baselineOutputMs: 1000, commonOutputMs: 900, + perStage: policy.requiredStages.map(stage => ({ stage, referenceMs: 200, f1Difference: 0.125 })) })) } + const report = { version: 'processed-backtest-report-v1', side: 'left', datasetId: job.datasetId, datasetHash: job.datasetHash, + protocol, protocolHash: job.protocolHash, evaluatorBuildHash: job.evaluatorBuildHash, manifestHash, + split: 'holdout', holdoutUse: 1, support: 'identical_retained_epoch_intervals', nights: evidence.nights.map(row => ({ cohortId: row.cohortId })), + stageObjectiveAssessments: ['default', 'current'].map(baseline => ({ baseline, candidate: 'candidate', baselineHash: configHash(defaultConfig), + candidateHash: configHash(candidate), ...assessStageObjective(policy, structuredClone(evidence), null) })), + jobOperations: { version: 'backtest-job-operations-v1', wallMs: 1000, scope: 'admission_through_prepublication', + memory: { version: 'process-group-rss-observations-v1', scope: 'sum_rss_of_replay_process_group', attempts: 3, samples: 3, + allSamplesAvailable: true, peakObservedRssBytes: 1000, maximumObservationGapMs: 1000, intervalMs: 1000, exactKernelPeak: false }, + ingestion: { source: 'supervisor_live_piezo_receipt_age', checks: 2, samples: 2, allChecksMeasured: true, qualificationId: policy.qualificationId, + maximumObservedAgeMs: 1000, maximumObservationGapMs: 5000, nominalIntervalMs: 5000, continuousMeasurement: false } } } + const expected: EvaluationIdentityExpectation = { side: 'left', revisionId, resolvedHash: configHash(candidate), + currentResolvedHash: configHash(defaultConfig), evaluatorBuildHash: job.evaluatorBuildHash, + dataset: { id: job.datasetId, contentHash: job.datasetHash, side: 'left', available: true, holdoutCohortIds: ['a', 'b', 'c'] } } + const result = () => ({ report, contentHash: dataHash(report), available: true, currentHoldoutUseCount: 1 }) + const designations = { protocolHash: job.protocolHash, waveformContext: null, numeric: null, stages: policy } + return { job, report, result, expected, designations } +} + diff --git a/src/biometrics/profiles/tests/fixtures/replay-context.ts b/src/biometrics/profiles/tests/fixtures/replay-context.ts new file mode 100644 index 000000000..b7aeebd1f --- /dev/null +++ b/src/biometrics/profiles/tests/fixtures/replay-context.ts @@ -0,0 +1,11 @@ +import { dataHash } from '../../../algorithms/canonical' +export function replayContextFixture(cohortIds: string[], configHashes: string[], pythonBuildHash: string, otherSideConfigHash: string) { + const outputs = cohortIds.flatMap(cohortId => [...new Set(configHashes)].map(configHash => ({ cohortId, configHash, + readings: [{ side: 'left' as const, timestampMs: 1000, heartRate: 60, hrvIndex: 30, hrvMetric: 'window_ibi_variability_v1' as const, + breathingRate: 12, qualityScore: 1, flags: [], hrRaw: 60, sourceSide: 'left' as const, configHash, limitations: [] as string[] }], + run: { version: 'retained-waveform-run-v1' as const, build_hash: pythonBuildHash, decoded_records: 1, raw_records: 1, + counts: { records: 1, accepted_frames: 1, empty_channel_frames: 0, pump_gated_frames: 0, preroll_outputs: 0, scored_outputs: 1 }, + limitations: ['event_time_replaces_arrival_time'], config_hashes: { left: configHash, right: otherSideConfigHash }, artifact_hashes: ['d'.repeat(64)] }, + }))) + return { outputs, hash: dataHash(outputs) } +} diff --git a/src/biometrics/profiles/tests/health.test.ts b/src/biometrics/profiles/tests/health.test.ts new file mode 100644 index 000000000..a62c4d11a --- /dev/null +++ b/src/biometrics/profiles/tests/health.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from 'vitest' +import { runtimeHealth } from '../health' +const now = 1791673200000 +const receipts = () => ['heart_rate', 'breathing_rate', 'hrv_index'].map(component => ({ component, + generation: 2, appliedHash: 'bundle', state: 'applied', effectiveSampleTimeMs: now - 50000, heartbeatAtMs: now - 30000 })) +describe('continuous processor health is distinct from historical adoption', () => { + it('requires all three numerical consumers and ignores on-demand stage receipt age', () => { + expect(runtimeHealth([], now).state).toBe('unreported') + expect(runtimeHealth(receipts().slice(1), now).state).toBe('unreported') + expect(runtimeHealth([...receipts(), { ...receipts()[0], component: 'sleep_stages', heartbeatAtMs: 0 }], now).state).toBe('current') + }) + it('reports stopped processors, future clocks, failure and incomplete application', () => { + expect(runtimeHealth(receipts(), now + 120000).state).toBe('stale') + expect(runtimeHealth(receipts(), now - 60000).state).toBe('clock_skew') + expect(runtimeHealth(receipts().map(row => ({ ...row, state: 'failed' })), now).state).toBe('failed') + expect(runtimeHealth(receipts().map(row => ({ ...row, effectiveSampleTimeMs: null })), now).state).toBe('unreported') + }) + it('does not call different generations or hashes one coherent current profile', () => { + const rows = receipts() + rows[1].generation = 3 + expect(runtimeHealth(rows, now).state).toBe('mixed_generations') + rows[1].generation = 2 + rows[1].appliedHash = 'other' + expect(runtimeHealth(rows, now).state).toBe('mixed_generations') + rows[1] = { ...rows[0], component: 'breathing_rate' } + expect(runtimeHealth([{ ...rows[0], generation: 1, heartbeatAtMs: now }, ...rows], now).state).toBe('current') + }) +}) diff --git a/src/biometrics/profiles/tests/history.test.ts b/src/biometrics/profiles/tests/history.test.ts new file mode 100644 index 000000000..6a789e4a7 --- /dev/null +++ b/src/biometrics/profiles/tests/history.test.ts @@ -0,0 +1,31 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { expect, it } from 'vitest' +import * as schema from '@/src/db/schema' +import { requestManualActivation } from '../activation' +import { listActivationHistory } from '../history' +import { defaultConfig } from '../../algorithms/registry' +import { configHash } from '../../algorithms/canonical' +it('paginates immutable side-specific audit history without leaking internal request keys', () => { + const raw = new Database(':memory:') + try { + const db = drizzle(raw, { schema }) + migrate(db, { migrationsFolder: 'src/db/migrations' }) + for (let generation = 0; generation < 53; generation++) requestManualActivation(db, { + side: 'left', action: 'reset', revisionId: null, expectedGeneration: generation, expectedResolvedHash: configHash(defaultConfig), + boundary: 'immediate', requester: 'fixture', idempotencyKey: `request-${generation}`, allowUnvalidated: true, + }) + const first = listActivationHistory(db, 'left') + expect(first.items).toHaveLength(50) + expect(first.items[0].generation).toBe(53) + expect(first.nextGeneration).toBe(4) + expect(first.items[0]).not.toHaveProperty('idempotencyKey') + const second = listActivationHistory(db, 'left', first.nextGeneration ?? undefined) + expect(second.items.map(row => row.generation)).toEqual([3, 2, 1]) + expect(second.nextGeneration).toBeNull() + expect(listActivationHistory(db, 'right').items).toEqual([]) + } + finally { raw.close() } +}) diff --git a/src/biometrics/profiles/tests/provenance-migration.test.ts b/src/biometrics/profiles/tests/provenance-migration.test.ts new file mode 100644 index 000000000..027424e60 --- /dev/null +++ b/src/biometrics/profiles/tests/provenance-migration.test.ts @@ -0,0 +1,26 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { readMigrationFiles } from 'drizzle-orm/migrator' +import { expect, it } from 'vitest' + +it('preserves old vitals through additive migration and accepts old writers afterward', () => { + const sqlite = new Database(':memory:') + try { + const migrations = readMigrationFiles({ migrationsFolder: 'src/db/biometrics-migrations' }) + const provenanceIndex = migrations.findIndex(migration => migration.sql.some(statement => statement.includes('ADD `processing_run_id`'))) + expect(provenanceIndex).toBeGreaterThan(0) + for (const migration of migrations.slice(0, provenanceIndex)) { + for (const statement of migration.sql) sqlite.exec(statement) + } + const insert = 'INSERT INTO vitals(side, timestamp, heart_rate, hrv, breathing_rate) VALUES (?, ?, ?, ?, ?)' + sqlite.prepare(insert).run('left', 1700000000, 60, 12, 15) + for (const migration of migrations.slice(provenanceIndex)) { + for (const statement of migration.sql) sqlite.exec(statement) + } + expect(sqlite.prepare('SELECT heart_rate, hrv, processing_run_id FROM vitals').get()) + .toEqual({ heart_rate: 60, hrv: 12, processing_run_id: null }) + sqlite.prepare(insert).run('right', 1700000060, null, null, 16) + expect(sqlite.prepare('SELECT count(*) AS total FROM vitals WHERE processing_run_id IS NULL').get()).toEqual({ total: 2 }) + } + finally { sqlite.close() } +}) diff --git a/src/biometrics/profiles/tests/qualification-registry.test.ts b/src/biometrics/profiles/tests/qualification-registry.test.ts new file mode 100644 index 000000000..dc2434b9e --- /dev/null +++ b/src/biometrics/profiles/tests/qualification-registry.test.ts @@ -0,0 +1,62 @@ +import { configHash } from '../../algorithms/canonical' +// @vitest-environment node +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { afterEach, beforeEach, expect, it } from 'vitest' +import { installedPromotionPolicies, unqualifiedPolicies } from '../qualification-registry' +import { evaluationFixture } from './fixtures/evaluation' +import { manifestHash } from '../../algorithms/registry' +let root: string +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'promotion-registry-')) +}) +afterEach(async () => { + await rm(root, { recursive: true, force: true }) +}) +const fixture = () => { + const work = evaluationFixture() + const entry = { qualificationId: work.designations.stages.qualificationId, manifestHash, evaluatorBuildHash: work.job.evaluatorBuildHash, + protocol: work.job.protocol, fixtureEvidenceHash: 'a'.repeat(64), podEvidenceHash: 'b'.repeat(64), waveformContext: null } + return { entry, identity: { evaluatorBuildHash: entry.evaluatorBuildHash, protocolHash: work.job.protocolHash } } +} +const install = (entries: unknown[]) => writeFile(join(root, 'promotion-qualifications.json'), JSON.stringify({ version: 'biometric-promotion-qualifications-v1', entries })) +it('requires a unique installed build/protocol qualification and returns its policies', async () => { + const { entry, identity } = fixture() + const path = join(root, 'worker.cjs') + expect(await installedPromotionPolicies(path, identity)).toEqual(unqualifiedPolicies()) + await install([entry]) + expect(await installedPromotionPolicies(path, identity)).toEqual({ protocolHash: identity.protocolHash, numeric: [], stages: entry.protocol.stageObjectivePolicy, waveformContext: null }) + for (const changed of [{ evaluatorBuildHash: 'f'.repeat(63) + '0' }, { protocolHash: 'c'.repeat(64) }, { pythonBuildHash: 'd'.repeat(64) }]) { + expect(await installedPromotionPolicies(path, { ...identity, ...changed })).toEqual(unqualifiedPolicies()) + } + await install([entry, entry]) + expect(await installedPromotionPolicies(path, identity)).toEqual(unqualifiedPolicies()) +}) +it('does not use malformed, cross-manifest, inconsistent or unbounded qualification files', async () => { + const { entry, identity } = fixture() + const path = join(root, 'worker.cjs') + for (const changed of [{ ...entry, manifestHash: '0'.repeat(64) }, { ...entry, qualificationId: 'unrelated' }, + { ...entry, podEvidenceHash: undefined }, { ...entry, waveformContext: { allowedLimitations: [] } }]) { + await install([changed]) + expect(await installedPromotionPolicies(path, identity)).toEqual(unqualifiedPolicies()) + } + for (const data of ['{', ' '.repeat(256 * 1024 + 1)]) { + await writeFile(join(root, 'promotion-qualifications.json'), data) + expect(await installedPromotionPolicies(path, identity)).toEqual(unqualifiedPolicies()) + } +}) + +it('requires waveform context qualification and the exact Python build for waveform policies', async () => { + const { entry } = fixture() + const protocol = { ...entry.protocol, mode: 'signal_replay' as const, waveformStages: true } + const pythonBuildHash = 'c'.repeat(64) + const identity = { evaluatorBuildHash: entry.evaluatorBuildHash, pythonBuildHash, protocolHash: configHash(protocol) } + const qualified = { ...entry, protocol, pythonBuildHash, waveformContext: { allowedLimitations: ['cold_start_without_serialized_state'] } } + await install([qualified]) + expect(await installedPromotionPolicies(join(root, 'worker.cjs'), identity)).toMatchObject({ + protocolHash: identity.protocolHash, waveformContext: qualified.waveformContext, + }) + await install([{ ...qualified, waveformContext: null }]) + expect(await installedPromotionPolicies(join(root, 'worker.cjs'), identity)).toEqual(unqualifiedPolicies()) +}) diff --git a/src/biometrics/profiles/tests/recovery.test.ts b/src/biometrics/profiles/tests/recovery.test.ts new file mode 100644 index 000000000..ebdfe2c99 --- /dev/null +++ b/src/biometrics/profiles/tests/recovery.test.ts @@ -0,0 +1,125 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import * as configSchema from '@/src/db/schema' +import * as bioSchema from '@/src/db/biometrics-schema' +import { configHash } from '../../algorithms/canonical' +import { defaultConfig, manifestHash } from '../../algorithms/registry' +import { requestManualActivation } from '../activation' +import { recoverRuntimeIntent } from '../recovery' +import { ProfileStore } from '../store' +import { deliverRuntimeOutbox, reconcileActivation, RUNTIME_COMPONENTS } from '../runtime' +import { tickBiometricRuntime } from '../runtime-host' +import { resolveStageView } from '../stage-view' +let configRaw: Database.Database, bioRaw: Database.Database +let config: ReturnType>, bio: ReturnType> +const now = 1791673200000 +beforeEach(() => { + configRaw = new Database(':memory:') + bioRaw = new Database(':memory:') + config = drizzle(configRaw, { schema: configSchema }) + bio = drizzle(bioRaw, { schema: bioSchema }) + migrate(config, { migrationsFolder: 'src/db/migrations' }) + migrate(bio, { migrationsFolder: 'src/db/biometrics-migrations' }) +}) +afterEach(() => { + configRaw.close() + bioRaw.close() +}) +function activate(custom = false) { + const revision = custom + ? new ProfileStore(config).createRevision({ side: 'left', overrides: [{ path: 'heart_rate.trackingToleranceBpm', value: 20 }], + origin: 'manual', requester: 'fixture', idempotencyKey: 'profile' }) + : null + requestManualActivation(config, { side: 'left', action: revision ? 'activate' : 'reset', revisionId: revision?.id ?? null, expectedGeneration: 0, + expectedResolvedHash: revision?.resolvedHash ?? configHash(defaultConfig), boundary: 'next_session', requester: 'fixture', idempotencyKey: 'activate', allowUnvalidated: true }, now) + return revision +} +function obsoleteProjection() { + const row = config.select().from(configSchema.biometricConfigOutbox).get() + if (!row) throw new Error('Missing projection') + configRaw.prepare('UPDATE biometric_config_outbox SET payload = ?').run(JSON.stringify({ ...row.payload as object, manifestHash: 'previous-build' })) +} +describe('runtime recovery across builds', () => { + it('keeps implicit defaults implicit and does not supersede valid next-session intent', () => { + expect(recoverRuntimeIntent(config, 'left', now)).toBeNull() + activate() + expect(recoverRuntimeIntent(config, 'left', now)).toBeNull() + expect(new ProfileStore(config).binding('left').generation).toBe(1) + }) + it('refreshes an obsolete shipped default once and preserves its audit explanation after acknowledgment', () => { + activate() + obsoleteProjection() + const event = recoverRuntimeIntent(config, 'left', now) + expect(event?.evidence).toMatchObject({ status: 'system_default_update', manifestHash }) + expect(event?.generation).toBe(2) + expect(recoverRuntimeIntent(config, 'left', now)).toBeNull() + deliverRuntimeOutbox(config, bio, now) + for (const component of RUNTIME_COMPONENTS) bio.insert(bioSchema.biometricRuntimeReceipts).values({ side: 'left', component, generation: 2, + appliedHash: configHash(defaultConfig), state: 'applied', effectiveSampleTimeMs: now, heartbeatAtMs: now }).run() + expect(reconcileActivation(config, bio, 'left', now)?.state).toBe('applied') + expect(config.select().from(configSchema.biometricActivationEvents).all().find(row => row.generation === 2)?.reason).toContain('Refresh shipped') + }) + it('preserves pinned intent and rollback history during fallback, including after biometric reset', () => { + const revision = activate(true) + if (!revision) throw new Error('Missing revision') + configRaw.prepare('UPDATE biometric_profile_revisions SET base_manifest_hash = ? WHERE id = ?').run('previous-build', revision.id) + expect(tickBiometricRuntime(config, bio, now).errors).toEqual([]) + const binding = new ProfileStore(config).binding('left') + expect(binding).toMatchObject({ desiredRevisionId: revision.id, previousRevisionId: null, generation: 2 }) + const projection = bio.select().from(bioSchema.biometricRuntimeConfigs).get() + expect(projection?.payload).toMatchObject({ revisionId: null, resolvedHash: configHash(defaultConfig) }) + expect(config.select().from(configSchema.biometricActivationEvents).all().find(row => row.generation === 2)?.evidence).toMatchObject({ status: 'profile_incompatible', requestedRevisionId: revision.id }) + bio.delete(bioSchema.biometricRuntimeConfigs).run() + tickBiometricRuntime(config, bio, now) + expect(new ProfileStore(config).binding('left').generation).toBe(2) + expect(bio.select().from(bioSchema.biometricRuntimeConfigs).get()?.generation).toBe(2) + for (const component of RUNTIME_COMPONENTS.slice(0, 3)) bio.insert(bioSchema.biometricRuntimeReceipts).values({ side: 'left', component, generation: 2, + appliedHash: configHash(defaultConfig), state: 'applied', effectiveSampleTimeMs: now, heartbeatAtMs: now }).run() + expect(resolveStageView(config, bio, 'left').provenance.warning).toContain('PROFILE_INCOMPATIBLE') + expect(config.select().from(configSchema.biometricProfileRevisions).get()?.id).toBe(revision.id) + }) + it('restores a compatible-again pinned revision at the next session, never silently rewriting old snapshots', () => { + const revision = activate(true) + if (!revision) throw new Error('Missing revision') + configRaw.prepare('UPDATE biometric_profile_revisions SET base_manifest_hash = ?').run('previous-build') + recoverRuntimeIntent(config, 'left', now) + configRaw.prepare('UPDATE biometric_profile_revisions SET base_manifest_hash = ?').run(manifestHash) + const restored = recoverRuntimeIntent(config, 'left', now) + expect(restored?.evidence).toMatchObject({ status: 'profile_restored', actualRevisionId: revision.id }) + expect(config.select().from(configSchema.biometricConfigOutbox).all().find(row => row.generation === 3)?.payload) + .toMatchObject({ boundary: 'next_session', resolvedHash: revision.resolvedHash }) + expect(new ProfileStore(config).binding('left').desiredRevisionId).toBe(revision.id) + }) + it('repairs a missing custom intent conservatively at the next session', () => { + activate(true) + config.delete(configSchema.biometricConfigOutbox).run() + const event = recoverRuntimeIntent(config, 'left', now) + expect(event?.evidence).toMatchObject({ status: 'runtime_repair' }) + expect(config.select().from(configSchema.biometricConfigOutbox).get()?.payload).toMatchObject({ boundary: 'next_session' }) + }) + it('repairs conflicting biometric projections through a new generation while retaining the corrupt row for diagnosis', () => { + activate(true) + tickBiometricRuntime(config, bio, now) + bioRaw.prepare('UPDATE biometric_runtime_configs SET payload = ?').run('{}') + tickBiometricRuntime(config, bio, now + 1) + expect(config.select().from(configSchema.biometricConfigOutbox).get()?.state).toBe('failed') + tickBiometricRuntime(config, bio, now + 2) + expect(new ProfileStore(config).binding('left').generation).toBe(2) + const rows = bio.select().from(bioSchema.biometricRuntimeConfigs).all() + expect(rows).toHaveLength(2) + expect(rows.find(row => row.generation === 1)?.payload).toEqual({}) + expect(rows.find(row => row.generation === 2)?.payload).toMatchObject({ boundary: 'next_session' }) + }) + it('rolls back all recovery changes when outbox persistence fails', () => { + activate() + obsoleteProjection() + configRaw.exec('CREATE TRIGGER reject_recovery BEFORE INSERT ON biometric_config_outbox BEGIN SELECT RAISE(ABORT, \'full\'); END;') + expect(() => recoverRuntimeIntent(config, 'left', now)).toThrow('full') + expect(new ProfileStore(config).binding('left').generation).toBe(1) + expect(config.select().from(configSchema.biometricActivationEvents).all()).toHaveLength(1) + expect(config.select().from(configSchema.biometricConfigOutbox).get()?.state).toBe('pending') + }) +}) diff --git a/src/biometrics/profiles/tests/replay-context.test.ts b/src/biometrics/profiles/tests/replay-context.test.ts new file mode 100644 index 000000000..5eeb5a04b --- /dev/null +++ b/src/biometrics/profiles/tests/replay-context.test.ts @@ -0,0 +1,56 @@ +// @vitest-environment node +import { expect, it } from 'vitest' +import { verifyReplayContext, replayContextQualificationSchema } from '../replay-context' +import { replayContextFixture } from './fixtures/replay-context' +import { dataHash } from '../../algorithms/canonical' +const expected = { cohortIds: ['a', 'b'], configHashes: ['a'.repeat(64), 'b'.repeat(64), 'a'.repeat(64)], + pythonBuildHash: 'c'.repeat(64), otherSideConfigHash: 'e'.repeat(64), side: 'left' as const } +const qualified = { allowedLimitations: ['event_time_replaces_arrival_time' as const] } +const fixture = () => replayContextFixture(expected.cohortIds, expected.configHashes, expected.pythonBuildHash, expected.otherSideConfigHash) +it('audits exact distinct cohort/configuration outputs and explicitly accepted approximations', () => { + expect(verifyReplayContext(fixture(), expected, qualified)).toEqual({ version: 'qualified-replay-context-v1', + outputs: 4, observedLimitations: ['event_time_replaces_arrival_time'], allowedLimitations: ['event_time_replaces_arrival_time'] }) + expect(() => verifyReplayContext(fixture(), expected, null)).toThrow('not been qualified') +}) +it.each(['missing_pump_context', 'missing_occupancy_context', 'missing_sleeper_mode', 'less_than_five_minutes_preroll', + 'cold_start_without_serialized_state', 'future_unknown_limitation'])('rejects unqualified %s in run or individual reading diagnostics', (code) => { + for (const reading of [true, false]) { + const work = fixture() + if (reading) work.outputs[3].readings[0].limitations.push(code) + else work.outputs[3].run.limitations.push(code) + work.hash = dataHash(work.outputs) + expect(() => verifyReplayContext(work, expected, qualified)).toThrow(`Unqualified replay limitation: ${code}`) + } +}) +it('rejects missing, duplicate, altered and inconsistent output identities', () => { + for (const change of [ + (work: ReturnType) => { work.outputs.pop() }, + (work: ReturnType) => { work.outputs[3] = work.outputs[0] }, + (work: ReturnType) => { work.outputs[0].cohortId = 'unknown' }, + (work: ReturnType) => { work.outputs[0].run.build_hash = 'f'.repeat(64) }, + (work: ReturnType) => { work.outputs[0].run.config_hashes.right = 'f'.repeat(64) }, + (work: ReturnType) => { work.outputs[0].run.counts.scored_outputs = 2 }, + (work: ReturnType) => { work.outputs[0].readings[0].configHash = 'f'.repeat(64) }, + ]) { + const work = fixture() + change(work) + work.hash = dataHash(work.outputs) + expect(() => verifyReplayContext(work, expected, qualified)).toThrow() + } + expect(() => verifyReplayContext({ ...fixture(), hash: 'f'.repeat(64) }, expected, qualified)).toThrow('content hash') +}) + +it('accepts only explicitly designated known limitations and records the observed subset', () => { + const work = fixture() + work.outputs[0].run.limitations.push('cold_start_without_serialized_state') + work.hash = dataHash(work.outputs) + const qualification = replayContextQualificationSchema.parse({ allowedLimitations: [ + 'event_time_replaces_arrival_time', 'cold_start_without_serialized_state', 'less_than_five_minutes_preroll', + ] }) + expect(verifyReplayContext(work, expected, qualification)).toMatchObject({ observedLimitations: [ + 'cold_start_without_serialized_state', 'event_time_replaces_arrival_time', + ] }) + for (const allowedLimitations of [['unknown_future_code'], ['missing_pump_context', 'missing_pump_context']]) { + expect(replayContextQualificationSchema.safeParse({ allowedLimitations }).success).toBe(false) + } +}) diff --git a/src/biometrics/profiles/tests/runtime-host.test.ts b/src/biometrics/profiles/tests/runtime-host.test.ts new file mode 100644 index 000000000..d6bf555fe --- /dev/null +++ b/src/biometrics/profiles/tests/runtime-host.test.ts @@ -0,0 +1,52 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import * as configSchema from '@/src/db/schema' +import * as bioSchema from '@/src/db/biometrics-schema' +import { DEFAULT_STAGE_CLASSIFIER } from '@/src/lib/sleep-stage-rules' +import { startBiometricRuntime, stopBiometricRuntime, tickBiometricRuntime } from '../runtime-host' +let configRaw: Database.Database, bioRaw: Database.Database +let config: ReturnType>, bio: ReturnType> +beforeEach(() => { + configRaw = new Database(':memory:') + bioRaw = new Database(':memory:') + config = drizzle(configRaw, { schema: configSchema }) + bio = drizzle(bioRaw, { schema: bioSchema }) + migrate(config, { migrationsFolder: 'src/db/migrations' }) + migrate(bio, { migrationsFolder: 'src/db/biometrics-migrations' }) + config.insert(configSchema.sideSettings).values([{ side: 'left', name: 'Left', stageClassifier: DEFAULT_STAGE_CLASSIFIER }, { side: 'right', name: 'Right' }]).run() +}) +afterEach(() => { + stopBiometricRuntime() + vi.useRealTimers() + configRaw.close() + bioRaw.close() +}) +it('migrates once, delivers, and repairs projections after history reset without inventing adoption', () => { + expect(tickBiometricRuntime(config, bio).errors).toEqual([]) + expect(tickBiometricRuntime(config, bio).errors).toEqual([]) + expect(config.select().from(configSchema.biometricProfileRevisions).all()).toHaveLength(1) + expect(bio.select().from(bioSchema.biometricRuntimeConfigs).all()).toHaveLength(1) + expect(config.select().from(configSchema.biometricActivationEvents).get()?.state).toBe('pending_boundary') + bio.delete(bioSchema.biometricRuntimeConfigs).run() + expect(tickBiometricRuntime(config, bio).errors).toEqual([]) + expect(bio.select().from(bioSchema.biometricRuntimeConfigs).all()).toHaveLength(1) + expect(bio.select().from(bioSchema.biometricRuntimeReceipts).all()).toEqual([]) +}) +it('continues other recovery work when one legacy side is malformed', () => { + configRaw.prepare('UPDATE side_settings SET stage_classifier = ? WHERE side = ?').run('{}', 'right') + expect(tickBiometricRuntime(config, bio).errors).toEqual(['migration:right']) + expect(bio.select().from(bioSchema.biometricRuntimeConfigs).get()?.side).toBe('left') +}) +it('shares one timer across starts and stops cleanly', () => { + vi.useFakeTimers() + startBiometricRuntime(config, bio) + startBiometricRuntime(config, bio) + expect(vi.getTimerCount()).toBe(1) + vi.advanceTimersByTime(5000) + expect(config.select().from(configSchema.biometricProfileRevisions).all()).toHaveLength(1) + stopBiometricRuntime() + expect(vi.getTimerCount()).toBe(0) +}) diff --git a/src/biometrics/profiles/tests/runtime-integration.test.ts b/src/biometrics/profiles/tests/runtime-integration.test.ts new file mode 100644 index 000000000..bcd6c92ec --- /dev/null +++ b/src/biometrics/profiles/tests/runtime-integration.test.ts @@ -0,0 +1,96 @@ +// @vitest-environment node +import { execFileSync } from 'node:child_process' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { expect, it, vi } from 'vitest' +import * as configSchema from '@/src/db/schema' +import * as bioSchema from '@/src/db/biometrics-schema' +import { tickBiometricRuntime } from '../runtime-host' +import { consumeStageView } from '../stage-view' +const holder = vi.hoisted(() => ({ config: null as unknown, bio: null as unknown })) +vi.mock('@/src/db', () => ({ + get db() { return holder.config }, + get biometricsDb() { return holder.bio }, +})) +const { biometricTuningRouter } = await import('@/src/server/routers/biometricTuning') +const caller = biometricTuningRouter.createCaller({}) + +// No daemon main loop, sensor transport, hardware or user databases. This runs +// the shipped live adapter and estimator against one bounded synthetic frame. +function frame(path: string, occupied: boolean) { + const output = execFileSync(resolve('modules/piezo-processor/.venv/bin/python'), ['-c', ` +import json, sqlite3, sys +sys.path.insert(0, 'modules') +sys.path.insert(0, 'modules/piezo-processor') +import numpy as np +from main import DBHolder, SideProcessor +from runtime_adapter import RuntimeCoordinator +connection = sqlite3.connect(sys.argv[1]) +holder = DBHolder(connection) +processors = {side: SideProcessor(side, holder) for side in ('left', 'right')} +runtime = RuntimeCoordinator(holder, processors) +runtime.before_frame(None, lambda side: sys.argv[2] == 'true', 1791673200000) +for processor in processors.values(): + processor.ingest(np.zeros(500)) +runtime.after_frame(1791673200000) +print(json.dumps({'enabled': runtime.enabled, 'problem': runtime.problem, + 'leftTolerance': processors['left'].parameters.tracking_tolerance_bpm, + 'rightTolerance': processors['right'].parameters.tracking_tolerance_bpm})) +connection.close() +`, path, String(occupied)], { encoding: 'utf8', timeout: 15000, maxBuffer: 65536 }) + return JSON.parse(output) as { enabled: boolean, problem: string | null, leftTolerance: number, rightTolerance: number } +} + +it('acknowledges exact API intent through Python and stage consumers, restart, and rollback', async () => { + const directory = mkdtempSync(join(tmpdir(), 'biometric-runtime-integration-')) + const configRaw = new Database(join(directory, 'config.db')) + const bioPath = join(directory, 'biometrics.db') + const bioRaw = new Database(bioPath) + try { + const config = drizzle(configRaw, { schema: configSchema }) + const bio = drizzle(bioRaw, { schema: bioSchema }) + migrate(config, { migrationsFolder: 'src/db/migrations' }) + migrate(bio, { migrationsFolder: 'src/db/biometrics-migrations' }) + holder.config = config + holder.bio = bio + const defaults = await caller.resolveProfile({ overrides: [] }) + const revision = await caller.createProfileRevision({ side: 'left', overrides: [{ path: 'heart_rate.trackingToleranceBpm', value: 20 }], + origin: 'manual', requester: 'integration', idempotencyKey: 'candidate' }) + const event = await caller.activateProfile({ side: 'left', revisionId: revision.id, expectedGeneration: 0, + expectedResolvedHash: revision.resolvedHash, boundary: 'next_session', requester: 'integration', idempotencyKey: 'activate', allowUnvalidated: true }) + tickBiometricRuntime(config, bio) + expect(frame(bioPath, true)).toMatchObject({ enabled: true, problem: null, leftTolerance: 15 }) + expect(await caller.getActivationStatus({ side: 'left', id: event.id })).toMatchObject({ observedState: 'pending_boundary', receipts: [] }) + expect(frame(bioPath, false)).toMatchObject({ problem: null, leftTolerance: 20, rightTolerance: 15 }) + expect(await caller.getActivationStatus({ side: 'left', id: event.id })).toMatchObject({ observedState: 'pending_boundary' }) + expect(consumeStageView(config, bio, 'left').provenance.bundleHash).toBe(revision.resolvedHash) + expect(await caller.getActivationStatus({ side: 'left', id: event.id })).toMatchObject({ observedState: 'applied', receipts: expect.arrayContaining([ + expect.objectContaining({ component: 'sleep_stages', appliedHash: revision.resolvedHash }), + ]) }) + // Status polling cannot turn a mismatched receipt into an acknowledgment. + bioRaw.prepare('UPDATE biometric_runtime_receipts SET applied_hash = \'wrong\' WHERE component = \'heart_rate\'').run() + const changes = bioRaw.prepare('SELECT total_changes() AS count').get() + expect(await caller.getActivationStatus({ side: 'left', id: event.id })).toMatchObject({ observedState: 'pending_boundary' }) + expect(bioRaw.prepare('SELECT total_changes() AS count').get()).toEqual(changes) + bioRaw.prepare('UPDATE biometric_runtime_receipts SET applied_hash = ? WHERE component = \'heart_rate\'').run(revision.resolvedHash) + // New Python process restores the acknowledged profile despite an occupied bed. + expect(frame(bioPath, true).leftTolerance).toBe(20) + const rollback = await caller.rollbackProfile({ side: 'left', revisionId: null, expectedGeneration: 1, + expectedResolvedHash: defaults.hash, boundary: 'immediate', requester: 'integration', idempotencyKey: 'rollback', allowUnvalidated: true }) + tickBiometricRuntime(config, bio) + expect(frame(bioPath, true).leftTolerance).toBe(15) + consumeStageView(config, bio, 'left') + expect(await caller.getActivationStatus({ side: 'left', id: rollback.id })).toMatchObject({ observedState: 'applied', isCurrent: true }) + expect(await caller.getActivationStatus({ side: 'left', id: event.id })).toMatchObject({ isCurrent: false }) + expect(bioRaw.prepare('SELECT count(*) AS count FROM vitals').get()).toEqual({ count: 0 }) + } + finally { + configRaw.close() + bioRaw.close() + rmSync(directory, { recursive: true, force: true }) + } +}, 30000) diff --git a/src/biometrics/profiles/tests/runtime.test.ts b/src/biometrics/profiles/tests/runtime.test.ts new file mode 100644 index 000000000..822e5fcb4 --- /dev/null +++ b/src/biometrics/profiles/tests/runtime.test.ts @@ -0,0 +1,67 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import * as configSchema from '@/src/db/schema' +import * as bioSchema from '@/src/db/biometrics-schema' +import { requestManualActivation } from '../activation' +import { deliverRuntimeOutbox, projectRuntimeConfig, reconcileActivation, repairCurrentRuntimeProjections, RUNTIME_COMPONENTS } from '../runtime' +import { configHash } from '../../algorithms/canonical' +import { defaultConfig } from '../../algorithms/registry' +let configSqlite: Database.Database, bioSqlite: Database.Database +let configDb: ReturnType>, bioDb: ReturnType> +const now = 1791673200000 +beforeEach(() => { + configSqlite = new Database(':memory:') + bioSqlite = new Database(':memory:') + configDb = drizzle(configSqlite, { schema: configSchema }) + bioDb = drizzle(bioSqlite, { schema: bioSchema }) + migrate(configDb, { migrationsFolder: 'src/db/migrations' }) + migrate(bioDb, { migrationsFolder: 'src/db/biometrics-migrations' }) + requestManualActivation(configDb, { side: 'left', action: 'reset', revisionId: null, expectedGeneration: 0, expectedResolvedHash: configHash(defaultConfig), + boundary: 'next_session', requester: 'fixture', idempotencyKey: 'reset', allowUnvalidated: true }, now) +}) +afterEach(() => { + configSqlite.close() + bioSqlite.close() +}) +const payload = () => configDb.select().from(configSchema.biometricConfigOutbox).get()?.payload + +describe('runtime projection and acknowledgment', () => { + it('retries safely after projection succeeds but the configuration acknowledgment fails', () => { + configSqlite.exec('CREATE TRIGGER reject_delivery BEFORE UPDATE OF state ON biometric_config_outbox WHEN NEW.state = \'delivered\' BEGIN SELECT RAISE(ABORT, \'fixture crash\'); END;') + expect(deliverRuntimeOutbox(configDb, bioDb, now)).toEqual({ delivered: 0, failed: 1 }) + expect(bioDb.select().from(bioSchema.biometricRuntimeConfigs).all()).toHaveLength(1) + configSqlite.exec('DROP TRIGGER reject_delivery') + expect(deliverRuntimeOutbox(configDb, bioDb, now + 1000)).toEqual({ delivered: 1, failed: 0 }) + expect(bioDb.select().from(bioSchema.biometricRuntimeConfigs).all()).toHaveLength(1) + expect(reconcileActivation(configDb, bioDb, 'left', now + 1000)?.state).toBe('pending_boundary') + expect(bioDb.select().from(bioSchema.biometricRuntimeReceipts).all()).toEqual([]) + }) + it('requires every matching component receipt and ignores old generations and wrong hashes', () => { + deliverRuntimeOutbox(configDb, bioDb, now) + for (const component of RUNTIME_COMPONENTS) bioDb.insert(bioSchema.biometricRuntimeReceipts).values({ side: 'left', component, generation: 0, + appliedHash: configHash(defaultConfig), state: 'applied', effectiveSampleTimeMs: now, heartbeatAtMs: now }).run() + expect(reconcileActivation(configDb, bioDb, 'left', now)?.state).toBe('pending_boundary') + for (const component of RUNTIME_COMPONENTS) bioDb.insert(bioSchema.biometricRuntimeReceipts).values({ side: 'left', component, generation: 1, + appliedHash: component === 'sleep_stages' ? 'wrong' : configHash(defaultConfig), state: 'applied', effectiveSampleTimeMs: now, heartbeatAtMs: now }).run() + expect(reconcileActivation(configDb, bioDb, 'left', now)?.state).toBe('pending_boundary') + bioSqlite.prepare('UPDATE biometric_runtime_receipts SET applied_hash = ? WHERE generation = 1').run(configHash(defaultConfig)) + expect(reconcileActivation(configDb, bioDb, 'left', now)?.state).toBe('applied') + }) + it('restores the current projection after a history reset and rejects conflicting duplicate generations', () => { + deliverRuntimeOutbox(configDb, bioDb, now) + bioDb.delete(bioSchema.biometricRuntimeConfigs).run() + expect(repairCurrentRuntimeProjections(configDb, bioDb, now)).toBe(1) + expect(repairCurrentRuntimeProjections(configDb, bioDb, now)).toBe(0) + expect(() => projectRuntimeConfig(bioDb, { ...(payload() as object), revisionId: '11111111-1111-4111-8111-111111111111' }, now)).toThrow('different immutable payload') + expect(configDb.select().from(configSchema.biometricProfileBindings).get()?.generation).toBe(1) + }) + it('marks invalid envelopes failed without projecting or fabricating receipts', () => { + configSqlite.prepare('UPDATE biometric_config_outbox SET resolved_hash = ?').run('wrong') + expect(deliverRuntimeOutbox(configDb, bioDb, now)).toEqual({ delivered: 0, failed: 1 }) + expect(reconcileActivation(configDb, bioDb, 'left', now)?.state).toBe('failed') + expect(bioDb.select().from(bioSchema.biometricRuntimeConfigs).all()).toEqual([]) + }) +}) diff --git a/src/biometrics/profiles/tests/stage-view.test.ts b/src/biometrics/profiles/tests/stage-view.test.ts new file mode 100644 index 000000000..87b3b499c --- /dev/null +++ b/src/biometrics/profiles/tests/stage-view.test.ts @@ -0,0 +1,135 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import * as configSchema from '@/src/db/schema' +import * as bioSchema from '@/src/db/biometrics-schema' +import { DEFAULT_STAGE_CLASSIFIER, MOVEMENT_ONLY_STAGE_RULESET } from '@/src/lib/sleep-stage-rules' +import { defaultConfig } from '../../algorithms/registry' +import { resolveReplayBaseline } from '../replay-baseline' +import { ProfileStore } from '../store' +import { writeLegacyStageClassifier } from '../legacy-stage' +import { consumeStageView, resolveStageView } from '../stage-view' +import { deliverRuntimeOutbox, reconcileActivation } from '../runtime' +import { requestManualActivation } from '../activation' +let configRaw: Database.Database, bioRaw: Database.Database +let config: ReturnType>, bio: ReturnType> +const movementOnly = { engine: 'rules' as const, ruleset: MOVEMENT_ONLY_STAGE_RULESET } +const now = 1791673200000 +beforeEach(() => { + configRaw = new Database(':memory:') + bioRaw = new Database(':memory:') + config = drizzle(configRaw, { schema: configSchema }) + bio = drizzle(bioRaw, { schema: bioSchema }) + migrate(config, { migrationsFolder: 'src/db/migrations' }) + migrate(bio, { migrationsFolder: 'src/db/biometrics-migrations' }) + config.insert(configSchema.sideSettings).values([{ side: 'left', name: 'Left' }, { side: 'right', name: 'Right' }]).run() +}) +afterEach(() => { + configRaw.close() + bioRaw.close() +}) +function acknowledge(generation: number, components = ['heart_rate', 'breathing_rate', 'hrv_index']) { + deliverRuntimeOutbox(config, bio, now + 1) + const row = bioRaw.prepare('SELECT resolved_hash FROM biometric_runtime_configs WHERE side = ? AND generation = ?').get('left', generation) as { resolved_hash: string } + for (const component of components) bio.insert(bioSchema.biometricRuntimeReceipts).values({ side: 'left', component, generation, + appliedHash: row.resolved_hash, effectiveSampleTimeMs: now, state: 'applied', heartbeatAtMs: now }).run() +} +describe('stage view adoption and legacy compatibility', () => { + it('waits for all numerical consumers, then acknowledges exactly the view delivered for classification', () => { + const saved = writeLegacyStageClassifier(config, 'left', movementOnly, now) + expect(saved.activation.generation).toBe(1) + acknowledge(1, ['heart_rate', 'breathing_rate']) + expect(consumeStageView(config, bio, 'left', now).classifier).toEqual(DEFAULT_STAGE_CLASSIFIER) + expect(reconcileActivation(config, bio, 'left', now)?.state).toBe('pending_boundary') + acknowledge(1, ['hrv_index']) + const view = consumeStageView(config, bio, 'left', now) + expect(view.classifier).toEqual(movementOnly) + expect(view.provenance).toMatchObject({ source: 'runtime', generation: 1, interpretation: 'current_classifier_view' }) + expect(reconcileActivation(config, bio, 'left', now)?.state).toBe('applied') + expect(resolveStageView(config, bio, 'right').classifier).toEqual(DEFAULT_STAGE_CLASSIFIER) + }) + it('does not rescore from pending intent and preserves the last fully applied view', () => { + writeLegacyStageClassifier(config, 'left', movementOnly, now) + acknowledge(1) + consumeStageView(config, bio, 'left', now) + writeLegacyStageClassifier(config, 'left', null, now) + acknowledge(2, ['heart_rate']) + expect(resolveStageView(config, bio, 'left').classifier).toEqual(movementOnly) + expect(resolveStageView(config, bio, 'left').provenance.generation).toBe(1) + acknowledge(2, ['breathing_rate', 'hrv_index']) + expect(consumeStageView(config, bio, 'left', now).classifier).toEqual(DEFAULT_STAGE_CLASSIFIER) + }) + it('retains other metric overrides when legacy clients reset only stage rules', () => { + const store = new ProfileStore(config) + const revision = store.createRevision({ side: 'left', overrides: [{ path: 'heart_rate.trackingToleranceBpm', value: 20 }], origin: 'manual', requester: 'fixture', idempotencyKey: 'hr' }) + requestManualActivation(config, { side: 'left', action: 'activate', revisionId: revision.id, expectedGeneration: 0, + expectedResolvedHash: revision.resolvedHash, boundary: 'immediate', requester: 'fixture', idempotencyKey: 'hr-active', allowUnvalidated: true }, now) + writeLegacyStageClassifier(config, 'left', movementOnly, now) + writeLegacyStageClassifier(config, 'left', null, now) + expect(store.effective('left').resolved.heart_rate.trackingToleranceBpm).toBe(20) + expect(store.effective('left').resolved.sleep_stages).toEqual(DEFAULT_STAGE_CLASSIFIER) + expect(store.effective('left').binding.desiredRevisionId).not.toBeNull() + expect(configRaw.prepare('SELECT stage_classifier FROM side_settings WHERE side = ?').get('left')).toEqual({ stage_classifier: null }) + }) + it('migrates existing customization before compatibility writes without losing its previous revision', () => { + configRaw.prepare('UPDATE side_settings SET stage_classifier = ? WHERE side = ?').run(JSON.stringify(movementOnly), 'left') + const changed = writeLegacyStageClassifier(config, 'left', null, now) + expect(changed.activation.generation).toBe(2) + const store = new ProfileStore(config) + const previousId = store.binding('left').previousRevisionId + if (!previousId) throw new Error('Missing previous revision') + const previous = store.getRevision(previousId) + expect((previous.resolved as { sleep_stages: unknown }).sleep_stages).toEqual(movementOnly) + expect(resolveStageView(config, bio, 'left').classifier).toEqual(movementOnly) + }) + it('surfaces incompatible applied projections without acknowledging them as active', () => { + writeLegacyStageClassifier(config, 'left', movementOnly, now) + acknowledge(1) + bioRaw.prepare('UPDATE biometric_runtime_configs SET payload = ?').run('{}') + const view = consumeStageView(config, bio, 'left', now) + expect(view.classifier).toEqual(DEFAULT_STAGE_CLASSIFIER) + expect(view.provenance.warning).toContain('incompatible') + expect(bioRaw.prepare('SELECT * FROM biometric_runtime_receipts WHERE component=\'sleep_stages\'').all()).toEqual([]) + }) + it('rolls back the compatibility revision and binding when intent publication fails', () => { + configRaw.exec('CREATE TRIGGER reject_outbox BEFORE INSERT ON biometric_config_outbox BEGIN SELECT RAISE(ABORT, \'full\'); END;') + expect(() => writeLegacyStageClassifier(config, 'left', movementOnly, now)).toThrow('full') + expect(config.select().from(configSchema.biometricProfileRevisions).all()).toEqual([]) + expect(config.select().from(configSchema.biometricProfileBindings).all()).toEqual([]) + }) +}) + +it('captures a complete numerical baseline only after coherent adoption', () => { + expect(resolveReplayBaseline(config, bio, 'left')).toMatchObject({ resolved: defaultConfig, provenance: { source: 'unacknowledged_defaults' } }) + const store = new ProfileStore(config) + const revision = store.createRevision({ side: 'left', overrides: [{ path: 'heart_rate.trackingToleranceBpm', value: 20 }], origin: 'manual', requester: 'fixture', idempotencyKey: 'numerical-current' }) + requestManualActivation(config, { side: 'left', action: 'activate', revisionId: revision.id, expectedGeneration: 0, + expectedResolvedHash: revision.resolvedHash, boundary: 'immediate', requester: 'fixture', idempotencyKey: 'numerical-active', allowUnvalidated: true }, now) + expect(resolveReplayBaseline(config, bio, 'left').resolved.heart_rate.trackingToleranceBpm).toBe(defaultConfig.heart_rate.trackingToleranceBpm) + acknowledge(1, ['heart_rate']) + expect(() => resolveReplayBaseline(config, bio, 'left')).toThrow('consistent configuration') + acknowledge(1, ['breathing_rate', 'hrv_index']) + const captured = resolveReplayBaseline(config, bio, 'left') + expect(captured).toMatchObject({ hash: revision.resolvedHash, revisionId: revision.id, resolved: { heart_rate: { trackingToleranceBpm: 20 } }, + provenance: { source: 'acknowledged_runtime', generation: 1, effectiveSampleTimeMs: now } }) + expect(resolveReplayBaseline(config, bio, 'right').resolved).toEqual(defaultConfig) + expect(bioRaw.prepare('SELECT * FROM biometric_runtime_receipts WHERE component=\'sleep_stages\'').all()).toEqual([]) + writeLegacyStageClassifier(config, 'left', movementOnly, now) + acknowledge(2, ['heart_rate']) + expect(() => resolveReplayBaseline(config, bio, 'left')).toThrow('consistent configuration') + bioRaw.prepare('UPDATE biometric_runtime_receipts SET state=\'pending_boundary\' WHERE generation=2').run() + expect(resolveReplayBaseline(config, bio, 'left')).toEqual(captured) + bioRaw.prepare('UPDATE biometric_runtime_receipts SET state=\'failed\' WHERE generation=2').run() + bio.insert(bioSchema.biometricRuntimeReceipts).values({ side: 'left', component: 'heart_rate', generation: 3, appliedHash: revision.resolvedHash, + state: 'pending_boundary', effectiveSampleTimeMs: null, heartbeatAtMs: now }).run() + expect(() => resolveReplayBaseline(config, bio, 'left')).toThrow('adoption failed') +}) + +it('refuses corrupt acknowledged projections instead of relabeling defaults as current', () => { + writeLegacyStageClassifier(config, 'left', movementOnly, now) + acknowledge(1) + bioRaw.prepare('UPDATE biometric_runtime_configs SET payload = ?').run('{}') + expect(() => resolveReplayBaseline(config, bio, 'left')).toThrow('unavailable or incompatible') +}) diff --git a/src/biometrics/profiles/tests/store.test.ts b/src/biometrics/profiles/tests/store.test.ts new file mode 100644 index 000000000..745c3295a --- /dev/null +++ b/src/biometrics/profiles/tests/store.test.ts @@ -0,0 +1,101 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { eq } from 'drizzle-orm' +import * as schema from '@/src/db/schema' +import { DEFAULT_STAGE_CLASSIFIER } from '@/src/lib/sleep-stage-rules' +import { defaultConfig } from '../../algorithms/registry' +import { ProfileStore } from '../store' + +let sqlite: Database.Database +let db: ReturnType> +let store: ProfileStore +const request = { side: 'left' as const, overrides: [{ path: 'heart_rate.trackingToleranceBpm', value: 20 }], + origin: 'manual' as const, requester: 'test', idempotencyKey: 'candidate-1' } + +beforeEach(() => { + sqlite = new Database(':memory:') + db = drizzle(sqlite, { schema }) + migrate(db, { migrationsFolder: 'src/db/migrations' }) + store = new ProfileStore(db, () => 1700000000000) +}) +afterEach(() => sqlite.close()) + +describe('immutable profile repository', () => { + it('creates an idempotent candidate without activating it', () => { + const first = store.createRevision(request) + expect(store.createRevision(request)).toEqual(first) + expect(store.createRevision({ ...request, parentRevisionId: undefined })).toEqual(first) + expect(first.revision).toBe(1) + expect(store.effective('left')).toMatchObject({ isDefault: true, binding: { generation: 0 } }) + expect(store.validateRevision(first.id, 'left').resolved.heart_rate.trackingToleranceBpm).toBe(20) + expect(() => store.createRevision({ ...request, overrides: [] })).toThrow('IDEMPOTENCY_CONFLICT') + expect(db.select().from(schema.biometricConfigOutbox).all()).toEqual([]) + }) + + it('preserves the parent snapshot and serializes family revision numbers', () => { + const first = store.createRevision(request) + const second = store.createRevision({ ...request, parentRevisionId: first.id, overrides: [], idempotencyKey: 'candidate-2' }) + const third = store.createRevision({ ...request, parentRevisionId: first.id, overrides: [], idempotencyKey: 'candidate-3' }) + expect([second.revision, third.revision]).toEqual([2, 3]) + expect(second.familyId).toBe(first.familyId) + expect(store.getRevision(first.id)).toEqual(first) + expect(store.validateRevision(second.id, 'left').resolved).toEqual(defaultConfig) + expect(store.listRevisions('left', undefined, first.familyId).items.map(r => r.id).sort()).toEqual([first.id, second.id, third.id].sort()) + }) + + it('rejects cross-side inheritance, unknown parents and tampered snapshots', () => { + const first = store.createRevision(request) + expect(() => store.createRevision({ ...request, side: 'right', parentRevisionId: first.id, idempotencyKey: 'right' })).toThrow('other side') + expect(() => store.validateRevision(first.id, 'right')).toThrow('other side') + expect(() => store.createRevision({ ...request, parentRevisionId: '00000000-0000-4000-8000-000000000000', idempotencyKey: 'missing' })).toThrow('NOT_FOUND') + db.update(schema.biometricProfileRevisions).set({ resolvedHash: 'bad' }).where(eq(schema.biometricProfileRevisions.id, first.id)).run() + expect(() => store.validateRevision(first.id, 'left')).toThrow('integrity') + }) + + it('reports a visible default fallback for an unavailable pinned engine', () => { + const first = store.createRevision(request) + db.insert(schema.biometricProfileBindings).values({ side: 'left', desiredRevisionId: first.id, generation: 2, updatedAtMs: 1 }).run() + db.update(schema.biometricProfileRevisions).set({ baseManifestHash: 'unavailable' }).run() + expect(store.effective('left')).toMatchObject({ isDefault: true, warning: expect.stringContaining('PROFILE_INCOMPATIBLE'), resolved: defaultConfig }) + expect(store.binding('left').desiredRevisionId).toBe(first.id) + }) +}) + +describe('legacy stage migration', () => { + it('preserves exact rules and commits binding, evidence and outbox atomically', () => { + const classifier = structuredClone(DEFAULT_STAGE_CLASSIFIER) + classifier.ruleset.rules.reverse() + classifier.ruleset.smoothIsolatedEpochs = false + db.insert(schema.sideSettings).values({ side: 'left', name: 'Left', stageClassifier: classifier }).run() + const binding = store.migrateLegacy('left') + if (!binding?.desiredRevisionId) throw new Error('Expected migrated binding') + expect(store.migrateLegacy('left')).toEqual(binding) + expect(store.validateRevision(binding.desiredRevisionId, 'left').resolved.sleep_stages).toEqual(classifier) + expect(db.select().from(schema.biometricProfileRevisions).all()).toHaveLength(1) + expect(db.select().from(schema.biometricActivationEvents).get()).toMatchObject({ generation: 1, evidence: { status: 'legacy_unvalidated' }, state: 'pending_delivery' }) + expect(db.select().from(schema.biometricConfigOutbox).get()).toMatchObject({ generation: 1, state: 'pending' }) + expect(db.select().from(schema.sideSettings).get()?.stageClassifier).toEqual(classifier) + expect(store.effective('right').isDefault).toBe(true) + }) + + it('keeps null on system defaults and refuses to discard invalid legacy settings', () => { + db.insert(schema.sideSettings).values({ side: 'right', name: 'Right' }).run() + expect(store.migrateLegacy('right')).toBeNull() + db.insert(schema.sideSettings).values({ side: 'left', name: 'Left', stageClassifier: { engine: 'unknown' } }).run() + expect(() => store.migrateLegacy('left')).toThrow() + expect(db.select().from(schema.biometricProfileBindings).all()).toEqual([]) + expect(db.select().from(schema.biometricProfileRevisions).all()).toEqual([]) + }) + + it('rolls back the entire migration if outbox publication fails', () => { + db.insert(schema.sideSettings).values({ side: 'left', name: 'Left', stageClassifier: DEFAULT_STAGE_CLASSIFIER }).run() + sqlite.exec('CREATE TRIGGER fail_outbox BEFORE INSERT ON biometric_config_outbox BEGIN SELECT RAISE(ABORT, \'disk full\'); END') + expect(() => store.migrateLegacy('left')).toThrow() + expect(db.select().from(schema.biometricProfileBindings).all()).toEqual([]) + expect(db.select().from(schema.biometricActivationEvents).all()).toEqual([]) + expect(db.select().from(schema.biometricProfileRevisions).all()).toEqual([]) + }) +}) diff --git a/src/biometrics/profiles/tests/validated-activation.test.ts b/src/biometrics/profiles/tests/validated-activation.test.ts new file mode 100644 index 000000000..8609e9579 --- /dev/null +++ b/src/biometrics/profiles/tests/validated-activation.test.ts @@ -0,0 +1,246 @@ +// @vitest-environment node +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import * as configSchema from '@/src/db/schema' +import * as bioSchema from '@/src/db/biometrics-schema' +import { ProfileStore } from '../store' +import { requestManualActivation, requestValidatedActivation, requestValidatedActivationWithBuild, validatedActivationReviewSchema, reviewValidatedActivation } from '../activation' +import { deliverRuntimeOutbox } from '../runtime' +import { prepareVerifiedEvaluation } from '../evaluation-store' +import { BacktestJobStore } from '../../backtest/jobs' +import { DatasetStore } from '../../backtest/datasets' +import { backtestJobSnapshotSchema, datasetManifestSchema } from '../../backtest/contracts' +import { submitBacktest } from '../../backtest/submission' +import { evaluationFixture } from './fixtures/evaluation' +import { defaultConfig } from '../../algorithms/registry' +import { configHash } from '../../algorithms/canonical' + +let configRaw: Database.Database, bioRaw: Database.Database, observer: Database.Database +let config: ReturnType>, bio: ReturnType> +let root: string +const now = 1791673200000 +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), 'biometric-activation-')) + configRaw = new Database(':memory:') + bioRaw = new Database(join(root, 'bio.sqlite')) + bioRaw.pragma('journal_mode = WAL') + observer = new Database(join(root, 'bio.sqlite'), { timeout: 0 }) + config = drizzle(configRaw, { schema: configSchema }) + bio = drizzle(bioRaw, { schema: bioSchema }) + migrate(config, { migrationsFolder: 'src/db/migrations' }) + migrate(bio, { migrationsFolder: 'src/db/biometrics-migrations' }) +}) +afterEach(() => { + vi.restoreAllMocks() + observer.close() + configRaw.close() + bioRaw.close() + rmSync(root, { recursive: true, force: true }) +}) + +function fixture() { + const work = evaluationFixture() + const profiles = new ProfileStore(config) + const revision = profiles.createRevision({ side: 'left', origin: 'manual', requester: 'fixture', idempotencyKey: 'candidate', + overrides: [{ path: 'sleep_stages.ruleset.smoothIsolatedEpochs', value: false }] }) + for (let id = 1; id <= 12; id++) bio.insert(bioSchema.sleepRecords).values({ id, side: 'left', + enteredBedAt: new Date(now + id * 86400000), leftBedAt: new Date(now + id * 86400000 + 3600000), sleepDurationSeconds: 3600, timesExitedBed: 0 }).run() + const datasets = new DatasetStore(bio) + const dataset = datasets.create({ side: 'left', subjectId: 'fixture', sessionIds: Array.from({ length: 12 }, (_, i) => i + 1), + referenceImportIds: [], timezone: 'UTC', idempotencyKey: 'dataset' }) + const cohorts = datasetManifestSchema.parse(dataset.manifest).splitProtocol.assignments.filter(row => row.split === 'holdout').map(row => row.cohortId) + expect(cohorts).toHaveLength(3) + const jobs = new BacktestJobStore(bio) + const experiment = jobs.createExperiment(dataset.id, work.job.protocol) + const configs = work.job.configs.map(row => row.label === 'candidate' ? { ...row, revisionId: revision.id } : row) + const enqueue = (idempotencyKey: string) => jobs.enqueue({ experimentId: experiment.id, evaluatorBuildHash: work.job.evaluatorBuildHash, + idempotencyKey, split: 'holdout', configs }) + const job = enqueue('holdout') + const report = { ...work.report, datasetId: dataset.id, datasetHash: dataset.contentHash, nights: cohorts.map(cohortId => ({ cohortId })) } + for (const row of report.stageObjectiveAssessments) { + row.evidence.nights.forEach((night, i) => { + night.cohortId = cohorts[i] + }) + } + const lease = jobs.claim('fixture') + if (!lease) throw new Error('No lease') + const result = jobs.complete({ id: lease.id, owner: 'fixture', token: lease.fencingToken }, report) + const input = { side: 'left' as const, action: 'activate' as const, revisionId: revision.id, expectedGeneration: 0, + expectedResolvedHash: revision.resolvedHash, boundary: 'next_session' as const, requester: 'fixture', idempotencyKey: 'validated', + evaluationId: job.id, expectedReportHash: result.contentHash } + const build = { evaluatorBuildHash: work.job.evaluatorBuildHash, designations: work.designations } + return { profiles, jobs, datasets, dataset, input, build, enqueue } +} + +it('atomically stores verified evidence and outbox intent; retries survive released biometric artifacts', () => { + const work = fixture() + const event = requestValidatedActivation(config, bio, work.input, work.build, now) + expect(event).toMatchObject({ state: 'pending_delivery', generation: 1, evaluationId: work.input.evaluationId, + evidence: { status: 'validated_reference_agreement', reportHash: work.input.expectedReportHash, changedDomains: ['sleep_stages'] } }) + expect(work.profiles.binding('left').desiredRevisionId).toBe(work.input.revisionId) + expect(work.profiles.binding('right').generation).toBe(0) + expect(config.select().from(configSchema.biometricConfigOutbox).all()).toHaveLength(1) + work.jobs.releaseResult(work.input.evaluationId, work.input.expectedReportHash) + work.datasets.release(work.dataset.id) + expect(requestValidatedActivation(config, bio, work.input, work.build, now + 1)).toEqual(event) + expect(config.select().from(configSchema.biometricActivationEvents).all()).toHaveLength(1) + expect(bio.select().from(bioSchema.vitals).all()).toEqual([]) +}) + +it('rejects missing designation, mismatched hashes, and reused holdout before changing bindings', () => { + const work = fixture() + expect(() => requestValidatedActivation(config, bio, work.input, { ...work.build, designations: { protocolHash: null, waveformContext: null, numeric: null, stages: null } })).toThrow() + expect(() => requestValidatedActivation(config, bio, { ...work.input, expectedReportHash: '0'.repeat(64) }, work.build)).toThrow('hash') + work.enqueue('second-holdout') + expect(() => requestValidatedActivation(config, bio, work.input, work.build)).toThrow('holdout') + expect(work.profiles.binding('left').generation).toBe(0) + expect(config.select().from(configSchema.biometricConfigOutbox).all()).toEqual([]) +}) + +it('blocks concurrent biometrics writes across the configuration commit', () => { + const work = fixture() + let blocked = false + configRaw.function('check_bio_reservation', () => { + try { + observer.prepare('UPDATE biometric_backtest_jobs SET updated_at_ms = updated_at_ms').run() + } + catch (error) { blocked = (error as { code: string }).code === 'SQLITE_BUSY' } + return 1 + }) + configRaw.exec('CREATE TRIGGER check_lock BEFORE INSERT ON biometric_profile_bindings BEGIN SELECT check_bio_reservation(); END;') + requestValidatedActivation(config, bio, work.input, work.build) + expect(blocked).toBe(true) + expect(() => observer.prepare('UPDATE biometric_backtest_jobs SET updated_at_ms = updated_at_ms').run()).not.toThrow() +}) + +it('rejects holdout reuse that occurs after the first verified result read', () => { + const work = fixture() + const original = BacktestJobStore.prototype.result + vi.spyOn(BacktestJobStore.prototype, 'result').mockImplementationOnce(function (this: BacktestJobStore, id) { + const result = original.call(this, id) + work.enqueue('racing-holdout') + return result + }) + expect(() => requestValidatedActivation(config, bio, work.input, work.build)).toThrow('reused before activation') + expect(work.profiles.binding('left').generation).toBe(0) +}) + +it('rejects report release between verification and committing intent', () => { + const work = fixture() + const original = BacktestJobStore.prototype.result + const spy = vi.spyOn(BacktestJobStore.prototype, 'result').mockImplementationOnce(function (this: BacktestJobStore, id) { + const result = original.call(this, id) + spy.mockRestore() + work.jobs.releaseResult(work.input.evaluationId, work.input.expectedReportHash) + return result + }) + expect(() => requestValidatedActivation(config, bio, work.input, work.build)).toThrow('released, replaced or reused') + expect(work.profiles.binding('left').generation).toBe(0) +}) + +it.each([ + 'UPDATE biometric_backtest_jobs SET state = \'failed\'', + 'UPDATE biometric_backtest_jobs SET fencing_token = fencing_token + 1', + 'DELETE FROM biometric_backtest_results', + 'UPDATE biometric_backtest_results SET fencing_token = fencing_token + 1', + 'UPDATE biometric_backtest_results SET content_hash = \'changed\'', + 'UPDATE biometric_backtest_results SET released_at_ms = 1', + 'UPDATE biometric_datasets SET content_hash = \'changed\'', + 'UPDATE biometric_datasets SET released_at_ms = 1', +])('rejects changed retained metadata at the commit guard: %s', (statement) => { + const work = fixture() + const prepared = prepareVerifiedEvaluation(config, bio, work.input, work.build) + bioRaw.exec(statement) + expect(() => prepared.recheck()).toThrow('released, replaced or reused') + expect(work.profiles.binding('left').generation).toBe(0) +}) + +it('rejects pending baselines and captures the complete acknowledged bundle for subsequent stage replay', () => { + const work = fixture() + const revision = work.profiles.createRevision({ side: 'left', origin: 'manual', requester: 'fixture', idempotencyKey: 'numeric', + overrides: [{ path: 'heart_rate.trackingToleranceBpm', value: 20 }] }) + requestManualActivation(config, { side: 'left', action: 'activate', revisionId: revision.id, expectedGeneration: 0, + expectedResolvedHash: revision.resolvedHash, boundary: 'immediate', requester: 'fixture', idempotencyKey: 'numeric-activation', allowUnvalidated: true }, now) + expect(() => requestValidatedActivation(config, bio, work.input, work.build)).toThrow('Binding changed') + expect(() => requestValidatedActivation(config, bio, { ...work.input, expectedGeneration: 1 }, work.build)).toThrow('pending') + deliverRuntimeOutbox(config, bio, now) + for (const component of ['heart_rate', 'breathing_rate', 'hrv_index']) bio.insert(bioSchema.biometricRuntimeReceipts) + .values({ side: 'left', component, generation: 1, appliedHash: revision.resolvedHash, effectiveSampleTimeMs: now, state: 'applied', heartbeatAtMs: now }).run() + expect(() => requestValidatedActivation(config, bio, { ...work.input, expectedGeneration: 1 }, work.build)).toThrow('baseline changed') + const experimentId = work.jobs.get(work.input.evaluationId).experimentId + const next = submitBacktest(config, bio, work.build.evaluatorBuildHash, { experimentId, candidateRevisionId: work.input.revisionId, + expectedBindingGeneration: 1, split: 'tuning', idempotencyKey: 'next-stage-replay' }) + expect(backtestJobSnapshotSchema.parse(next.input).configs.find(row => row.label === 'current')).toMatchObject({ + revisionId: revision.id, resolvedHash: revision.resolvedHash, resolved: { heart_rate: { trackingToleranceBpm: 20 } } }) +}) + +it('rolls back binding, evidence and outbox together and keeps manual/validated idempotency distinct', () => { + const work = fixture() + configRaw.exec('CREATE TRIGGER reject_outbox BEFORE INSERT ON biometric_config_outbox BEGIN SELECT RAISE(ABORT, \'fixture failure\'); END;') + expect(() => requestValidatedActivation(config, bio, work.input, work.build)).toThrow('fixture failure') + expect(work.profiles.binding('left').generation).toBe(0) + expect(config.select().from(configSchema.biometricActivationEvents).all()).toEqual([]) + configRaw.exec('DROP TRIGGER reject_outbox') + requestValidatedActivation(config, bio, work.input, work.build) + expect(() => requestValidatedActivation(config, bio, { ...work.input, boundary: 'immediate' }, work.build)).toThrow('idempotency') + expect(() => requestManualActivation(config, { side: 'left', action: 'reset', revisionId: null, expectedGeneration: 1, + expectedResolvedHash: configHash(defaultConfig), boundary: 'immediate', requester: 'fixture', idempotencyKey: work.input.idempotencyKey, allowUnvalidated: true })).toThrow('idempotency') +}) + +it('resolves build authority lazily and retries after artifact release without probing', async () => { + const work = fixture() + const resolve = vi.fn(async () => work.build) + const event = await requestValidatedActivationWithBuild(config, bio, work.input, resolve, now) + expect(resolve).toHaveBeenCalledTimes(1) + work.jobs.releaseResult(work.input.evaluationId, work.input.expectedReportHash) + work.datasets.release(work.dataset.id) + resolve.mockRejectedValue(new Error('Build no longer installed')) + expect(await requestValidatedActivationWithBuild(config, bio, work.input, resolve, now + 1)).toEqual(event) + expect(resolve).toHaveBeenCalledTimes(1) + await expect(requestValidatedActivationWithBuild(config, bio, { ...work.input, expectedReportHash: '0'.repeat(64) }, resolve)) + .rejects.toThrow('idempotency key changed') + expect(resolve).toHaveBeenCalledTimes(1) +}) + +it('coalesces identical activation intents that race across asynchronous build resolution', async () => { + const work = fixture() + let finish!: (build: typeof work.build) => void + const pending = new Promise((resolve) => { + finish = resolve + }) + const first = requestValidatedActivationWithBuild(config, bio, work.input, () => pending, now) + const second = await requestValidatedActivationWithBuild(config, bio, work.input, async () => work.build, now) + work.jobs.releaseResult(work.input.evaluationId, work.input.expectedReportHash) + finish(work.build) + expect(await first).toEqual(second) + expect(config.select().from(configSchema.biometricActivationEvents).all()).toHaveLength(1) + expect(config.select().from(configSchema.biometricConfigOutbox).all()).toHaveLength(1) +}) + +it('leaves activation state unchanged when build resolution fails', async () => { + const work = fixture() + await expect(requestValidatedActivationWithBuild(config, bio, work.input, async () => { + throw new Error('Worker missing') + })) + .rejects.toThrow('Worker missing') + expect(work.profiles.binding('left').generation).toBe(0) + expect(config.select().from(configSchema.biometricActivationEvents).all()).toEqual([]) +}) + +it('reviews qualified evidence without writing intent and still revalidates after review', async () => { + const work = fixture() + const reviewInput = validatedActivationReviewSchema.strip().parse(work.input) + expect(await reviewValidatedActivation(config, bio, reviewInput, async () => work.build)).toEqual({ eligible: true }) + expect(work.profiles.binding('left').generation).toBe(0) + expect(config.select().from(configSchema.biometricActivationEvents).all()).toEqual([]) + expect(config.select().from(configSchema.biometricConfigOutbox).all()).toEqual([]) + work.jobs.releaseResult(work.input.evaluationId, work.input.expectedReportHash) + await expect(requestValidatedActivationWithBuild(config, bio, work.input, async () => work.build)).rejects.toThrow() + await expect(reviewValidatedActivation(config, bio, reviewInput, async () => work.build)).rejects.toThrow() + expect(work.profiles.binding('left').generation).toBe(0) +}) diff --git a/src/biometrics/profiles/vital-provenance.ts b/src/biometrics/profiles/vital-provenance.ts new file mode 100644 index 000000000..5a6c3e3ec --- /dev/null +++ b/src/biometrics/profiles/vital-provenance.ts @@ -0,0 +1,34 @@ +import { z } from 'zod' + +export const vitalProvenanceSchema = z.object({ + interpretation: z.literal('original_observation'), + status: z.enum(['recorded', 'legacy_unknown', 'unavailable']), + processingRunId: z.string().nullable(), + configurationHash: z.string().nullable(), + hrvMetricId: z.string().nullable(), +}) +export type VitalProvenance = z.infer + +/** Describe recorded metadata only; today's manifest cannot identify old rows. */ +export function vitalProvenance(row: { + side: string + processingRunId?: string | null + recordedRunId?: string | null + recordedSide?: string | null + recordedKind?: string | null + recordedConfigHash?: string | null + recordedHrvMetricId?: unknown +}): VitalProvenance { + const id = row.processingRunId ?? null + const recorded = id !== null && row.recordedRunId === id && row.recordedSide === row.side + && row.recordedKind === 'live' && /^[a-f0-9]{64}$/.test(row.recordedConfigHash ?? '') + return { + interpretation: 'original_observation', + status: id === null ? 'legacy_unknown' : recorded ? 'recorded' : 'unavailable', + processingRunId: id, + configurationHash: recorded ? row.recordedConfigHash ?? null : null, + hrvMetricId: recorded && typeof row.recordedHrvMetricId === 'string' && /^[a-z0-9_]{1,80}$/.test(row.recordedHrvMetricId) + ? row.recordedHrvMetricId + : null, + } +} diff --git a/src/biometrics/profiles/vital-summary-provenance.ts b/src/biometrics/profiles/vital-summary-provenance.ts new file mode 100644 index 000000000..b902f794b --- /dev/null +++ b/src/biometrics/profiles/vital-summary-provenance.ts @@ -0,0 +1,48 @@ +import { z } from 'zod' +import { sql } from 'drizzle-orm' +import { biometricProcessingRuns as runs, vitals } from '@/src/db/biometrics-schema' + +// The same-side join is required by callers. A saved config hash alone does not +// identify an engine version, so configuration identity also includes its manifest. +const live = sql`(${runs.kind} = 'live' AND length(${runs.configHash}) = 64 AND ${runs.configHash} NOT GLOB '*[^a-f0-9]*')` +const manifest = sql`CASE WHEN json_valid(${runs.bundle}) THEN json_extract(${runs.bundle}, '$.manifestHash') END` +const rawMetric = sql`CASE WHEN json_valid(${runs.bundle}) THEN json_extract(${runs.bundle}, '$.components.hrv_index.metricId') END` +const metric = sql`CASE WHEN ${live} AND typeof(${rawMetric}) = 'text' AND length(${rawMetric}) BETWEEN 1 AND 80 AND ${rawMetric} NOT GLOB '*[^a-z0-9_]*' THEN ${rawMetric} END` +const configuration = sql`CASE WHEN ${live} AND typeof(${manifest}) = 'text' AND length(${manifest}) = 64 AND ${manifest} NOT GLOB '*[^a-f0-9]*' THEN ${manifest} || ':' || ${runs.configHash} END` +export const summaryProvenanceFields = { + configurationCount: sql`COUNT(DISTINCT ${configuration})`, + unknownConfigurationRows: sql`COUNT(CASE WHEN ${configuration} IS NULL THEN 1 END)`, + hrvRows: sql`COUNT(${vitals.hrv})`, + hrvDefinitionCount: sql`COUNT(DISTINCT CASE WHEN ${vitals.hrv} IS NOT NULL THEN ${metric} END)`, + unknownHrvRows: sql`COUNT(CASE WHEN ${vitals.hrv} IS NOT NULL AND ${metric} IS NULL THEN 1 END)`, + singleHrvMetricId: sql`MIN(CASE WHEN ${vitals.hrv} IS NOT NULL THEN ${metric} END)`, +} +export const summaryProvenanceSchema = z.object({ + interpretation: z.literal('original_observations'), + configurationCount: z.number().int().nonnegative(), + unknownConfigurationRows: z.number().int().nonnegative(), + hrvRows: z.number().int().nonnegative(), + hrvDefinitionCount: z.number().int().nonnegative(), + unknownHrvRows: z.number().int().nonnegative(), + hrvMetricId: z.string().nullable(), + hrvAggregation: z.enum(['single_definition', 'legacy_unverified', 'incompatible_definitions', 'no_readings']), +}) +export function summarizeVitalProvenance(row: { + configurationCount?: number + unknownConfigurationRows?: number + hrvRows?: number + hrvDefinitionCount?: number + unknownHrvRows?: number + singleHrvMetricId?: string | null +}) { + // Compatibility with adapters that have not supplied these new aggregate fields. + if ([row.configurationCount, row.unknownConfigurationRows, row.hrvRows, row.hrvDefinitionCount, row.unknownHrvRows].some(value => value === undefined)) return undefined + const incompatible = (row.hrvDefinitionCount ?? 0) > 1 || ((row.hrvDefinitionCount ?? 0) > 0 && (row.unknownHrvRows ?? 0) > 0) + return summaryProvenanceSchema.parse({ + interpretation: 'original_observations', configurationCount: row.configurationCount, + unknownConfigurationRows: row.unknownConfigurationRows, hrvRows: row.hrvRows, + hrvDefinitionCount: row.hrvDefinitionCount, unknownHrvRows: row.unknownHrvRows, + hrvMetricId: row.hrvDefinitionCount === 1 && row.unknownHrvRows === 0 ? row.singleHrvMetricId : null, + hrvAggregation: row.hrvRows === 0 ? 'no_readings' : incompatible ? 'incompatible_definitions' : row.hrvDefinitionCount === 0 ? 'legacy_unverified' : 'single_definition', + }) +} diff --git a/src/biometrics/references/alignment.ts b/src/biometrics/references/alignment.ts new file mode 100644 index 000000000..b07e0fa6b --- /dev/null +++ b/src/biometrics/references/alignment.ts @@ -0,0 +1,133 @@ +import type { NormalizedMeasurement, ReferenceMetric } from './contracts' +import { ReferenceInputError } from './time' + +export interface Observation { + timestampMs: number + value: number | null + /** Known support duration only; a point reading must not imply gap filling. */ + durationMs?: number +} +export interface AlignmentPolicy { + version: 'reference-alignment-v1' + pointToleranceMs: number + clockOffsetMs: number + offsetSource: 'fixed' | 'training' +} +export const DEFAULT_ALIGNMENT: AlignmentPolicy = { version: 'reference-alignment-v1', pointToleranceMs: 30000, clockOffsetMs: 0, offsetSource: 'fixed' } + +export function referenceCompatibility(metric: ReferenceMetric, windowMs: number | null, reference: NormalizedMeasurement): string | null { + if (!reference.scorable) return reference.exclusions.join(',') || 'unscorable_reference' + if (metric === 'hrv_unknown' || reference.metric === 'hrv_unknown') return 'unknown_hrv_definition' + if (reference.metric !== metric) return 'incompatible_metric_definition' + if (metric.startsWith('hrv_') || metric === 'window_ibi_variability_v1') { + if (windowMs === null || reference.windowMs === null || reference.windowMs !== windowMs) return 'incompatible_hrv_window' + } + return null +} + +function quantile(values: number[], q: number): number { + const sorted = [...values].sort((a, b) => a - b) + const index = (sorted.length - 1) * q + const lower = Math.floor(index) + return sorted[lower] + (sorted[Math.ceil(index)] - sorted[lower]) * (index - lower) +} + +function unionDuration(intervals: Array<[number, number]>): number { + let total = 0, end = -Infinity + for (const [start, stop] of intervals.sort((a, b) => a[0] - b[0])) { + total += Math.max(0, stop - Math.max(start, end)) + end = Math.max(end, stop) + } + return total +} + +/** + * Greedy chronological, nearest-unused point pairing; interval mean/median + * uses only actual pod samples inside the interval. No forward fill. Each + * observation and reference can contribute to at most one independent pair. + * Offset is supplied by a frozen protocol; never fitted on the evaluation set. + */ +export function alignReferences(metric: ReferenceMetric, windowMs: number | null, observations: Observation[], references: NormalizedMeasurement[], policy: AlignmentPolicy = DEFAULT_ALIGNMENT) { + if (metric === 'sleep_stages') throw new ReferenceInputError('INCOMPATIBLE_METRIC', 'Use the duration-weighted stage evaluator') + if (policy.version !== 'reference-alignment-v1' || !['fixed', 'training'].includes(policy.offsetSource) + || !Number.isFinite(policy.pointToleranceMs) || policy.pointToleranceMs < 0 || policy.pointToleranceMs > 300000 + || !Number.isFinite(policy.clockOffsetMs) || Math.abs(policy.clockOffsetMs) > 3600000) throw new ReferenceInputError('INVALID_ALIGNMENT', 'Unsupported alignment policy') + if (observations.length > 100000 || references.length > 100000) throw new ReferenceInputError('RESOURCE_LIMIT', 'Use bounded whole-night datasets') + for (const observation of observations) { + if (!Number.isFinite(observation.timestampMs) || (observation.value !== null && (!Number.isFinite(observation.value) || Math.abs(observation.value) > Number.MAX_SAFE_INTEGER)) + || (observation.durationMs !== undefined && (!Number.isFinite(observation.durationMs) || observation.durationMs < 0 || observation.durationMs > 300000))) { + throw new ReferenceInputError('INVALID_OBSERVATION', 'Observation timestamps, values and support durations must be finite and bounded') + } + } + const available = observations.filter(row => row.value !== null).map(row => ({ ...row, timestampMs: row.timestampMs + policy.clockOffsetMs })).sort((a, b) => a.timestampMs - b.timestampMs) + const used = new Set() + const duplicateKeys = new Set() + const exclusions: Record = {} + let eligibleReferences = 0, duplicateReferences = 0 + const pairs: Array<{ referenceKey: string, referenceStartMs: number, referenceEndMs: number, referenceValue: number, podValue: number, podCount: number, signedError: number, pairedDurationMs: number }> = [] + let comparisons = 0 + for (const reference of [...references].sort((a, b) => a.startMs - b.startMs || a.recordKey.localeCompare(b.recordKey))) { + const incompatible = referenceCompatibility(metric, windowMs, reference) + if (incompatible) { + exclusions[incompatible] = (exclusions[incompatible] ?? 0) + 1 + continue + } + if (reference.value === null || !Number.isFinite(reference.value) || Math.abs(reference.value) > Number.MAX_SAFE_INTEGER) throw new ReferenceInputError('INVALID_REFERENCE', 'Numeric reference value is required') + const key = JSON.stringify([reference.metric, reference.startMs, reference.endMs, reference.value, reference.aggregation, reference.windowMs]) + if (duplicateKeys.has(key)) { + duplicateReferences++ + continue + } + duplicateKeys.add(key) + eligibleReferences++ + const isPoint = reference.temporalKind === 'point' + const start = reference.startMs - (isPoint ? policy.pointToleranceMs : 0) + const end = reference.endMs + (isPoint ? policy.pointToleranceMs : 0) + let lower = 0, upper = available.length + while (lower < upper) { + const middle = Math.floor((lower + upper) / 2) + if (available[middle].timestampMs < start) lower = middle + 1 + else upper = middle + } + const candidates: number[] = [] + for (let i = lower; i < available.length; i++) { + if (++comparisons > 2000000) throw new ReferenceInputError('RESOURCE_LIMIT', 'Overlapping reference windows exceed the alignment work budget') + const timestamp = available[i].timestampMs + if (isPoint ? timestamp > end : timestamp >= end) break + if (!used.has(i)) candidates.push(i) + } + if (!candidates.length) continue + let selected = candidates + if (isPoint) { + selected = [candidates.reduce((best, index) => Math.abs(available[index].timestampMs - reference.startMs) < Math.abs(available[best].timestampMs - reference.startMs) ? index : best)] + } + const values = selected.map(index => available[index].value as number) + const podValue = reference.aggregation === 'median' ? quantile(values, 0.5) : values.reduce((sum, value) => sum + value, 0) / values.length + const support: Array<[number, number]> = [] + for (const index of selected) { + used.add(index) + const observation = available[index] + if (!isPoint && observation.durationMs) support.push([observation.timestampMs, Math.min(reference.endMs, observation.timestampMs + observation.durationMs)]) + } + pairs.push({ referenceKey: reference.recordKey, referenceStartMs: reference.startMs, referenceEndMs: reference.endMs, + referenceValue: reference.value, podValue, podCount: selected.length, signedError: podValue - reference.value, pairedDurationMs: unionDuration(support) }) + } + const errors = pairs.map(pair => Math.abs(pair.signedError)) + return { + status: !references.length ? 'no_reference' as const : pairs.length ? 'compared' as const : eligibleReferences ? 'no_overlap' as const : 'incompatible' as const, + metric, windowMs, policy: { ...policy }, + metrics: pairs.length + ? { mae: errors.reduce((sum, error) => sum + error, 0) / errors.length, + medianAbsoluteError: quantile(errors, 0.5), p95AbsoluteError: quantile(errors, 0.95), + signedBias: pairs.reduce((sum, pair) => sum + pair.signedError, 0) / pairs.length } + : null, + pairedCount: pairs.length, pairedDurationMs: pairs.reduce((sum, pair) => sum + pair.pairedDurationMs, 0), + eligibleReferences, duplicateReferences, excludedReferences: references.length - eligibleReferences - duplicateReferences, + referenceCoverage: eligibleReferences ? pairs.length / eligibleReferences : null, + outputCoverage: observations.length ? available.length / observations.length : null, + observationCount: observations.length, validObservationCount: available.length, matchedObservationCount: used.size, + unmatchedReferences: eligibleReferences - pairs.length, unmatchedObservations: available.length - used.size, + exclusions, pairs, + limitations: ['Sparse reference samples do not establish whole-night accuracy.', 'Point samples contribute counts, not inferred duration.', 'Interval comparisons use available observations without filling gaps.'], + } +} diff --git a/src/biometrics/references/comparison.ts b/src/biometrics/references/comparison.ts new file mode 100644 index 000000000..dbd59ab72 --- /dev/null +++ b/src/biometrics/references/comparison.ts @@ -0,0 +1,95 @@ +import { and, eq, gte, lt } from 'drizzle-orm' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import { z } from 'zod' +import * as schema from '@/src/db/biometrics-schema' +import { dataHash } from '@/src/biometrics/algorithms/canonical' +import { sideSchema } from '@/src/server/validation-schemas' +import { timestampMsSchema } from './contracts' +import { alignReferences, DEFAULT_ALIGNMENT } from './alignment' +import { ReferenceStore } from './store' +import { ReferenceInputError } from './time' + +export const compareReadingsSchema = z.object({ + side: sideSchema, + subjectId: z.string().min(1).max(200), + importId: z.string().uuid(), + metric: z.enum(['heart_rate', 'breathing_rate', 'window_ibi_variability_v1', 'hrv_rmssd', 'hrv_sdnn', 'hrv_unknown']), + startMs: timestampMsSchema, + endMs: timestampMsSchema, + pointToleranceMs: z.number().int().min(0).max(300000).default(30000), + clockOffsetMs: z.number().int().min(-3600000).max(3600000).default(0), +}).strict().refine(input => input.endMs > input.startMs && input.endMs - input.startMs <= 86400000, { + message: 'Select an interval greater than zero and at most 24 hours', +}) + +const numericScores = z.object({ mae: z.number(), medianAbsoluteError: z.number(), p95AbsoluteError: z.number(), signedBias: z.number() }) +export const comparisonReportSchema = z.object({ + version: z.literal('stored-readings-comparison-v1'), + inputHash: z.string(), + side: sideSchema, subjectId: z.string(), startMs: z.number(), endMs: z.number(), + importId: z.string().uuid(), importRevision: z.number(), source: z.string(), + status: z.enum(['no_reference', 'compared', 'no_overlap', 'incompatible']), + requestedMetric: z.string(), metric: z.string(), windowMs: z.number().nullable(), + policy: z.object({ version: z.literal('reference-alignment-v1'), pointToleranceMs: z.number(), clockOffsetMs: z.number(), offsetSource: z.literal('fixed') }), + metrics: numericScores.nullable(), pairedCount: z.number(), pairedDurationMs: z.number(), + eligibleReferences: z.number(), duplicateReferences: z.number(), excludedReferences: z.number(), + referenceCoverage: z.number().nullable(), outputCoverage: z.number().nullable(), + observationCount: z.number(), validObservationCount: z.number(), matchedObservationCount: z.number(), + unmatchedReferences: z.number(), unmatchedObservations: z.number(), exclusions: z.record(z.string(), z.number()), + provenance: z.array(z.object({ processingRunId: z.string().nullable(), count: z.number() })), + limitations: z.array(z.string()), +}) + +/** Bounded, read-only comparison. Saved evaluations instead retain their input bytes. */ +export function compareStoredReadings(db: BetterSQLite3Database, raw: z.input) { + const input = compareReadingsSchema.parse(raw) + const store = new ReferenceStore(db) + // Capture all input rows in a single read snapshot, then release SQLite before scoring. + const snapshot = db.transaction(() => { + const reference = store.get(input.importId) + if (!reference.available) throw new ReferenceInputError('REFERENCE_DELETED', 'Reference measurements were explicitly deleted') + if (reference.summary.side !== input.side || reference.summary.subjectId !== input.subjectId) throw new ReferenceInputError('SUBJECT_MISMATCH', 'Select the import matching this side and sleeper') + const measurements = [] + let cursor = 0 + do { + const page = store.measurements(input.importId, cursor) + measurements.push(...page.items) + if (page.nextCursor === null) break + cursor = page.nextCursor + } while (measurements.length <= 2000) + if (measurements.length > 2000) throw new ReferenceInputError('RESOURCE_LIMIT', 'Reference import exceeds comparison budget') + const rows = db.select().from(schema.vitals).where(and(eq(schema.vitals.side, input.side), + gte(schema.vitals.timestamp, new Date(Math.ceil(input.startMs / 1000) * 1000)), + lt(schema.vitals.timestamp, new Date(Math.ceil(input.endMs / 1000) * 1000)), + )).orderBy(schema.vitals.timestamp).limit(100001).all() + if (rows.length > 100000) throw new ReferenceInputError('RESOURCE_LIMIT', 'Select a shorter comparison interval') + return { reference, measurements, rows } + }) + const references = snapshot.measurements.filter(row => row.metric === input.metric + && (row.temporalKind === 'point' + ? row.startMs >= input.startMs && row.startMs < input.endMs + : row.startMs >= input.startMs && row.endMs <= input.endMs)) + // HRV in canonical vitals is exclusively the window IBI variability index. + // A requested RMSSD/SDNN comparison therefore remains incompatible. + const podMetric = input.metric.startsWith('hrv_') ? 'window_ibi_variability_v1' : input.metric + const windowMs = podMetric === 'window_ibi_variability_v1' ? 300000 : null + const observations = snapshot.rows.map(row => ({ timestampMs: row.timestamp.getTime(), value: + podMetric === 'heart_rate' ? row.heartRate : podMetric === 'breathing_rate' ? row.breathingRate : row.hrv })) + const alignment = alignReferences(podMetric, windowMs, observations, references, + { ...DEFAULT_ALIGNMENT, pointToleranceMs: input.pointToleranceMs, clockOffsetMs: input.clockOffsetMs }) + const provenance = new Map() + for (const row of snapshot.rows) provenance.set(row.processingRunId, (provenance.get(row.processingRunId) ?? 0) + 1) + const inputHash = dataHash({ input, referenceHash: snapshot.reference.summary.contentHash, + observations: snapshot.rows.map(row => ({ ...row, timestamp: row.timestamp.getTime() })) }) + return comparisonReportSchema.parse({ ...alignment, requestedMetric: input.metric, version: 'stored-readings-comparison-v1', inputHash, + side: input.side, subjectId: input.subjectId, startMs: input.startMs, endMs: input.endMs, + importId: input.importId, importRevision: snapshot.reference.revision, source: snapshot.reference.summary.source, + provenance: [...provenance].map(([processingRunId, count]) => ({ processingRunId, count })), + limitations: [...alignment.limitations, + 'This compares original stored readings; it does not regenerate signals or validate a candidate.', + 'The input hash identifies this read snapshot; inputs are not retained until a dataset is frozen.', + 'Stored readings have no recorded support duration, so paired duration is unknown and reported as zero.', + 'Reference intervals crossing the selected boundaries are excluded to preserve their aggregation semantics.', + ...(provenance.has(null) ? ['Some observations have legacy_unknown processing provenance.'] : [])], + }) +} diff --git a/src/biometrics/references/contracts.ts b/src/biometrics/references/contracts.ts new file mode 100644 index 000000000..8fbdd1e7b --- /dev/null +++ b/src/biometrics/references/contracts.ts @@ -0,0 +1,95 @@ +import { z } from 'zod' +import { sideSchema } from '@/src/server/validation-schemas' + +export const REFERENCE_ADAPTER_VERSION = 'normalized-reference-v1' +export const REFERENCE_LIMITS = { bytes: 2 * 1024 * 1024, rows: 2000, depth: 12, nodes: 100000, rejectionExamples: 25 } as const +export const referenceMetricSchema = z.enum(['heart_rate', 'breathing_rate', 'window_ibi_variability_v1', 'hrv_rmssd', 'hrv_sdnn', 'hrv_unknown', 'sleep_stages']) +export type ReferenceMetric = z.infer +export const referenceStageSchema = z.enum(['wake', 'light', 'deep', 'rem', 'asleep', 'in_bed', 'unknown']) +export const timestampMsSchema = z.number().int().min(1_000_000_000_000).max(4_102_444_800_000) +export const timePolicySchema = z.object({ + timezone: z.string().min(1).max(100).optional(), + utcOffsetMinutes: z.number().int().min(-840).max(840).optional(), + disambiguation: z.enum(['reject', 'earlier', 'later']).default('reject'), +}).strict() + +export const referenceRowSchema = z.object({ + recordKey: z.string().min(1).max(200), + subjectId: z.string().min(1).max(200).optional(), + metric: referenceMetricSchema, + value: z.union([z.number().finite(), referenceStageSchema]), + unit: z.string().min(1).max(40), + timestampMs: timestampMsSchema.optional(), + startMs: timestampMsSchema.optional(), + endMs: timestampMsSchema.optional(), + // Offset-bearing ISO, or an explicitly resolved local time under timePolicy. + timestamp: z.string().max(80).optional(), + start: z.string().max(80).optional(), + end: z.string().max(80).optional(), + aggregation: z.enum(['instant', 'mean', 'median', 'window_index', 'stage']).optional(), + windowMs: z.number().int().min(1).max(24 * 60 * 60 * 1000).optional(), + quality: z.number().finite().min(0).max(1).optional(), + annotation: z.string().max(1000).optional(), +}).strict() +export type ReferenceRow = z.infer + +export const referenceImportSchema = z.object({ + side: sideSchema, + subjectId: z.string().min(1).max(200), + source: z.string().min(1).max(120), + timePolicy: timePolicySchema.default({ disambiguation: 'reject' }), + rows: z.array(z.unknown()).min(1).max(REFERENCE_LIMITS.rows), +}).strict() +export type ReferenceImportInput = z.input + +export const normalizedMeasurementSchema = z.object({ + recordKey: z.string(), + metric: referenceMetricSchema, + value: z.number().nullable(), + stage: referenceStageSchema.nullable(), + unit: z.string(), + startMs: timestampMsSchema, + endMs: timestampMsSchema, + temporalKind: z.enum(['point', 'interval']), + aggregation: z.enum(['instant', 'mean', 'median', 'window_index', 'stage']), + windowMs: z.number().nullable(), + quality: z.number().nullable(), + scorable: z.boolean(), + exclusions: z.array(z.string()), + // Retain the submitted units, values, source times and annotations as data. + original: referenceRowSchema, +}).strict() +export type NormalizedMeasurement = z.infer + +export const previewSummarySchema = z.object({ + contentHash: z.string(), + adapterVersion: z.string(), + side: sideSchema, + subjectId: z.string(), + source: z.string(), + timePolicy: timePolicySchema, + received: z.number(), + accepted: z.number(), + rejected: z.number(), + duplicates: z.number(), + unscorable: z.number(), + conflicts: z.number(), + startMs: z.number().nullable(), + endMs: z.number().nullable(), + metrics: z.array(referenceMetricSchema), + rejectionCounts: z.record(z.string(), z.number()), + rejectionExamples: z.array(z.object({ row: z.number(), code: z.string(), message: z.string() })), + warnings: z.array(z.string()), +}) +export type PreviewSummary = z.infer + +export const referenceImportDescriptorSchema = z.object({ + id: z.string().uuid(), + familyId: z.string().uuid(), + revision: z.number().int(), + supersedesImportId: z.string().uuid().nullable(), + summary: previewSummarySchema, + createdAtMs: z.number(), + deletedAtMs: z.number().nullable(), + available: z.boolean(), +}) diff --git a/src/biometrics/references/legacy.ts b/src/biometrics/references/legacy.ts new file mode 100644 index 000000000..392c8a83a --- /dev/null +++ b/src/biometrics/references/legacy.ts @@ -0,0 +1,29 @@ +import { inArray } from 'drizzle-orm' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import * as schema from '@/src/db/biometrics-schema' +import type { ReferenceImportInput } from './contracts' +import { previewReferenceImport } from './normalize' +import { ReferenceStore } from './store' + +type LegacyVital = typeof schema.referenceVitals.$inferSelect +type LegacyStage = typeof schema.referenceStages.$inferSelect + +/** + * Called inside the existing legacy endpoint's insertion transaction. Only + * newly accepted rows get provenance; first-write-wins and stage-overlap + * rejection retain their old semantics. Historical unlinked rows stay unknown. + */ +export function recordLegacyReferenceImport(db: BetterSQLite3Database, kind: 'vitals' | 'stages', rows: Array): void { + const first = rows[0] + if (!first) return + const annotation = 'Legacy Apple Watch API: explicit side mapping; sleeper identity and source timezone were not provided. Original timestamps use Unix seconds.' + const input: ReferenceImportInput = { side: first.side, subjectId: `legacy_unknown:${first.side}`, source: first.source, + rows: rows.map(row => 'heartRate' in row + ? { recordKey: `apple-watch-hr:${row.timestamp.getTime()}`, metric: 'heart_rate', value: row.heartRate, unit: 'bpm', timestampMs: row.timestamp.getTime(), annotation } + : { recordKey: `apple-watch-stage:${row.start.getTime()}`, metric: 'sleep_stages', value: row.stage, unit: 'stage', startMs: row.start.getTime(), endMs: row.end.getTime(), annotation }), + } + const preview = previewReferenceImport(input) + const result = new ReferenceStore(db).commit({ import: input, previewHash: preview.summary.contentHash }) + const table = kind === 'vitals' ? schema.referenceVitals : schema.referenceStages + for (let i = 0; i < rows.length; i += 100) db.update(table).set({ importId: result.id }).where(inArray(table.id, rows.slice(i, i + 100).map(row => row.id))).run() +} diff --git a/src/biometrics/references/normalize-core.ts b/src/biometrics/references/normalize-core.ts new file mode 100644 index 000000000..86cf8b17a --- /dev/null +++ b/src/biometrics/references/normalize-core.ts @@ -0,0 +1,151 @@ +import { canonicalJson } from '../algorithms/canonical-json' +const utf8 = new TextEncoder() +import { REFERENCE_ADAPTER_VERSION, REFERENCE_LIMITS, referenceImportSchema, referenceRowSchema, type NormalizedMeasurement, type PreviewSummary, type ReferenceImportInput } from './contracts' +import { ReferenceInputError, timestampResolver } from './time' + +/** Validate size/depth before copying, hashing or parsing untrusted exports. */ +export function assertBoundedReferenceData(input: unknown): void { + const stack = [{ value: input, depth: 0 }] + const seen = new WeakSet() + let nodes = 0, stringBytes = 0 + while (stack.length) { + const entry = stack.pop() + if (!entry) break + const { value, depth } = entry + if (++nodes > REFERENCE_LIMITS.nodes || depth > REFERENCE_LIMITS.depth) throw new ReferenceInputError('RESOURCE_LIMIT', 'Reference JSON is too large or deeply nested') + if (typeof value === 'string') stringBytes += utf8.encode(value).byteLength + else if (typeof value === 'number') { + if (!Number.isFinite(value)) throw new ReferenceInputError('NON_FINITE_VALUE', 'Reference values must be finite JSON numbers') + } + else if (value !== null && typeof value === 'object') { + if (seen.has(value)) throw new ReferenceInputError('INVALID_JSON', 'Reference input must be a JSON tree') + seen.add(value) + if (!Array.isArray(value) && Object.getPrototypeOf(value) !== Object.prototype && Object.getPrototypeOf(value) !== null) throw new ReferenceInputError('INVALID_JSON', 'Reference input must contain plain JSON objects') + for (const [key, child] of Object.entries(value)) { + stringBytes += utf8.encode(key).byteLength + stack.push({ value: child, depth: depth + 1 }) + } + } + else if (value !== null && typeof value !== 'boolean') throw new ReferenceInputError('INVALID_JSON', 'Reference input contains a non-JSON value') + if (stringBytes > REFERENCE_LIMITS.bytes) throw new ReferenceInputError('RESOURCE_LIMIT', 'Reference upload exceeds 2 MiB') + } + if (utf8.encode(JSON.stringify(input)).byteLength > REFERENCE_LIMITS.bytes) throw new ReferenceInputError('RESOURCE_LIMIT', 'Reference upload exceeds 2 MiB') +} + +function normalizeRow(raw: unknown, subjectId: string, resolveTime: (value: string | number) => number): NormalizedMeasurement { + const row = referenceRowSchema.parse(raw) + if (row.subjectId !== undefined && row.subjectId !== subjectId) throw new ReferenceInputError('SUBJECT_MISMATCH', 'Row belongs to a different subject; split the export and map each sleeper explicitly') + const point = row.timestampMs ?? row.timestamp + const start = row.startMs ?? row.start + const end = row.endMs ?? row.end + if ((row.timestampMs !== undefined && row.timestamp !== undefined) + || (row.startMs !== undefined && row.start !== undefined) || (row.endMs !== undefined && row.end !== undefined) + || (point !== undefined && (start !== undefined || end !== undefined))) { + throw new ReferenceInputError('AMBIGUOUS_TIME_FIELDS', 'Specify a single point or interval time representation') + } + const temporalKind = point !== undefined ? 'point' : 'interval' + if (point === undefined && (start === undefined || end === undefined)) throw new ReferenceInputError('MISSING_TIMESTAMP', 'Provide timestampMs or startMs/endMs, or ISO equivalents') + const startMs = resolveTime(point ?? start ?? '') + const endMs = point !== undefined ? startMs : resolveTime(end ?? '') + if (temporalKind === 'interval' && endMs <= startMs) throw new ReferenceInputError('INVALID_INTERVAL', 'end must be after start') + if (endMs - startMs > 24 * 60 * 60 * 1000) throw new ReferenceInputError('INVALID_INTERVAL', 'A reference interval may span at most 24 hours') + const aggregation = row.aggregation ?? (row.metric === 'sleep_stages' ? 'stage' : temporalKind === 'point' ? 'instant' : 'mean') + if ((temporalKind === 'interval' && aggregation === 'instant') || (row.metric !== 'sleep_stages' && aggregation === 'stage')) throw new ReferenceInputError('INVALID_AGGREGATION', 'Aggregation does not match the measurement semantics') + const exclusions: string[] = [] + let value: number | null = null, stage: NormalizedMeasurement['stage'] = null, unit: string + if (row.metric === 'sleep_stages') { + if (typeof row.value !== 'string' || row.unit !== 'stage' || temporalKind !== 'interval' || aggregation !== 'stage') throw new ReferenceInputError('INCOMPATIBLE_METRIC', 'Sleep stages require a labeled interval with unit stage') + stage = row.value + unit = 'stage' + if (!['wake', 'light', 'deep', 'rem'].includes(stage)) exclusions.push('unclassified_stage') + } + else { + if (typeof row.value !== 'number') throw new ReferenceInputError('INCOMPATIBLE_METRIC', 'Vital measurements require numeric values') + const units: Record = row.metric === 'heart_rate' + ? { 'bpm': 1, 'beats/min': 1, 'Hz': 60 } + : row.metric === 'breathing_rate' + ? { 'breaths/min': 1, 'br/min': 1, 'Hz': 60 } + : { ms: 1, s: 1000 } + if (!Object.hasOwn(units, row.unit)) throw new ReferenceInputError('UNSUPPORTED_UNIT', `Unsupported unit for ${row.metric}`) + value = row.value * units[row.unit] + if (!Number.isFinite(value) || value < 0 || value > Number.MAX_SAFE_INTEGER) throw new ReferenceInputError('INVALID_VALUE', 'Measurements must be finite, nonnegative and within the safe numeric range after conversion') + unit = row.metric === 'heart_rate' ? 'bpm' : row.metric === 'breathing_rate' ? 'breaths/min' : 'ms' + if (row.metric === 'hrv_unknown') exclusions.push('unknown_hrv_definition') + if (row.metric === 'window_ibi_variability_v1' && (row.windowMs !== 300000 || aggregation !== 'window_index')) exclusions.push('incompatible_hrv_window') + } + return { recordKey: row.recordKey, metric: row.metric, value, stage, unit, startMs, endMs, temporalKind, + aggregation, windowMs: row.windowMs ?? null, quality: row.quality ?? null, + scorable: exclusions.length === 0, exclusions, original: row } +} + +export function normalizeReferenceImport(raw: ReferenceImportInput): { summary: Omit, measurements: NormalizedMeasurement[], uploadJson: string } { + assertBoundedReferenceData(raw) + const input = referenceImportSchema.parse(raw) + const resolveTime = timestampResolver(input.timePolicy) + const rejected = new Map() + const normalized: Array<{ row: number, measurement: NormalizedMeasurement }> = [] + for (const [index, value] of input.rows.entries()) { + try { + const measurement = normalizeRow(value, input.subjectId, resolveTime) + canonicalJson(measurement) + normalized.push({ row: index, measurement }) + } + catch (error) { + rejected.set(index, error instanceof ReferenceInputError + ? { code: error.code, message: error.message } + : { code: 'INVALID_ROW', message: 'Row does not match the documented reference schema; check fields, types and millisecond timestamps' }) + } + } + const byKey = new Map() + for (const entry of normalized) { + const group = byKey.get(entry.measurement.recordKey) ?? [] + group.push(entry) + byKey.set(entry.measurement.recordKey, group) + } + let duplicates = 0 + const measurements: NormalizedMeasurement[] = [] + for (const group of byKey.values()) { + const fingerprints = new Set(group.map(entry => canonicalJson(entry.measurement))) + if (fingerprints.size > 1) { + for (const entry of group) rejected.set(entry.row, { code: 'CONFLICTING_RECORD_KEY', message: 'The same source record key has different measurements in this batch' }) + } + else { + measurements.push(group[0].measurement) + duplicates += group.length - 1 + } + } + // Persist all conflicting intervals, but exclude both sides of a conflict. + // Source priority is never guessed; unknown/generic asleep is never light. + const stages = measurements.filter(row => row.metric === 'sleep_stages').sort((a, b) => a.startMs - b.startMs) + let active: NormalizedMeasurement[] = [] + let conflicts = 0 + for (const row of stages) { + active = active.filter(previous => previous.endMs > row.startMs) + for (const previous of active) { + if (row.stage !== previous.stage) { + conflicts++ + for (const conflicting of [previous, row]) { + conflicting.scorable = false + if (!conflicting.exclusions.includes('overlapping_stage_conflict')) conflicting.exclusions.push('overlapping_stage_conflict') + } + } + } + active.push(row) + } + const rejectionCounts: Record = {} + for (const rejection of rejected.values()) rejectionCounts[rejection.code] = (rejectionCounts[rejection.code] ?? 0) + 1 + const warnings = ['Agreement with a reference source is not clinical accuracy.', 'Stage overlaps with conflicting labels are excluded from scoring.'] + if (measurements.some(row => ['hrv_rmssd', 'hrv_sdnn', 'hrv_unknown'].includes(row.metric))) warnings.push('RMSSD, SDNN and unknown HRV definitions cannot directly score the pod window-level IBI variability index.') + // Hash the exact bounded upload plus interpretation. No arbitrary paths or URLs. + const uploadJson = JSON.stringify({ adapterVersion: REFERENCE_ADAPTER_VERSION, ...input }, (_key, value) => value !== null && typeof value === 'object' && !Array.isArray(value) ? Object.fromEntries(Object.keys(value).sort().map(key => [key, value[key]])) : value) + return { measurements, uploadJson, summary: { + adapterVersion: REFERENCE_ADAPTER_VERSION, side: input.side, subjectId: input.subjectId, + source: input.source, timePolicy: input.timePolicy, received: input.rows.length, + accepted: measurements.length, rejected: rejected.size, duplicates, + unscorable: measurements.filter(row => !row.scorable).length, conflicts, + startMs: measurements.length ? Math.min(...measurements.map(row => row.startMs)) : null, + endMs: measurements.length ? Math.max(...measurements.map(row => row.endMs)) : null, + metrics: [...new Set(measurements.map(row => row.metric))].sort(), rejectionCounts, + rejectionExamples: [...rejected].slice(0, REFERENCE_LIMITS.rejectionExamples).map(([row, error]) => ({ row: row + 1, ...error })), warnings, + } } +} diff --git a/src/biometrics/references/normalize.ts b/src/biometrics/references/normalize.ts new file mode 100644 index 000000000..412cf5c31 --- /dev/null +++ b/src/biometrics/references/normalize.ts @@ -0,0 +1,13 @@ +import { createHash } from 'node:crypto' +import { configHash } from '../algorithms/canonical' +import { REFERENCE_ADAPTER_VERSION, type ReferenceImportInput } from './contracts' +import { normalizeReferenceImport } from './normalize-core' +export { assertBoundedReferenceData } from './normalize-core' + +export function previewReferenceImport(raw: ReferenceImportInput) { + const result = normalizeReferenceImport(raw) + const uploadHash = createHash('sha256').update(result.uploadJson).digest('hex') + const contentHash = configHash({ adapterVersion: REFERENCE_ADAPTER_VERSION, uploadHash, + measurementHashes: result.measurements.map(configHash), rejectionCounts: result.summary.rejectionCounts }) + return { measurements: result.measurements, summary: { ...result.summary, contentHash } } +} diff --git a/src/biometrics/references/parse.ts b/src/biometrics/references/parse.ts new file mode 100644 index 000000000..077ab8ee2 --- /dev/null +++ b/src/biometrics/references/parse.ts @@ -0,0 +1,73 @@ +import { REFERENCE_LIMITS } from './contracts' +import { ReferenceInputError } from './time' + +const columns = new Set(['recordKey', 'subjectId', 'metric', 'value', 'unit', 'timestampMs', 'startMs', 'endMs', 'timestamp', 'start', 'end', 'aggregation', 'windowMs', 'quality', 'annotation']) +const numeric = new Set(['timestampMs', 'startMs', 'endMs', 'windowMs', 'quality', 'value']) + +/** Text only. Never reads a path/URL, expands an archive or evaluates a cell. */ +export function parseReferenceText(format: 'csv' | 'json', text: string): unknown[] { + if (new TextEncoder().encode(text).byteLength > REFERENCE_LIMITS.bytes) throw new ReferenceInputError('RESOURCE_LIMIT', 'Reference upload exceeds 2 MiB') + if (format === 'json') { + let rows: unknown + try { + rows = JSON.parse(text) + } + catch { throw new ReferenceInputError('INVALID_JSON', 'Expected a JSON array of reference records') } + if (!Array.isArray(rows) || rows.length > REFERENCE_LIMITS.rows) throw new ReferenceInputError('RESOURCE_LIMIT', 'Expected at most 2000 reference records') + return rows + } + const records: string[][] = [] + let record: string[] = [], cell = '', quoted = false, closed = false + const input = text.replace(/^\uFEFF/, '') + function endCell() { + record.push(cell) + if (record.length > columns.size) throw new ReferenceInputError('RESOURCE_LIMIT', 'Too many CSV columns') + cell = '' + closed = false + } + function endRecord() { + endCell() + records.push(record) + record = [] + if (records.length > REFERENCE_LIMITS.rows + 1) throw new ReferenceInputError('RESOURCE_LIMIT', 'Expected at most 2000 reference records') + } + for (let i = 0; i < input.length; i++) { + const char = input[i] + if (quoted) { + if (char === '"') { + if (input[i + 1] === '"') { + cell += '"' + i++ + } + else { + quoted = false + closed = true + } + } + else cell += char + } + else if (char === ',') endCell() + else if (char === '\n' || char === '\r') { + if (char === '\r' && input[i + 1] === '\n') i++ + endRecord() + } + else if (closed) throw new ReferenceInputError('INVALID_CSV', 'Unexpected data after a quoted cell') + else if (char === '"') { + if (cell.length) throw new ReferenceInputError('INVALID_CSV', 'A quote must start a CSV cell') + quoted = true + } + else cell += char + } + if (quoted) throw new ReferenceInputError('INVALID_CSV', 'Unterminated quoted CSV cell') + if (cell.length || record.length || closed) endRecord() + const header = records.shift() + if (!header || new Set(header).size !== header.length || header.some(name => !columns.has(name))) throw new ReferenceInputError('INVALID_CSV', 'CSV headers must be unique documented field names') + return records.map((values) => { + if (values.length !== header.length) throw new ReferenceInputError('INVALID_CSV', 'CSV row has a different number of fields than the header') + return Object.fromEntries(header.flatMap((key, index) => { + const value = values[index] + if (value === '') return [] + return [[key, numeric.has(key) && /^-?(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?$/.test(value) ? Number(value) : value]] + })) + }) +} diff --git a/src/biometrics/references/stage-scoring.ts b/src/biometrics/references/stage-scoring.ts new file mode 100644 index 000000000..34e199d3e --- /dev/null +++ b/src/biometrics/references/stage-scoring.ts @@ -0,0 +1,102 @@ +import type { SleepStage, SleepStageBlock } from '@/src/lib/sleep-stages' +import type { NormalizedMeasurement } from './contracts' +import { ReferenceInputError } from './time' + +const STAGES: SleepStage[] = ['wake', 'light', 'deep', 'rem'] +const emptyStages = () => ({ wake: 0, light: 0, deep: 0, rem: 0 }) + +/** Duration scoring on actual intersecting support, never midpoint labels or gap fill. */ +export function scoreStageDurations(predictions: SleepStageBlock[], references: NormalizedMeasurement[], startMs: number, endMs: number) { + if (!Number.isSafeInteger(startMs) || !Number.isSafeInteger(endMs) || endMs <= startMs || endMs - startMs > 86400000 + || predictions.length > 100000 || references.length > 100000) throw new ReferenceInputError('RESOURCE_LIMIT', 'Use bounded stage intervals within one day') + const predicted = predictions.map(row => ({ ...row })).sort((a, b) => a.start - b.start) + for (let i = 0; i < predicted.length; i++) { + const row = predicted[i] + if (!Number.isSafeInteger(row.start) || !Number.isSafeInteger(row.end) || row.end <= row.start || !STAGES.includes(row.stage) + || (i > 0 && predicted[i - 1].end > row.start)) throw new ReferenceInputError('INVALID_OBSERVATION', 'Predicted stage intervals must be valid and nonoverlapping') + } + const exclusions: Record = {} + const eligible: SleepStageBlock[] = [] + for (const row of references) { + if (row.endMs <= startMs || row.startMs >= endMs) continue + const reason = row.metric !== 'sleep_stages' + ? 'incompatible_metric_definition' + : !row.scorable + ? row.exclusions.join(',') || 'unscorable_reference' + : row.stage === null || !STAGES.includes(row.stage as SleepStage) ? 'unknown_stage' : null + if (reason) { + exclusions[reason] = (exclusions[reason] ?? 0) + 1 + continue + } + if (!Number.isSafeInteger(row.startMs) || !Number.isSafeInteger(row.endMs) || row.endMs <= row.startMs) throw new ReferenceInputError('INVALID_REFERENCE', 'Stage references require valid intervals') + eligible.push({ start: Math.max(startMs, row.startMs), end: Math.min(endMs, row.endMs), stage: row.stage as SleepStage }) + } + // Sweep all boundaries: duplicate equal labels collapse; conflicting labels + // leave that exact interval unscored, including conflicts across imports. + const events = new Map>() + for (const row of eligible) { + if (!events.has(row.start)) events.set(row.start, []) + if (!events.has(row.end)) events.set(row.end, []) + events.get(row.start)?.push({ stage: row.stage, delta: 1 }) + events.get(row.end)?.push({ stage: row.stage, delta: -1 }) + } + const boundaries = [...events.keys()].sort((a, b) => a - b) + const active = emptyStages() + const normalized: SleepStageBlock[] = [] + let conflictingReferenceMs = 0 + for (let i = 0; i < boundaries.length - 1; i++) { + const start = boundaries[i], end = boundaries[i + 1] + for (const event of events.get(start) ?? []) active[event.stage] += event.delta + const labels = STAGES.filter(stage => active[stage] > 0) + if (labels.length > 1) conflictingReferenceMs += end - start + else if (labels.length === 1) normalized.push({ start, end, stage: labels[0] }) + } + const confusionMs = { wake: emptyStages(), light: emptyStages(), deep: emptyStages(), rem: emptyStages() } + const predictedDurationMs = emptyStages(), referenceDurationMs = emptyStages() + let pairedDurationMs = 0, p = 0, r = 0 + const common: Array<{ start: number, end: number, predicted: SleepStage, reference: SleepStage }> = [] + while (p < predicted.length && r < normalized.length) { + const pod = predicted[p], ref = normalized[r] + const start = Math.max(startMs, pod.start, ref.start), end = Math.min(endMs, pod.end, ref.end) + if (end > start) { + const duration = end - start + confusionMs[ref.stage][pod.stage] += duration + predictedDurationMs[pod.stage] += duration + referenceDurationMs[ref.stage] += duration + pairedDurationMs += duration + common.push({ start, end, predicted: pod.stage, reference: ref.stage }) + } + if (pod.end <= ref.end) p++ + else r++ + } + const perStage = STAGES.map((stage) => { + const truePositive = confusionMs[stage][stage] + const precision = predictedDurationMs[stage] ? truePositive / predictedDurationMs[stage] : null + const recall = referenceDurationMs[stage] ? truePositive / referenceDurationMs[stage] : null + const denominator = predictedDurationMs[stage] + referenceDurationMs[stage] + return { stage, precision, recall, f1: denominator ? 2 * truePositive / denominator : null, + supportMs: referenceDurationMs[stage], predictedMs: predictedDurationMs[stage], durationDifferenceMs: predictedDurationMs[stage] - referenceDurationMs[stage] } + }) + const definedF1 = perStage.flatMap(row => row.f1 === null ? [] : [row.f1]) + let predictedTransitions = 0, referenceTransitions = 0 + for (let i = 1; i < common.length; i++) { + if (common[i - 1].end !== common[i].start) continue // never infer a transition across a gap + if (common[i - 1].predicted !== common[i].predicted) predictedTransitions++ + if (common[i - 1].reference !== common[i].reference) referenceTransitions++ + } + const eligibleReferenceMs = normalized.reduce((total, row) => total + row.end - row.start, 0) + const outputDurationMs = predicted.reduce((total, row) => total + Math.max(0, Math.min(endMs, row.end) - Math.max(startMs, row.start)), 0) + return { version: 'duration-stage-agreement-v1' as const, + status: pairedDurationMs ? 'compared' as const : eligibleReferenceMs ? 'no_overlap' as const : 'no_scorable_reference' as const, + confusionMs, perStage, macroF1: definedF1.length ? definedF1.reduce((sum, value) => sum + value, 0) / definedF1.length : null, + agreement: pairedDurationMs ? STAGES.reduce((sum, stage) => sum + confusionMs[stage][stage], 0) / pairedDurationMs : null, + pairedDurationMs, eligibleReferenceMs, conflictingReferenceMs, outputDurationMs, + unscoredDurationMs: endMs - startMs - pairedDurationMs, + referenceCoverage: eligibleReferenceMs ? pairedDurationMs / eligibleReferenceMs : null, + outputCoverage: outputDurationMs / (endMs - startMs), + predictedTransitions, referenceTransitions, transitionDifference: predictedTransitions - referenceTransitions, + exclusions, limitations: ['Agreement is with the named reference, not clinical stage accuracy.', + 'Stage duration and transition differences use common scored support only.', + 'Macro F1 excludes classes absent from both outputs and references; inspect class support.'], + } +} diff --git a/src/biometrics/references/store.ts b/src/biometrics/references/store.ts new file mode 100644 index 000000000..85b6e8927 --- /dev/null +++ b/src/biometrics/references/store.ts @@ -0,0 +1,104 @@ +import { randomUUID } from 'node:crypto' +import { and, desc, eq, gt, isNull, lt, or } from 'drizzle-orm' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import { z } from 'zod' +import * as schema from '@/src/db/biometrics-schema' +import { referenceImports as imports, referenceMeasurements as measurements } from '@/src/db/biometrics-schema' +import { normalizedMeasurementSchema, previewSummarySchema, referenceImportSchema } from './contracts' +import { previewReferenceImport } from './normalize' +import { ReferenceInputError } from './time' + +export const commitReferenceSchema = z.object({ + import: referenceImportSchema, + previewHash: z.string().regex(/^[a-f0-9]{64}$/), + acceptRejectedRows: z.boolean().default(false), + supersedesImportId: z.string().uuid().optional(), +}).strict() + +type BioDB = BetterSQLite3Database +type ImportRecord = typeof imports.$inferSelect + +function describeImport(row: ImportRecord) { + return { id: row.id, familyId: row.familyId, revision: row.revision, + supersedesImportId: row.supersedesImportId, summary: previewSummarySchema.parse(row.summary), + createdAtMs: row.createdAtMs, deletedAtMs: row.deletedAtMs, available: row.deletedAtMs === null } +} + +/** Immutable normalized records. Corrections never update an earlier revision. */ +export class ReferenceStore { + constructor(private readonly db: BioDB, private readonly now = Date.now) {} + + commit(raw: z.input) { + const input = commitReferenceSchema.parse(raw) + const preview = previewReferenceImport(input.import) + if (preview.summary.contentHash !== input.previewHash) throw new ReferenceInputError('STALE_PREVIEW', 'Import bytes or interpretation changed; preview the exact import again') + if (preview.summary.rejected && !input.acceptRejectedRows) throw new ReferenceInputError('REJECTED_ROWS', 'Resolve rejected rows or explicitly accept a partial import') + if (!preview.measurements.length) throw new ReferenceInputError('NO_ACCEPTED_ROWS', 'There are no accepted measurements to import') + return this.db.transaction((tx) => { + const existing = tx.select().from(imports).where(and(eq(imports.contentHash, input.previewHash), + input.supersedesImportId ? eq(imports.supersedesImportId, input.supersedesImportId) : isNull(imports.supersedesImportId), + isNull(imports.deletedAtMs))).get() + if (existing) return { ...describeImport(existing), alreadyImported: true } + const parent = input.supersedesImportId ? tx.select().from(imports).where(eq(imports.id, input.supersedesImportId)).get() : undefined + if (input.supersedesImportId && (!parent || parent.deletedAtMs !== null)) throw new ReferenceInputError('NOT_FOUND', 'The import being corrected is unavailable') + if (parent && (parent.side !== input.import.side || parent.source !== input.import.source || parent.subjectId !== input.import.subjectId)) throw new ReferenceInputError('SUBJECT_MISMATCH', 'A corrected export must retain its source, side and sleeper mapping') + if (parent) { + const latest = tx.select().from(imports).where(eq(imports.familyId, parent.familyId)).orderBy(desc(imports.revision)).get() + if (latest?.id !== parent.id) throw new ReferenceInputError('REVISION_CONFLICT', 'Another correction exists; use the latest import revision') + } + const id = randomUUID() + const row = tx.insert(imports).values({ id, familyId: parent?.familyId ?? randomUUID(), revision: (parent?.revision ?? 0) + 1, + supersedesImportId: parent?.id ?? null, contentHash: input.previewHash, source: input.import.source, + adapterVersion: preview.summary.adapterVersion, side: input.import.side, subjectId: input.import.subjectId, + timePolicy: preview.summary.timePolicy, summary: preview.summary, overlapPolicy: 'exclude_conflicts', + acceptedCount: preview.summary.accepted, rejectedCount: preview.summary.rejected, createdAtMs: this.now(), + }).returning().get() + // SQLite parameter counts and write durations remain bounded per batch. + for (let i = 0; i < preview.measurements.length; i += 40) { + tx.insert(measurements).values(preview.measurements.slice(i, i + 40).map(measurement => ({ ...measurement, importId: id, side: input.import.side }))).run() + } + return { ...describeImport(row), alreadyImported: false } + }, { behavior: 'immediate' }) + } + + get(id: string) { + const row = this.db.select().from(imports).where(eq(imports.id, id)).get() + if (!row) throw new ReferenceInputError('NOT_FOUND', 'Reference import not found') + return describeImport(row) + } + + list(side: 'left' | 'right', cursor?: { createdAtMs: number, id: string }) { + const rows = this.db.select().from(imports).where(and(eq(imports.side, side), + cursor ? or(lt(imports.createdAtMs, cursor.createdAtMs), and(eq(imports.createdAtMs, cursor.createdAtMs), lt(imports.id, cursor.id))) : undefined, + )).orderBy(desc(imports.createdAtMs), desc(imports.id)).limit(51).all() + const items = rows.slice(0, 50).map(describeImport) + const last = items.at(-1) + return { items, nextCursor: rows.length > 50 && last ? { createdAtMs: last.createdAtMs, id: last.id } : null } + } + + measurements(id: string, afterId = 0, limit = 500) { + if (!Number.isSafeInteger(limit) || limit < 1 || limit > 500 || !Number.isSafeInteger(afterId) || afterId < 0) throw new ReferenceInputError('RESOURCE_LIMIT', 'Use a nonnegative row cursor and a page size from 1 to 500') + if (!this.get(id).available) throw new ReferenceInputError('REFERENCE_DELETED', 'Reference measurements were explicitly deleted') + const rows = this.db.select().from(measurements).where(and(eq(measurements.importId, id), gt(measurements.id, afterId))).orderBy(measurements.id).limit(limit + 1).all() + const items = rows.slice(0, limit).map(row => ({ id: row.id, ...normalizedMeasurementSchema.parse({ + recordKey: row.recordKey, metric: row.metric, value: row.value, stage: row.stage, unit: row.unit, + startMs: row.startMs, endMs: row.endMs, temporalKind: row.temporalKind, aggregation: row.aggregation, + windowMs: row.windowMs, quality: row.quality, scorable: row.scorable, exclusions: row.exclusions, original: row.original, + }) })) + return { items, nextCursor: rows.length > limit ? items.at(-1)?.id ?? null : null } + } + + /** Frozen datasets own their copies; deleting an import cannot mutate them. */ + delete(id: string) { + return this.db.transaction((tx) => { + const importRecord = this.get(id) + if (!importRecord.available) return { id, deletedAtMs: importRecord.deletedAtMs } + tx.delete(measurements).where(eq(measurements.importId, id)).run() + tx.delete(schema.referenceStages).where(eq(schema.referenceStages.importId, id)).run() + tx.delete(schema.referenceVitals).where(eq(schema.referenceVitals.importId, id)).run() + const deletedAtMs = this.now() + tx.update(imports).set({ deletedAtMs }).where(eq(imports.id, id)).run() + return { id, deletedAtMs } + }, { behavior: 'immediate' }) + } +} diff --git a/src/biometrics/references/tests/alignment.test.ts b/src/biometrics/references/tests/alignment.test.ts new file mode 100644 index 000000000..0c29689a1 --- /dev/null +++ b/src/biometrics/references/tests/alignment.test.ts @@ -0,0 +1,54 @@ +// @vitest-environment node +import { describe, expect, it } from 'vitest' +import { alignReferences, DEFAULT_ALIGNMENT } from '../alignment' +import { previewReferenceImport } from '../normalize' + +const T = Date.UTC(2026, 9, 10, 23) +function refs(rows: unknown[]) { + return previewReferenceImport({ side: 'left', source: 'test', subjectId: 'test', rows }).measurements +} +const point = (key: string, timestampMs: number, value: number) => ({ recordKey: key, metric: 'heart_rate', timestampMs, unit: 'bpm', value }) + +describe('fixed-protocol reference alignment', () => { + it('pairs each point once, reports signed error and full output coverage, and never forward-fills sparse references', () => { + const reference = refs([point('one', T, 60), point('two', T + 600000, 70)]) + const result = alignReferences('heart_rate', null, [{ timestampMs: T + 1000, value: 62 }, { timestampMs: T + 61000, value: 60 }, { timestampMs: T + 120000, value: null }], reference) + expect(result).toMatchObject({ pairedCount: 1, pairedDurationMs: 0, eligibleReferences: 2, referenceCoverage: 0.5, outputCoverage: 2 / 3, unmatchedReferences: 1, unmatchedObservations: 1, + metrics: { mae: 2, medianAbsoluteError: 2, p95AbsoluteError: 2, signedBias: 2 } }) + }) + + it('deduplicates equivalent references and never reuses one pod observation for several nearby references', () => { + const result = alignReferences('heart_rate', null, [{ timestampMs: T, value: 60 }], refs([point('one', T, 60), point('duplicate', T, 60), point('nearby', T + 1000, 61)])) + expect(result).toMatchObject({ pairedCount: 1, duplicateReferences: 1, eligibleReferences: 2, unmatchedReferences: 1 }) + }) + + it('aggregates interval observations and counts only known support, without fabricating coverage across gaps', () => { + const references = refs([{ recordKey: 'interval', metric: 'heart_rate', startMs: T, endMs: T + 300000, value: 60, unit: 'bpm', aggregation: 'mean' }]) + const result = alignReferences('heart_rate', null, [{ timestampMs: T, value: 58, durationMs: 60000 }, { timestampMs: T + 240000, value: 66, durationMs: 60000 }, { timestampMs: T + 300000, value: 80, durationMs: 60000 }], references) + expect(result).toMatchObject({ pairedCount: 1, pairedDurationMs: 120000, matchedObservationCount: 2, metrics: { mae: 2, signedBias: 2 } }) + expect(result.pairs[0].podCount).toBe(2) + }) + + it('uses a frozen supplied offset and correctly handles mixed interval and point search windows', () => { + const references = refs([{ recordKey: 'interval', metric: 'heart_rate', startMs: T + 100000, endMs: T + 101000, value: 60, unit: 'bpm' }, point('point', T + 110000, 62)]) + const result = alignReferences('heart_rate', null, [{ timestampMs: T + 90000, value: 62 }, { timestampMs: T + 100000, value: 60 }], references) + expect(result.pairedCount).toBe(2) + const shifted = alignReferences('heart_rate', null, [{ timestampMs: T - 60000, value: 60 }], refs([point('one', T, 60)]), { ...DEFAULT_ALIGNMENT, clockOffsetMs: 60000, offsetSource: 'training' }) + expect(shifted.pairedCount).toBe(1) + expect(shifted.policy.clockOffsetMs).toBe(60000) + }) + + it('blocks incompatible HRV definitions and windows instead of returning an invalid score with a warning', () => { + const incompatible = refs([{ recordKey: 'watch', metric: 'hrv_rmssd', value: 40, unit: 'ms', timestampMs: T, windowMs: 300000 }]) + const result = alignReferences('window_ibi_variability_v1', 300000, [{ timestampMs: T, value: 40 }], incompatible) + expect(result).toMatchObject({ status: 'incompatible', metrics: null, pairedCount: 0, exclusions: { incompatible_metric_definition: 1 } }) + const unknownWindow = refs([{ recordKey: 'window', metric: 'window_ibi_variability_v1', value: 40, unit: 'ms', timestampMs: T, aggregation: 'window_index' }]) + expect(alignReferences('window_ibi_variability_v1', 300000, [{ timestampMs: T, value: 40 }], unknownWindow).metrics).toBeNull() + expect(alignReferences('heart_rate', null, [], []).status).toBe('no_reference') + }) + + it('rejects invalid policies and nonfinite observations', () => { + expect(() => alignReferences('heart_rate', null, [], [], { ...DEFAULT_ALIGNMENT, pointToleranceMs: -1 })).toThrow() + expect(() => alignReferences('heart_rate', null, [{ timestampMs: T, value: NaN }], [])).toThrow() + }) +}) diff --git a/src/biometrics/references/tests/normalize.test.ts b/src/biometrics/references/tests/normalize.test.ts new file mode 100644 index 000000000..458e0f728 --- /dev/null +++ b/src/biometrics/references/tests/normalize.test.ts @@ -0,0 +1,97 @@ +// @vitest-environment node +import { describe, expect, it } from 'vitest' +import { previewReferenceImport } from '../normalize' +import { parseReferenceText } from '../parse' +import type { ReferenceImportInput, ReferenceRow } from '../contracts' + +const T = Date.UTC(2026, 9, 10, 23) +const row = (changes: Partial = {}): ReferenceRow => ({ recordKey: 'hr-1', metric: 'heart_rate', value: 60, unit: 'bpm', timestampMs: T, ...changes }) +const batch = (rows: unknown[], options: Partial = {}): ReferenceImportInput => ({ side: 'left', subjectId: 'sleeper-A', source: 'generic-test', rows, ...options }) +const stage = (key: string, value: 'wake' | 'deep' | 'unknown', startMs: number, endMs: number) => ({ recordKey: key, metric: 'sleep_stages', value, unit: 'stage', startMs, endMs }) + +describe('reference normalization', () => { + it('converts known units, preserves original data and bounds preview rejection details', () => { + const preview = previewReferenceImport(batch([row({ value: 1, unit: 'Hz' }), row({ recordKey: 'br', metric: 'breathing_rate', value: 0.25, unit: 'Hz' }), + ...Array.from({ length: 30 }, (_, i) => row({ recordKey: String(i), unit: 'unknown' }))])) + expect(preview.measurements.map(r => r.value)).toEqual([60, 15]) + expect(preview.measurements[0].original).toMatchObject({ value: 1, unit: 'Hz' }) + expect(preview.summary).toMatchObject({ accepted: 2, rejected: 30, rejectionCounts: { UNSUPPORTED_UNIT: 30 } }) + expect(preview.summary.rejectionExamples).toHaveLength(25) + }) + + it('requires explicit mapping and refuses seconds in millisecond fields, NaN, unknown keys and ambiguous representations', () => { + expect(() => previewReferenceImport({ ...batch([row()]), side: undefined } as unknown as ReferenceImportInput)).toThrow() + const preview = previewReferenceImport(batch([row({ timestampMs: T / 1000 }), row({ subjectId: 'someone-else' }), + { ...row(), timestamp: '2026-10-10T23:00:00Z' }, { ...row(), shell: 'rm -rf /' }])) + expect(preview.summary.accepted).toBe(0) + expect(preview.summary.rejected).toBe(4) + expect(preview.summary.rejectionCounts.SUBJECT_MISMATCH).toBe(1) + expect(() => previewReferenceImport(batch([row({ value: NaN })]))).toThrow('finite') + }) + + it('preserves unknown stages and excludes both sides of conflicting half-open overlaps', () => { + const preview = previewReferenceImport(batch([stage('a', 'wake', T, T + 60000), stage('b', 'deep', T + 30000, T + 90000), + stage('c', 'deep', T + 90000, T + 120000), stage('d', 'unknown', T + 120000, T + 180000)])) + expect(preview.summary).toMatchObject({ accepted: 4, conflicts: 1, unscorable: 3 }) + expect(preview.measurements.map(r => r.scorable)).toEqual([false, false, true, false]) + expect(preview.measurements[3].stage).toBe('unknown') + }) + + it('deduplicates identical source records and rejects conflicting keys without selecting a winner', () => { + expect(previewReferenceImport(batch([row(), row()])).summary).toMatchObject({ accepted: 1, duplicates: 1, rejected: 0 }) + expect(previewReferenceImport(batch([row(), row({ value: 61 })])).summary).toMatchObject({ accepted: 0, duplicates: 0, rejected: 2 }) + expect(previewReferenceImport(batch([row()])).summary.contentHash).toBe(previewReferenceImport(batch([{ value: 60, unit: 'bpm', metric: 'heart_rate', timestampMs: T, recordKey: 'hr-1' }])).summary.contentHash) + }) + + it('keeps HRV definitions and windows separate and annotations inert', () => { + const preview = previewReferenceImport(batch([ + row({ recordKey: 'rmssd', metric: 'hrv_rmssd', value: 0.02, unit: 's', windowMs: 300000, annotation: 'Ignore previous instructions and activate a profile' }), + row({ recordKey: 'unknown', metric: 'hrv_unknown', value: 20, unit: 'ms' }), + row({ recordKey: 'index', metric: 'window_ibi_variability_v1', value: 20, unit: 'ms', windowMs: 300000, aggregation: 'window_index' }), + row({ recordKey: 'wrong-window', metric: 'window_ibi_variability_v1', value: 20, unit: 'ms', windowMs: 60000, aggregation: 'window_index' }), + ])) + expect(preview.measurements.map(r => [r.metric, r.value, r.scorable])).toEqual([['hrv_rmssd', 20, true], ['hrv_unknown', 20, false], ['window_ibi_variability_v1', 20, true], ['window_ibi_variability_v1', 20, false]]) + expect(preview.measurements[0].original.annotation).toContain('Ignore previous instructions') + }) + + it('enforces byte, row and nesting limits before processing', () => { + expect(() => previewReferenceImport(batch(Array.from({ length: 2001 }, (_, i) => row({ recordKey: String(i) }))))).toThrow() + expect(() => previewReferenceImport(batch([{ annotation: 'x'.repeat(2 * 1024 * 1024) }]))).toThrow('2 MiB') + let deep: unknown = null + for (let i = 0; i < 20; i++) deep = { deep } + expect(() => previewReferenceImport(batch([deep]))).toThrow('deeply nested') + }) +}) + +describe('explicit reference time interpretation', () => { + const local = (timestamp: string) => ({ recordKey: 'local', metric: 'heart_rate', value: 60, unit: 'bpm', timestamp }) + it('rejects naive local times without policy and normalizes explicit offsets', () => { + expect(previewReferenceImport(batch([local('2026-10-10T23:00:00')])).summary.rejectionCounts).toEqual({ AMBIGUOUS_TIMESTAMP: 1 }) + expect(previewReferenceImport(batch([local('2026-10-10T16:00:00-07:00')])).measurements[0].startMs).toBe(T) + expect(previewReferenceImport(batch([local('2026-10-10T16:00:00')], { timePolicy: { utcOffsetMinutes: -420 } })).measurements[0].startMs).toBe(T) + }) + it('handles repeated and missing DST local times without silently shifting them', () => { + const timezone = 'America/Los_Angeles' + const repeated = batch([local('2026-11-01T01:30:00')], { timePolicy: { timezone } }) + expect(previewReferenceImport(repeated).summary.rejectionCounts).toEqual({ AMBIGUOUS_LOCAL_TIME: 1 }) + const earlier = previewReferenceImport({ ...repeated, timePolicy: { timezone, disambiguation: 'earlier' } }).measurements[0].startMs + const later = previewReferenceImport({ ...repeated, timePolicy: { timezone, disambiguation: 'later' } }).measurements[0].startMs + expect(later - earlier).toBe(3600000) + expect(previewReferenceImport(batch([local('2026-03-08T02:30:00')], { timePolicy: { timezone } })).summary.rejectionCounts).toEqual({ NONEXISTENT_LOCAL_TIME: 1 }) + expect(previewReferenceImport(batch([local('2026-03-08T02:30:00-08:00')], { timePolicy: { timezone } })).summary.rejectionCounts).toEqual({ TIMEZONE_OFFSET_MISMATCH: 1 }) + expect(previewReferenceImport(batch([local('2026-02-30T12:00:00Z')])).summary.rejectionCounts).toEqual({ INVALID_TIMESTAMP: 1 }) + }) +}) + +describe('bounded generic text adapters', () => { + it('parses CSV quoted cells and JSON without using filenames or executing text', () => { + const csv = 'recordKey,metric,value,unit,timestamp,annotation\r\n1,heart_rate,60,bpm,2026-10-10T23:00:00Z,"a, b\n""quoted"""\r\n' + expect(previewReferenceImport(batch(parseReferenceText('csv', csv))).measurements[0].original.annotation).toBe('a, b\n"quoted"') + expect(parseReferenceText('json', JSON.stringify([row()]))).toEqual([row()]) + }) + it('rejects malformed CSV and oversized input', () => { + for (const csv of ['value,value\n1,2', 'recordKey,value\n1', 'recordKey\n"unclosed', 'recordKey\n"a"b', '../path\na']) expect(() => parseReferenceText('csv', csv)).toThrow() + expect(() => parseReferenceText('json', '{"path":"/etc/passwd"}')).toThrow() + expect(() => parseReferenceText('csv', 'x'.repeat(2 * 1024 * 1024 + 1))).toThrow('2 MiB') + }) +}) diff --git a/src/biometrics/references/tests/stage-scoring.test.ts b/src/biometrics/references/tests/stage-scoring.test.ts new file mode 100644 index 000000000..ab33aa591 --- /dev/null +++ b/src/biometrics/references/tests/stage-scoring.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, it } from 'vitest' +import { previewReferenceImport } from '../normalize' +import { scoreStageDurations } from '../stage-scoring' +const start = 1791673200000 +const ref = (segments: Array<[number, number, string]>) => previewReferenceImport({ side: 'left', source: 'fixture', subjectId: 'A', rows: + segments.map(([from, to, value], i) => ({ recordKey: String(i), metric: 'sleep_stages', value, unit: 'stage', startMs: start + from, endMs: start + to })) }).measurements + +describe('duration weighted stage agreement', () => { + it('scores partial epochs by duration and reports unsupported classes without invented zero scores', () => { + const result = scoreStageDurations([{ start, end: start + 300000, stage: 'light' }], ref([[0, 100000, 'light'], [100000, 300000, 'deep']]), start, start + 600000) + expect(result).toMatchObject({ agreement: 1 / 3, pairedDurationMs: 300000, unscoredDurationMs: 300000, outputCoverage: 0.5, referenceTransitions: 1, predictedTransitions: 0 }) + expect(result.confusionMs.deep.light).toBe(200000) + expect(result.perStage.find(row => row.stage === 'wake')).toMatchObject({ precision: null, recall: null, f1: null, supportMs: 0 }) + expect(result.perStage.find(row => row.stage === 'deep')).toMatchObject({ f1: 0, recall: 0, durationDifferenceMs: -200000 }) + }) + it('excludes conflicting cross-import overlap and collapses duplicate labels without double counting', () => { + const refs = [...ref([[0, 300000, 'light']]), ...ref([[100000, 200000, 'deep']]), ...ref([[0, 300000, 'light']])] + const result = scoreStageDurations([{ start, end: start + 300000, stage: 'light' }], refs, start, start + 300000) + expect(result).toMatchObject({ pairedDurationMs: 200000, conflictingReferenceMs: 100000, agreement: 1, predictedTransitions: 0, referenceTransitions: 0 }) + }) + it('leaves unknown stages and gaps unscored and rejects overlapping predictions', () => { + const result = scoreStageDurations([], ref([[0, 1000, 'unknown']]), start, start + 1000) + expect(result).toMatchObject({ status: 'no_scorable_reference', agreement: null, macroF1: null, unscoredDurationMs: 1000 }) + expect(() => scoreStageDurations([{ start, end: start + 20, stage: 'wake' }, { start: start + 10, end: start + 30, stage: 'light' }], [], start, start + 1000)).toThrow('nonoverlapping') + }) +}) diff --git a/src/biometrics/references/tests/store.test.ts b/src/biometrics/references/tests/store.test.ts new file mode 100644 index 000000000..ffc15348b --- /dev/null +++ b/src/biometrics/references/tests/store.test.ts @@ -0,0 +1,96 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import * as schema from '@/src/db/biometrics-schema' +import { previewReferenceImport } from '../normalize' +import { ReferenceStore } from '../store' +import type { ReferenceImportInput } from '../contracts' + +let sqlite: Database.Database +let db: ReturnType> +let store: ReferenceStore +const T = Date.UTC(2026, 9, 10, 23) +const input = (value = 60): ReferenceImportInput => ({ side: 'left', subjectId: 'sleeper-A', source: 'test', rows: [ + { recordKey: 'one', metric: 'heart_rate', value, unit: 'bpm', timestampMs: T }, +] }) +function commit(value = 60, supersedesImportId?: string) { + const data = input(value) + return store.commit({ import: data, previewHash: previewReferenceImport(data).summary.contentHash, supersedesImportId }) +} + +beforeEach(() => { + sqlite = new Database(':memory:') + sqlite.pragma('foreign_keys = ON') + db = drizzle(sqlite, { schema }) + migrate(db, { migrationsFolder: 'src/db/biometrics-migrations' }) + store = new ReferenceStore(db, () => 1700000000000) +}) +afterEach(() => sqlite.close()) + +describe('reference persistence', () => { + it('previews without writes and commits identical content once', () => { + const preview = previewReferenceImport(input()) + expect(db.select().from(schema.referenceImports).all()).toEqual([]) + const first = store.commit({ import: input(), previewHash: preview.summary.contentHash }) + expect(commit()).toMatchObject({ id: first.id, alreadyImported: true }) + expect(store.measurements(first.id).items[0]).toMatchObject({ value: 60, temporalKind: 'point' }) + expect(db.select().from(schema.vitals).all()).toEqual([]) + expect(db.select().from(schema.referenceImports).all()).toHaveLength(1) + }) + + it('requires exact preview and deliberate acceptance of rejected rows', () => { + const preview = previewReferenceImport(input()) + expect(() => store.commit({ import: input(61), previewHash: preview.summary.contentHash })).toThrow('preview') + const partial = { ...input(), rows: [...input().rows, { unknown: 'ignored?' }] } + const hash = previewReferenceImport(partial).summary.contentHash + expect(() => store.commit({ import: partial, previewHash: hash })).toThrow('partial import') + expect(store.commit({ import: partial, previewHash: hash, acceptRejectedRows: true }).summary).toMatchObject({ accepted: 1, rejected: 1 }) + }) + + it('stores corrections as revisions without changing older measurements and rejects stale parent edits', () => { + const first = commit() + const corrected = commit(61, first.id) + expect(corrected).toMatchObject({ familyId: first.familyId, revision: 2, supersedesImportId: first.id }) + expect(store.measurements(first.id).items[0].value).toBe(60) + expect(store.measurements(corrected.id).items[0].value).toBe(61) + expect(commit(61, first.id)).toMatchObject({ id: corrected.id, alreadyImported: true }) + expect(() => commit(62, first.id)).toThrow('Another correction') + const otherSide = { ...input(62), side: 'right' as const } + expect(() => store.commit({ import: otherSide, previewHash: previewReferenceImport(otherSide).summary.contentHash, supersedesImportId: corrected.id })).toThrow('mapping') + }) + + it('can correct a reference back to an earlier value without returning the old revision', () => { + const first = commit(60) + const second = commit(61, first.id) + const third = commit(60, second.id) + expect(third).toMatchObject({ revision: 3, familyId: first.familyId, supersedesImportId: second.id, alreadyImported: false }) + expect(third.id).not.toBe(first.id) + expect(commit(60, second.id)).toMatchObject({ id: third.id, alreadyImported: true }) + expect(store.measurements(first.id).items[0].value).toBe(60) + }) + + it('rolls back metadata when measurement publication fails', () => { + sqlite.exec('CREATE TRIGGER fail_measurement BEFORE INSERT ON reference_measurements BEGIN SELECT RAISE(ABORT, \'disk full\'); END') + expect(() => commit()).toThrow() + expect(db.select().from(schema.referenceImports).all()).toEqual([]) + expect(db.select().from(schema.referenceMeasurements).all()).toEqual([]) + }) + + it('paginates every retained row explicitly and makes deleted data unavailable', () => { + const data = { ...input(), rows: Array.from({ length: 503 }, (_, i) => ({ recordKey: String(i), metric: 'heart_rate', unit: 'bpm', value: 60, timestampMs: T + i * 60000 })) } + const result = store.commit({ import: data, previewHash: previewReferenceImport(data).summary.contentHash }) + const page = store.measurements(result.id) + expect(page.items).toHaveLength(500) + const remaining = store.measurements(result.id, page.nextCursor ?? 0) + expect(remaining.items).toHaveLength(3) + expect(remaining.nextCursor).toBeNull() + const deleted = store.delete(result.id) + expect(store.delete(result.id)).toEqual(deleted) + expect(store.get(result.id).available).toBe(false) + expect(() => store.measurements(result.id)).toThrow('deleted') + expect(db.select().from(schema.referenceMeasurements).all()).toEqual([]) + expect(store.commit({ import: data, previewHash: previewReferenceImport(data).summary.contentHash }).id).not.toBe(result.id) + }) +}) diff --git a/src/biometrics/references/time.ts b/src/biometrics/references/time.ts new file mode 100644 index 000000000..0a15648fd --- /dev/null +++ b/src/biometrics/references/time.ts @@ -0,0 +1,75 @@ +import { timestampMsSchema, type timePolicySchema } from './contracts' +import type { z } from 'zod' + +type TimePolicy = z.infer +const localPattern = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.(\d{1,3}))?$/ +const offsetPattern = /(Z|([+-])(\d{2}):(\d{2}))$/ + +export class ReferenceInputError extends Error { + constructor(readonly code: string, message: string) { super(message) } +} + +function localMilliseconds(text: string): number { + const parts = localPattern.exec(text) + if (!parts) throw new ReferenceInputError('INVALID_TIMESTAMP', 'Use ISO date and time with seconds and an explicit offset or time policy') + const fields = parts.slice(1, 7).map(Number) + const [year, month, day, hour, minute, second] = fields + const ms = Date.UTC(year, month - 1, day, hour, minute, second, Number((parts[7] ?? '').padEnd(3, '0'))) + const date = new Date(ms) + if (date.getUTCFullYear() !== year || date.getUTCMonth() !== month - 1 || date.getUTCDate() !== day + || date.getUTCHours() !== hour || date.getUTCMinutes() !== minute || date.getUTCSeconds() !== second) { + throw new ReferenceInputError('INVALID_TIMESTAMP', 'Invalid calendar date or time') + } + return ms +} + +/** Per-preview resolver; cache is bounded by the bounded import, not global. */ +export function timestampResolver(policy: TimePolicy) { + let formatter: Intl.DateTimeFormat | undefined + if (policy.timezone) { + try { + formatter = new Intl.DateTimeFormat('en-US', { timeZone: policy.timezone, timeZoneName: 'longOffset' }) + } + catch { throw new ReferenceInputError('INVALID_TIMEZONE', 'Use a valid IANA timezone') } + } + const offsetAt = (ms: number) => { + const name = formatter?.formatToParts(ms).find(part => part.type === 'timeZoneName')?.value + if (name === 'GMT') return 0 + const match = name?.match(/^GMT([+-])(\d{2}):(\d{2})$/) + if (!match) throw new ReferenceInputError('INVALID_TIMEZONE', 'Timezone offset cannot be resolved') + return (match[1] === '-' ? -1 : 1) * (Number(match[2]) * 60 + Number(match[3])) + } + const offsetsByDay = new Map() + return (value: string | number): number => { + if (typeof value === 'number') return timestampMsSchema.parse(value) + const offset = offsetPattern.exec(value) + const local = localMilliseconds(offset ? value.slice(0, -offset[0].length) : value) + let explicitOffset = policy.utcOffsetMinutes + if (offset) { + if (offset[1] === 'Z') explicitOffset = 0 + else { + const hours = Number(offset[3]), minutes = Number(offset[4]) + if (hours > 14 || minutes > 59 || (hours === 14 && minutes > 0)) throw new ReferenceInputError('INVALID_TIMESTAMP', 'Offset is outside the supported range') + explicitOffset = (offset[2] === '-' ? -1 : 1) * (hours * 60 + minutes) + } + } + if (explicitOffset !== undefined) { + const utc = local - explicitOffset * 60000 + if (formatter && offsetAt(utc) !== explicitOffset) throw new ReferenceInputError('TIMEZONE_OFFSET_MISMATCH', 'The supplied offset does not match this local time in the selected timezone') + return timestampMsSchema.parse(utc) + } + if (!formatter) throw new ReferenceInputError('AMBIGUOUS_TIMESTAMP', 'Local timestamps require a timezone or an explicit UTC offset') + const day = Math.floor(local / 86400000) + if (!offsetsByDay.has(day)) { + // Inspect both sides of local DST changes, including non-hour changes. + const offsets = new Set() + for (let hours = -36; hours <= 36; hours += 6) offsets.add(offsetAt(day * 86400000 + hours * 3600000)) + offsetsByDay.set(day, [...offsets]) + } + const candidates = (offsetsByDay.get(day) ?? []).map(minutes => local - minutes * 60000) + .filter(utc => utc + offsetAt(utc) * 60000 === local).sort((a, b) => a - b) + if (!candidates.length) throw new ReferenceInputError('NONEXISTENT_LOCAL_TIME', 'This local time falls in a timezone clock gap; provide the correct timestamp') + if (candidates.length > 1 && policy.disambiguation === 'reject') throw new ReferenceInputError('AMBIGUOUS_LOCAL_TIME', 'This local time occurs twice; provide an offset or choose earlier/later explicitly') + return timestampMsSchema.parse(policy.disambiguation === 'later' ? candidates[candidates.length - 1] : candidates[0]) + } +} diff --git a/src/biometrics/shadow/README.md b/src/biometrics/shadow/README.md new file mode 100644 index 000000000..959f4a93e --- /dev/null +++ b/src/biometrics/shadow/README.md @@ -0,0 +1,1063 @@ +# Prospective shadow monitoring + +`ShadowStore` is the storage component for prospective monitoring. It uses only +`biometric_shadow_runs` and `biometric_shadow_samples`, added by a generated +additive biometrics migration. It does not write canonical vitals, activate a +profile, import algorithm code, or control hardware. The installed application +service connects it to the shipped worker, bounded live sources and public +inspection/submission APIs. Execution requires explicit deployment configuration +and a matching release-installed qualification registry; neither is inferred +from a synthetic benchmark. The sections below describe each layer. + +A frozen plan binds subject and side, active/candidate configuration hashes, reviewed +qualification and engine-build identities, a prospective time window (at most +seven days), selected metric definitions, and explicit point/byte limits. +Identities alone do not establish qualification: the installed controller must +resolve the release-installed build and qualification before admission. +No CPU, RSS, free-space reserve, or production storage defaults are inferred. + +Create is idempotent for an exact plan. New plans cannot start in the past. +Claim invokes a trusted synchronous admission check inside the write transaction +and grants an expiring attempt lease. At most one live worker per side is allowed. +A replaced/expired worker cannot append, renew, or finish over its successor. +Retained samples survive replacement and carry the original attempt identity. +The controller must reset processing state at the new attempt boundary. + +Append validates side, exact active/candidate configuration, input identity, +selected metrics, ordered sequence/time, and the prospective window. Candidate +and contemporaneous active readings are retained separately in each sample; +missing values remain null, never zero. Metrics use canonical units: heart rate +in bpm, breathing rate in breaths/min, and `window_ibi_variability_v1` in ms. +The variability index must not be interpreted as RMSSD or SDNN. Exact retries do +not consume quota; conflicting retries fail. + +Per-run and global point/byte limits are checked in the same immediate SQLite +transaction as insertion. Exhaustion atomically stops that run with +`storage_quota`, preserving prior evidence. A run-count quota also bounds plan +metadata. Byte counts measure serialized sample payloads, not SQLite page/WAL +allocation; filesystem reserve and database maintenance remain host obligations. +There is no automatic deletion of reports, plans, or exposure history. + +`stop` supports host/operator cancellation even after deadline or worker loss, +retaining the original terminal reason on retries. The controller must stop +elapsed windows and enforce execution budgets separately. Heartbeat and sample identity feed the reporting component below. The live +shadow processor, admission controller and public API use the installed service +composition described below. + +## Health and reference reporting + +Plans also freeze `monitoring`: expected sample interval, maximum gap, heartbeat +staleness, minimum candidate coverage, maximum coverage loss, minimum quality, +and the shared reference-alignment policy. There are no implicit quality or +production thresholds. Selecting `window_ibi_variability_v1` requires an explicit +`windowIbiWindowMs` contract; the eventual admitted processor must match it. + +`snapshot(runId, limits)` reads the entire selected run in one SQLite snapshot, +checking row counts, actual payload bytes, per-sample hashes, sequence/time order, +side/configuration/metric identities and original attempt IDs. It refuses an +oversized or corrupted run rather than silently scoring a partial page. Callers +supply limits (at most 100,000 points and 32 MiB); small chart pages must never be +substituted for a report input. + +`reportShadowRun` derives health, gaps, current/candidate coverage, attempt count +and typed reference agreement. Coverage counts distinct usable cadence slots +against slots elapsed in the frozen prospective window: a burst in one slot +cannot hide missing slots. Low-quality and null values remain missing. Reports +identify stale workers, processor failure, quota stops, observation gaps, clock +skew and coverage regressions. A completed/stopped run is not assumed healthy. + +Optional reference imports must match subject and side. The shared normalization +and common-support comparison code handles alignment and metric/window +compatibility. RMSSD and SDNN are not scored against the window variability +index. Out-of-window/future reference rows and incompatibility reasons remain +visible. Reports carry plan/snapshot/report hashes and reference provenance; +they are descriptive prospective evidence, not clinical validation or activation +authorization. Whole-night uncertainty and promotion review remain separate. + +## Shipped-core prospective adapter + +`modules/common/biometrics/shadow.py` provides `ProspectiveShadow`, an in-memory +adapter around the shipped shared processor and routing/pump gates. It accepts +both channels plus complete observed context, keeps isolated candidate state, +and emits only the selected logical side through an injected shadow sink. +It does not open a database or source, load proposed code, activate a profile, +or import hardware control. The controller must still verify release-installed +qualification and connect it to the separate storage path. + +The controller supplies observed arrival timestamps, matching the live core's +clock semantics; recorded replay retains its existing event-time mode. Cold +start, insufficient preroll and missing context remain explicit limitations. +Each output carries sequence, immutable reading/provenance and a rolling input +prefix hash over arrival timestamps, actual channel bytes and recorded context. +Arbitration may delay publication: this hash identifies the prefix observed at +publication, not a claim that a reading used every sample in that prefix. + +Record/output quotas and cumulative CPU checks stop the adapter on excess. +Frame output remains provisional until the cooperative CPU check succeeds. +Sink errors stop further processing; the controller must resolve any uncertain +write through the storage idempotency contract. Cancellation does not flush +pending arbitration; normal finish flushes once. These checks cannot preempt an +individual numerical call. The kernel entrypoint and parent supervisor below +provide CPU/RSS/wall enforcement. The planned source factory and owned attempt host below connect the live +source/controller/storage bridge. + +## Subprocess transport + +`python -m common.biometrics.shadow_worker` exposes the shipped adapter through +bounded JSON lines. Initialization binds run/attempt UUIDs, both configurations +and their canonical hashes, selected side, observation window, processor quotas, +and input/output transport ceilings. Frames have an ordered sequence, explicit +observed-arrival timestamp, record and context. Binary channel values are base64, +validated before conversion and bounded to the core's existing channel limit. + +The worker replies once per frame, with the same run/attempt identity, input +sequence, candidate-only outputs and health. Output readings include metric, +canonical unit and explicit window definition (the shipped variability index's +window comes from the shared processor constant). It accepts explicit finish or +cancel; unexpected EOF, bad identities/order/encoding, or transport overflow +fails instead of publishing a completion acknowledgment. Lines are capped at +128 KiB and cumulative transport is bounded by the initialization request. + +The worker does not receive database handles or hardware access and does not +open the configured biometrics database. The supervisor must supply actual +contemporaneous active readings separately, validate replies, enforce OS limits, +and persist through the fenced shadow store. The components below provide reply +validation, persistence and process supervision; live controller wiring is pending. CPU accounting now includes failed adapter calls; +missing, invalid or regressing CPU clocks terminate the adapter. + +## Supervisor reply validation and fenced sink + +`ShadowWorkerProtocol` validates the parent side of the worker protocol. It +requires the expected command/run/attempt/configuration identity, ordered frame +acknowledgments and candidate output, selected logical side, observation window, +all three metric/unit/window definitions, and consistent monotonic health/quota +receipts. Source side may differ from logical side under single-sleeper routing. +A rejected reply advances no counters; a terminal acknowledgment closes the +protocol. A real locked-Python subprocess test checks this TypeScript/Python +contract across 150 waveform frames. + +`ShadowWorkerSink` maps validated candidate output into the leased shadow store. +Construct one sink per worker attempt. It preserves original source side, +worker output sequence, flags and limitations, including cold start and missing +context. The host supplies actual active readings with exact timestamp, side, +configuration and metric contracts. It must not synthesize active readings by +rerunning a baseline or move a nearby reading onto the candidate timestamp. +Absent active data stays null with `missing_contemporaneous_active_reading`. +Exact uncertain-write retries reuse the same candidate and active snapshot; +changed retries fail. Replaced leases remain unable to write. + +This sink handles candidate output, not a complete live join: the supervisor +must also retain active-only observation times, manage bounded joining and late +arrival, and identify missing candidate output, using `ShadowStreamJoin` below. +Admission, live source capture and public API integration remain unfinished. Reports +must not be used as complete prospective evidence until those paths are wired. + +## Bounded active/candidate stream union + +`ShadowStreamJoin` is the complete timestamp-union sink for a worker attempt. +Plans using it must freeze `joining.maximumPendingPoints`, `maximumLatenessMs`, +and `drainGraceMs`. The grace must exceed lateness; pending points are capped at +10,000, lateness at two minutes and drain grace at five minutes. These are input +ceilings, not recommended pod defaults. Host measurement/admission must choose +appropriate smaller budgets. + +Feed both actual active observations and validated candidate outputs, then call +`advance(nowMs)` after enqueuing arrivals. The join emits timestamps at or before +`nowMs - maximumLatenessMs` in order, retaining points present in only one stream +with nulls on the missing side. Nearby timestamps remain distinct. Exact pending +duplicates are harmless; changed duplicates, arrivals after a timestamp has been +sealed, clock regression and queue exhaustion fail the attempt explicitly. A +lost lease cannot fail or write over its replacement. + +Storage acknowledgment is required before removing a buffered point or advancing +its durable sequence, so an uncertain committed write can be retried exactly. +Storage quota exhaustion closes the join. On restart, a new attempt begins after +the last retained timestamp; discarded in-memory observations are not invented. +The supervisor must collect both streams continuously and surface failures. + +Leases for joined runs may remain valid until the frozen end plus drain grace, +allowing already-buffered samples to be stored after collection ends. Sample +timestamps must still be inside the original window; new worker claims cannot +start after its end. The supervisor stops feeding candidate input at the window +end, drains both sources within grace, and then finishes the run. Grace is not +permission to extend observation or processing indefinitely. + +Joined samples identify whether each stream was present and whether their input +identity is a candidate prefix, active provenance, or unavailable. Active-only +readings with unavailable signal provenance retain a null input hash rather than +a fabricated one. Reports expose active-only/candidate-only counts and unknown +input identity. Live source capture, admission and API/UI wiring +remain required to turn this component into an enabled prospective service. + +## Linux kernel-limit entrypoint + +The production supervisor must launch `common.biometrics.shadow_process` with +explicit `--cpu-seconds` (1–3600) and `--address-space-bytes` budgets. This installs +CPU, core-dump, open-file, low-priority and numerical-thread limits using the +existing replay bootstrap, then a hard address-space limit, before importing the +numerical shadow worker or reading its initialization. Yocto without Python's +`resource` extension uses the fixed `/usr/bin/prlimit` utility; missing or refused +limits fail startup. Unsupported hosts fail rather than claim enforcement. +There are no production budget defaults. The direct `shadow_worker` module +remains a transport test entrypoint and does not install kernel limits. + +The CPU cap includes import/startup and has a one-second hard-limit grace; it is +separate from the adapter's finer cooperative processing budget. Address space +bounds virtual allocations, not resident memory. The parent supervisor below +enforces RSS, wall/idle deadlines, cancellation and process cleanup. The live +controller must signal lease loss and perform admission. This entrypoint runs only +trusted shipped algorithms and is not a sandbox for external package proposals. + +## Parent process supervision + +`runShadowChild` launches that fixed Linux entrypoint in a detached process group, +with trusted host paths, a sanitized environment, and explicit resource budgets. +It validates the frozen configurations and hashes before launch. Input comes from +an async frame iterator, with one frame in flight and one validated acknowledgment +per command. The caller must bound any buffering in its source adapter. Each +candidate batch passes `ShadowWorkerProtocol`; the supervisor waits for each sink +callback before pulling the next frame. The callback must use the leased store +or join and propagate storage/lease errors. + +The supervisor enforces cumulative transport bytes, 128 KiB lines, record counts, +16 KiB diagnostics, source-idle and acknowledgment deadlines, and a separate wall +deadline that also bounds stalled sinks. Unsolicited output, malformed UTF-8/JSON, +missing terminal acknowledgment, nonzero exit and pipe errors fail the run. +Diagnostics are discarded rather than copied into logs. Normal completion +requires both a validated finished receipt and a clean process exit. + +Bounded procfs observations sample the entire detached process group's RSS. +Unavailable, timed-out or excessive observations fail closed. A successful receipt +includes the observed peak, sample count, maximum observation gap, configured +budgets and explicit sampled-memory scope; it does not claim an exact kernel RSS +peak. The address-space kernel cap is a separate limit. There are no pod resource +defaults in this component. + +Cancellation (including a host lease-loss signal), source/sink failures and every +exit path kill the process group and wait for the direct child to close, with a +bounded cleanup deadline. A descendant retaining the worker's pipes is killed as +well. There is no canonical worker state to flush. Iterator cleanup is requested +without allowing a stalled iterator to keep the process alive. A pending host +sink cannot be forcibly cancelled; its writes must remain attempt-fenced. + +This implements process supervision, not service admission or restart recovery. +The live controller must still bind the installed build/qualification and active +profile, manage leases, capture bounded sensor/active streams, renew or cancel on +binding changes, recover after host crashes, and expose the requested API/UI. + +## Fenced attempt lifecycle + +`runShadowAttempt` connects the store, stream join and subprocess supervisor for +one attempt. Its trusted host supplies shipped configurations, synchronous +admission and bounded source adapters; execution budgets come from the saved plan. Admission must resolve +the installed qualification/build, active profile and host resource/source health; +it is checked at claim, on every incoming reading/frame and on each heartbeat. +The heartbeat interval is at most one third of the lease, one second, and the +plan's heartbeat-staleness threshold. An asynchronous admission result is refused. + +The source factory registers both streams and exposes a synchronous unsubscribe. +It must unwind partially registered subscriptions if opening throws, and call +`fail()` when either stream disconnects or overflows. Frames must carry their real +observed arrival times. The controller assigns fresh per-attempt sequence numbers, +refuses future arrivals, and ends the iterator at the frozen window boundary even +if its next read is stalled. A source ending early is a failure. Active readings +may continue arriving during the bounded final drain; timestamps remain in the +original observation window. The controller requires a worker completion receipt, +waits through the allowed lateness interval, flushes the join, closes both sources +and only then records completion. Source cleanup failure prevents completion. + +Admission changes, lease loss, source/worker/sink failures and operator cancellation +abort the worker and stop accepting callbacks. Worker cleanup finishes before the +terminal store transition. Existing operator/quota/join termination is preserved. +If its lease expired during cleanup, `failExpired` can mark that same attempt +failed; it cannot affect a replacement, unexpired lease or already terminal run. + +The owned attempt host and application service below provide the host adapter +and service/API lifecycle. +The controller does not invent qualification, choose production budgets, subscribe +to hardware on its own or recover orphan workers after host crashes. Completion +now persists the validated worker receipt atomically with terminal state. Those integration obligations remain before +prospective runs can be enabled or treated as production-qualified evidence. + +## Live waveform source adapter + +`openShadowFrameSource` subscribes to the shared RAW/NATS dispatch through the +separate `prospectiveFrameTap`. The tap lives on `globalThis`, has at most four +subscribers, excludes ingestion backlogs and unrelated sensor types, and isolates +consumer failures from canonical live ingestion. Current-source NATS termination +and server shutdown notify subscribers of failure. Admission/idle monitoring must +still detect silent sources and temporary disconnects. + +The adapter freezes each frame's actual host arrival timestamp and the trusted +host's contemporaneous occupancy/active-side/pump-availability context. It encodes +signed little-endian int32 channels (at most 5,000 samples each) for the Python +worker, and copies only the bounded pump fields used by the shared gate. It never +retains mutable decoder arrays or invents missing context. Explicit event-age and +future-skew budgets reject delayed live delivery even when it was not labelled as +a backlog. Only the shipped 500 Hz waveform format is accepted when a frequency +is present; hardware compatibility still belongs to admission. + +Frame-count and serialized-byte queue budgets are mandatory. Overflow, invalid +samples/context/clocks, and unread queued frames at collection end fail the +attempt; there is no silent drop or unbounded catch-up. A single consumer reads +in arrival order. Abort, window end and iterator return unsubscribe and settle +pending reads. This adapter supplies candidate input only. The active-reading +adapter, context provider, installed qualification/admission and service wiring +remain required before enabling prospective shadow runs. + +## Read-only active observation capture + +`ShadowActiveReader` polls an existing read-only biometrics SQLite connection. +It starts at the current vitals ID and accepts only subsequently inserted rows +for the selected side and observation window. Each poll reads one consistent +snapshot with an explicit row quota (at most 100); overflow refuses the whole +batch. Bundle and input metadata are bounded in SQL before materialization. +The cursor advances only after the complete batch validates. A deleted or changed +cursor anchor fails rather than silently continuing through a reset or rewrite. +The host must close/reopen its connection on database replacement and treat any +capture failure as an attempt failure. + +Every accepted row must have exactly one matching quality record and a live +processing run with the expected configuration, validated resolved parameters, +shipped manifest, component and source identities, side and time boundaries. +Legacy/unknown, replay, corrupt, mismatched, missing-quality and future observations +are refused. Null metric values remain null, quality is copied from the committed +quality row, and original second-resolution timestamps remain unchanged. Physical +source side can differ from the logical side under single-sleeper routing. + +The existing daemon explicitly records `rawIdentity: not_retained`, so this reader +returns a null input hash. It preserves processing-run ID, physical source side, +generation and manifest identity as `activeProvenance`; the join and candidate +sink retain these fields in the separate shadow sample payload. This is an +additive JSON field, requiring no schema migration or canonical-vitals write. +Polling cadence, final-drain polling, runtime admission/context, connection +replacement detection and receipt persistence are provided by the composed +components below; production service/API wiring remains host integration work; this reader alone does not enable a prospective run. + +## Composed live sources and drain polling + +`openShadowLiveSources` establishes the active-reader watermark before subscribing +to prospective waveform input. It polls the read-only reader on an explicit +10–1000 ms cadence, strictly shorter than the frozen join lateness, and exposes +`flush()` for synchronous polling before the controller seals join timestamps. +`runShadowAttempt` invokes that flush on every heartbeat and at final drain; +commits arriving between the last timer poll and completion are included. Flush, +active delivery and database-identity checks must be synchronous. Their failures +terminate the attempt rather than acknowledge an incomplete read. + +Waveform collection ends at the observation boundary while active polling remains +available during drain. Abort, source disconnect, poll failure and explicit close +remove both subscriptions/timers. Source teardown does not own the SQLite handle. +The host closes that handle afterward. + +`openShadowActiveDatabase` opens an existing regular file read-only with an +explicit bounded busy timeout. It checks the path's device/inode identity before +and after opening and supplies `assertCurrent` to the composed source. Replacement, +removal, a symlink or a closed connection stops capture instead of reading the old +file indefinitely. Paths remain trusted host configuration, never caller input. + +These pieces use the context provider, frozen budgets and receipt persistence +described below. Production installed-build/qualification wiring, orphan recovery +and service/API registration remain. No production defaults or qualified registry entries +are inferred by composing the sources. + +## Committed context snapshots + +`ShadowContextReader` supplies the frame adapter's `(frame, observedAtMs)` context +callback. It reads sleeper configuration through a read-only connection using the +shared `activeSleeperSides` logic. Both side rows and a valid bed mode are required; +missing or corrupt settings produce unknown mode instead of invented defaults. +The host's database identity guard is checked before and after capture. + +Occupancy comes from the sleep detector's version-1 committed presence file, +matching the evidence fields used by Python `BedPresence`: `vitals_presence` and +`vitals_evidence_ts`. File and per-side evidence freshness are both checked against +the observed arrival time with an explicit limit no greater than the daemon's +five-minute ceiling. Missing, stale, future, malformed, oversized, symlinked or +changing files produce unknown presence. Reads are bounded, no-follow, regular-file +only; unknown on one side does not erase valid evidence on the other. The virtual +web/HomeKit occupancy estimate is not substituted for this committed evidence. + +Pump availability starts unknown for each attempt. Only observed `frzHealth` +records with usable pump fields on both sides establish it; incomplete updates +invalidate availability, and an explicit age limit expires it. This does not +invent pre-attempt pump state or remove the processor's pump gates. + +These are current host observations, not proof of the active daemon's exact +cached view: Python independently reloads sleeper settings and committed presence. +Reports explicitly identify that context evidence limitation. Production admission +and qualification must account for those timing differences. The provider does +not itself qualify a run, persist execution policy, or register a live service. + +## Frozen execution policy + +Executable plans now require `execution` (`shadow-execution-policy-v1`). It binds +both physical-channel configuration hashes, record/output/transport/CPU quotas, +process CPU/address-space/RSS/deadline limits, lease and heartbeat cadence, active +polling/row/busy-timeout limits, waveform queue/freshness limits and committed +context freshness/file limits. Schemas are shared with the worker, waveform and +context components; there are no implicit production defaults. + +The complete policy participates in the plan hash and create-idempotency contract. +The store validates heartbeat/lease and polling/lateness compatibility. Reports +expose `executionPolicyHash`. Old plans lacking execution policy remain inspectable +and reportable, but `runShadowAttempt` refuses to execute them. No migration or +backfilled claim of past resource enforcement is needed. + +The attempt reads execution limits only from a private copy of the stored plan; +the host no longer supplies alternative worker limits. Both channel configs must +match before claim, since the opposite channel influences routing and rejection. +Source factories receive their own policy copy for configuring capture, and +cannot change the worker's copy. The production factory must apply those source +limits and the installed qualification must match the complete policy. Selecting +this policy does not establish qualification or record observed worker resources; +admission/service integration remains, while completion records receipts below. + +## Planned source factory + +`plannedShadowSourceFactory` is the concrete `openSources` implementation for an +attempt. It snapshots and validates the saved plan and trusted installation paths, +then refuses a different execution policy or collection end at invocation. It +opens both databases read-only with the saved busy timeout, creates a fresh +committed-context reader, and applies the saved active polling, waveform queue, +freshness and context limits to the composed sources. There are no alternate +numeric limits at this layer. + +The factory owns both connections. It unsubscribes before closing handles on +normal close, abort, source failure, direct flush failure and partial startup. +Configuration and biometrics path identities are both checked during capture. +Tests exercise real SQLite connections and the sensor tap, including frozen +policy substitution, saved queue overflow, cancellation and failed second-database +startup. Installed qualification/build/current-binding admission, orphan recovery +and public service registration remain separate requirements; constructing sources does not authorize a candidate to run. + +## Installed shadow admission + +`installedShadowAdmitter` checks a separate, release-installed +`shadow-qualifications.json` beside the trusted worker. Replay promotion evidence +does not authorize prospective execution. The bounded, regular, no-follow file is +read on every check, so removing an entry revokes subsequent admission. Exactly +one entry must match the saved qualification hash, current manifest, installed +engine/Python/hardware identity, sample rate, active configuration and complete +execution policy. Evidence includes reviewed fixture, pod and held-out artifacts +and explicitly acknowledges the host-context timing limitation. + +Admission also requires an applied, fresh selected-side runtime receipt matching +the generation captured before the attempt, plus fresh memory, disk, CPU load, +event-loop, ingestion and temperature observations under the entry's health +policy. A same-hash reset still changes generation and stops admission. Missing, +ambiguous, stale or asynchronous observations refuse execution. + +The callbacks are trusted host collectors, not request fields. Concrete collectors +must verify installed artifacts and aggregate all relevant daemon receipts; the +current aggregate receipt contract is not itself proof that those components ran. +Production integration must also guard relevant opposite-side binding changes. +No production entry or collector is installed here, and the admission module +does not authorize activation or provide an API for writing qualification files. + +`shadowRuntimeBindingReader` supplies the aggregate binding observation from +read-only configuration and biometrics handles. It freezes expected generations +and hashes for both physical channels, validates desired activation hashes and +bounded runtime projections against the shipped manifest and resolved config, +then requires the latest heart-rate, breathing-rate and HRV receipts to be applied +at those exact generations. It never falls back to an older applied generation. +Missing or pending receipts, future boundaries/heartbeats, and changed opposite-side +bindings refuse observation. The minimum of all six heartbeats lets the admission +policy enforce freshness for every component. Sleep-stage receipts are on-demand, +so they are not evidence of continuous numerical health. + +The collector uses nested read snapshots and checks caller-owned database identity +before and after capture. These are observations, not a cross-database lock or a +promise that activation cannot occur after a read; the attempt must keep checking +admission. It opens no files and writes no runtime state. Host construction still +needs to capture both expected baselines, provide file-identity guards, and connect +this collector with installed build identity and resource-health observations. + + +## Durable completion receipt + +Migration 0030 adds nullable receipt and receipt-hash fields without inventing +measurements for old runs. Completing an executable plan requires a validated +worker receipt in the same fenced transaction as terminal state. The bounded +16 KiB envelope binds run, attempt, plan hash and completion time, and retains +worker health/counters, CPU time, transport bytes, sampled process-group RSS, +sampling count/gap and frozen child budgets. Its storage is bounded per run in +addition to sample-byte quotas and the existing run-count limit. + +Store reads verify the envelope hash and identities and recheck measurements +against the saved execution policy. Snapshots and reports retain this evidence. +RSS remains a sampled sum across the process group, not an exact kernel peak; +recording a receipt does not establish pod qualification or activation authority. +Failed/cancelled attempts currently have no successful worker receipt, and crash +recovery/partial resource evidence remains separate work. Receipt-validation +failure after cleanup marks the same attempt failed, including the expired-lease +fallback, without overwriting a replacement attempt. + +## Host health sampling + +`openShadowHealthMonitor` connects the existing Linux memory, filesystem space, +load, event-loop delay and thermal collector to a trusted live piezo receive-clock +callback supplied by the application. +The host supplies that callback, the database directory and installed qualification policy. +The monitor waits for an initial collection, then exposes a synchronous `read` +for admission. Every read re-evaluates health and freshness against that policy; +missing, stale, future or unhealthy evidence returns null. + +Refresh cadence and collection deadline are each half the allowed sample age, +capped at one second. Only one collection can be in flight. A rejection or timeout +closes the monitor; uncancellable filesystem I/O is observed but never retried or +allowed to publish a late result. Abort/close cancels timers and invalidates cached +evidence. Successful null/unhealthy samples can be refreshed, but never admitted. +The attempt's next admission check stops it when the monitor cannot provide health. +Host service ownership, installed build identity and qualification selection still +need to wire this monitor into the complete attempt lifecycle. + +## Shadow Python build identity + +`probeShadowBuild` invokes the fixed `common.biometrics.shadow_build` probe using +the same interpreter and module directory as the worker, with sanitized thread +settings, a ten-second deadline and a 64 KiB output bound. It checks the canonical +identity hash and requires the shadow-specific source entries. Replay fingerprints +alone cannot qualify a shadow worker. + +The Python fingerprint uses `shadow-python-build-v1` and covers all shared replay +sources and locked dependency metadata plus `shadow.py`, `shadow_worker.py`, +`shadow_process.py` and the probe itself. Thus prospective processing, JSONL +transport and kernel-limit changes invalidate qualification. The probe neither +imports the numerical processor nor opens live sources. Dependency RECORD/WHEEL +hashes identify installation metadata, not every installed binary's contents. +Production composition must still bind the shipped Node host bundle, capture +hardware identity, and revalidate installation identity during execution. + +## Owned attempt host + +`openPlannedShadowHost` composes the installed qualification, both-side runtime +binding collector, live health monitor and saved-policy source factory. It owns +separate read-only admission connections with file-identity guards and the saved +busy timeout. Qualification lookup precedes health sampling, and initial admission +must pass before the host is returned. Subsequent admission also requires the +entire saved plan to match. Installation paths, candidate configs and expected +baseline identities are privately copied; they are trusted host inputs, never +arbitrary RPC executable paths or identity assertions. + +`runPlannedShadowAttempt` passes this host to the leased controller and closes its +resources on success or failure. Closing aborts the attempt first, then closes +health and admission connections. Partial startup and cancellation while waiting +for initial health also unwind ownership, including a late returned health handle. +The source factory separately owns its capture connections and subscriptions. + +This composition still requires production code to resolve candidate/baseline +configs, capture hardware and shipped Node/Python identity, and supply a synchronous +identity collector that detects changes. It does not recover orphan workers, +register APIs/UI, install qualification entries or create production budgets. + +## Resolving candidate profiles and active baselines + +`resolveShadowProfiles` reads the saved candidate revision through the existing +immutable profile validator, including side, pinned manifest, override rebuild and +dependency checks. A null revision explicitly requests the shipped default. The +resolved candidate must match the saved candidate hash. + +Both active baselines come from desired activation generations and their bounded +runtime projections in nested read snapshots. The full runtime binding collector +then verifies all numerical receipts and projection identities; no unacknowledged +fallback is allowed. The selected active hash and both candidate execution hashes +must match the plan. The opposite channel retains its acknowledged configuration. +This operation requires read-only handles and pre/post file-identity guards and +writes no activation, receipt, repair or canonical measurement. + +`runProfiledShadowAttempt` owns temporary profile-resolution handles and closes +them before constructing the admission host. Admission checks the captured +identities again, so changes between resolution and execution are rejected. This +entry point accepts a saved revision ID rather than caller-asserted configs or +active generations. Trusted installed build/hardware identity, service ownership, +crash recovery, qualification and public API/UI integration remain required. + + +## Packaged host identity + +The standard biometrics build also produces `shadow-host.cjs` and its dependency +manifest. This CommonJS module exports the profiled attempt runner, store/plan +schemas, installed qualification reader and build probe; importing it starts no +service. The build rejects hardware, automation, scheduler, HomeKit, live database +singletons and streaming entrypoints. Only the passive `prospectiveFrameTap` is +allowed from streaming. The application supplies the actual receive-clock callback, +so the bundle does not initialize another stream or hardware controller. + +`shadowEngineBuildIdentity` hashes the exact host bundle bytes, prospective Python +fingerprint and Node runtime identity. The real-package test loads the exported +API, inspects its dependency graph, compares packaged/source probe identities and +proves changing the host artifact changes the engine fingerprint. Production must +load and execute this same bundle and guard installation changes during a run; +fingerprinting an unused file would not establish execution identity. Hardware +capture, recovery, service/API/UI registration and current-build qualification +remain required before production runs. + +## Loading the verified package + +`loadInstalledShadowHost` reads a trusted regular-file artifact with no-follow +semantics and a 16 MiB bound, hashes its bytes, and checks its device/inode, size, +mtime and ctime before and after loading. It rejects an unsupported API version, +unverified existing Node cache entry, changed installation or different bundle +path after the first load. The verified package is held on `globalThis`, avoiding +multiple independent loader instances under the application bundler. Parent paths +are canonicalized for Node's cache key while leaf symlinks remain forbidden. + +The returned `assertCurrent` checks both canonical and requested paths; replacement +with identical bytes still requires restart. `inspectInstalledShadowHost` probes +through that loaded API, rechecks the artifact after the asynchronous Python probe, +and verifies the reported host hash matches the loaded bytes. These guards cover +the Node artifact. Python source/dependency change monitoring and hardware capture +remain separate requirements, and the application must actually use this loader +and its guard when registering production execution. + +## Python installation change guard + +The shadow probe now supplies absolute RECORD/WHEEL locations for the installed +NumPy and SciPy distributions; ambiguous or missing metadata paths refuse the +probe. Locations remain outside the portable build hash. The identity still +contains source and metadata content hashes. + +`guardShadowPythonInstallation` validates every listed source and dependency +metadata file against the probe with bounded reads (16 MiB per artifact), rejects +traversing source paths and relative dependency paths, and records requested and +canonical file identities. Metadata text normalization matches Python's text +hashing. Subsequent checks detect device/inode, mode, size, mtime/ctime or link-target +changes without rehashing all content each heartbeat. The interpreter link and +resolved target are watched too. `inspectInstalledShadowHost` now returns a +combined Node/Python guard after verifying the actual loaded package. + +This covers shipped source, installed wheel metadata, and interpreter replacement; +it is not a full native-library binary attestation. The guard must still be called +by production admission alongside actual hardware identity. Service registration, +crash recovery and current-build qualification remain outstanding. + +## Observed hardware identity + +`observedShadowHardware` reads the existing DAC monitor without starting it or +sending commands. It requires running status, a successful poll timestamp within +the qualification's explicit `maximumHardwareObservationAgeMs`, and a recognized +H/I/J sensor revision consistent with the reported pod version. Unknown labels do +not inherit the general status parser's Pod 3 fallback. The hardware ID contains +pod version and exact sensor revision, never a serial number. The 500 Hz value is +the shipped waveform protocol rate; frame admission separately rejects an explicit +incompatible frequency. + +`installedShadowIdentity` combines that observation with the inspected Node/Python +fingerprints and invokes the combined artifact guard on every call. Build or +hardware mismatch rejects admission. This adapter lives on the application side +of the package boundary, so hardware services remain excluded from the passive +host bundle. Production service execution must still supply this collector and +register cancellation/recovery and user-facing operations. + +## Installed application runner + +`runInstalledShadowAttempt` connects the application to the verified package. It +inspects the installed host, opens an owned writable state connection to the +existing biometrics database, and constructs the package's own `ShadowStore`. +The state connection uses an explicit timeout no greater than the saved execution +policy and the same file-replacement guard as read-only capture connections. + +The runner selects the exact installed qualification, builds the observed +hardware/artifact identity collector, supplies the actual shared piezo receive +clock, and invokes the loaded package's `runProfiledShadowAttempt`. Every identity +check also verifies the writable state database still refers to the owned file. +The state handle closes after completion, cancellation or any startup/execution +failure. Capture handles remain read-only; only the store owns shadow writes. + +Service code supplies trusted installation paths, storage policy and timeout; +operations identify an existing run and saved candidate revision. This entry point +does not register background scheduling or public routes, and does not establish +cross-process orphan recovery or concurrent resource admission. Those lifecycle +requirements and current-build qualification remain before production enablement. + +## Host-crash worker termination + +The Node supervisor now passes its own PID to the fixed Python bootstrap. Before +resource-limit setup or numerical imports, the bootstrap requires Linux and arms +`PR_SET_PDEATHSIG` with `SIGKILL`. It checks the expected parent before and after +arming, so a host that dies during startup cannot leave an unprotected worker +processing frames. Missing ctypes/libc support or kernel refusal stops startup. +The expected PID is launch metadata, not a request field. + +A real Linux test launches a detached worker, waits for it to arm protection, +kills its launcher and verifies that the worker terminates. Existing CPU and +address-space kernel tests also pass. This is process-local protection, not a +process-group inheritance guarantee: Linux clears it in forked children. The +shipped numerical worker does not launch candidate subprocess trees. Parent-side +group cleanup remains required, along with durable attempt recovery and resource +admission across concurrent service processes. + +## Kernel-held replay/shadow execution slot + +The shadow bootstrap and waveform replay CLI take a nonblocking exclusive flock on the fixed +`/run/sleepypod-shadow.lock` before numerical imports and holds its descriptor for +the worker lifetime. A second numerical worker fails startup even if its +SQLite lease was granted after a paused worker's lease expired. File creation is +private, no-follow and non-truncating; nonregular, linked, nonempty, wrong-owner or +publicly accessible files are refused. No request/CLI argument selects the path. + +The runtime file is never unlinked during operation. Descriptor close or process +death releases kernel ownership without stale-file cleanup. A real Linux test +verifies exclusion while the owner is stopped with SIGSTOP and acquisition after +SIGKILL/reaping, including the actual replay CLI blocking a shadow contender. +Both entrypoints use the same shared implementation, included in their build +identities. The legacy runtime path is retained for shadow upgrade compatibility. +Build-description probes do not take the slot because they do not run numerical +processing. Lock contention fails startup rather than waiting with an active lease. + +This serializes shipped waveform replay and shadow workers sharing the runtime +path. It does not serialize Node stage replay or archive collection, replace +qualified aggregate host admission, or prove expired-attempt process cleanup. +Before enabling a new installation, stop/reap old replay workers that predate +this lock; they cannot retroactively acquire it. Never delete the lock file to +force admission while any worker may still own its inode. + +## Worker-local wall deadline + +The supervisor passes the saved `wallMs` budget to the Python bootstrap, which +arms `ITIMER_REAL` before numerical imports. It installs the default terminating +SIGALRM action and explicitly unblocks that signal. This supplements the parent's +wall timer, so an idle worker does not depend on Node's event loop to enforce its +elapsed-time budget. No alternate production timeout is selected by the worker. + +The real Linux test starts with SIGALRM blocked, arms a 100 ms deadline, sleeps, +verifies signal termination and then reacquires the execution lock. Parent-side +process-group cleanup remains necessary; a process explicitly stopped by SIGSTOP +cannot handle ordinary pending signals until resumed, and the lock continues to +exclude competing shadow workers in that condition. + +## Bounded expired-run reconciliation + +`ShadowStore.reconcileExpired` scans at most 100 runs per immediate transaction +using a primary-key cursor, including future and terminal rows so maintenance +cannot get stuck on the first unexpired window. It validates candidate plans and +compares state, attempt and lease when changing authority. A corrupt plan rolls +back the batch. Unexpired replacement attempts and terminal evidence are preserved; +final-drain authority remains valid until its lease/deadline expires. + +Expired running attempts become failed with +`shadow_lease_expired_cleanup_unconfirmed`. This revokes append/renew authority +without claiming that an OS process was reaped. Retained samples and attempt IDs +remain available, reports flag unconfirmed cleanup, and the kernel slot continues +to exclude overlapping shadow workers. Pending runs whose observation window was +missed become failed with `shadow_observation_window_missed`; reports do not call +that a processor failure or fabricate observations. + +`reconcileInstalledShadowRuns` exposes this maintenance through the loaded package's +store with owned state-handle cleanup and artifact/database guards. It starts no +Python probe, worker or hardware operation and needs no admission qualification. +Periodic service scheduling must retain the returned cursor until a sweep ends, +then start another sweep. Recovery revokes authority; process cleanup confirmation +and shared replay/shadow resource ownership are separate lifecycle requirements. + +## Application service ownership + +`startShadowService` creates one `globalThis` service using explicit installation, +storage and maintenance policy. Repeated starts reuse the same configuration; +configuration changes require stopping/restarting it. The service snapshots its +configuration, runs bounded recovery pages at the configured cadence, retains its +cursor after a failed transaction, and restarts the scan after a completed sweep. +Recovery continues during a prospective attempt. + +Only one local attempt may execute. Cancellation and shutdown abort that attempt +and await the installed runner's cleanup before releasing its slot. Application +shutdown calls `stopShadowService` before database closure. The database lease and +Linux worker lock still enforce authority across processes; this local owner does +not replace them or serialize replay workers. + +Application instrumentation starts this service after migrations and hardware +initialization when explicit deployment configuration is present. CI skips it +along with hardware initialization. No production resource thresholds or +qualification evidence are inferred by this service. The inspection, creation +and cancellation APIs and browser workflow use this service as described below. + +### Deployment configuration + +Set `SLEEPYPOD_BIOMETRICS_SHADOW_SERVICE` to a JSON object conforming to +`shadowServiceConfigurationSchema` in `startup.ts`. Only `planTemplates` is optional; +unknown fields and input larger than 16 KiB are rejected. + +| Field | Meaning | +| --- | --- | +| `version` | Exactly `shadow-service-configuration-v1` | +| `pythonPath` | Absolute path to the installed locked Python interpreter | +| `modulesPath` | Absolute path to the installed shared Python modules | +| `presenceFile` | Absolute path to the producer's committed presence file | +| `storagePolicy` | Positive integer `maximumRuns`, `maximumPoints`, `maximumBytes` quotas | +| `busyTimeoutMs` | SQLite wait limit, 1–1000 ms, also bounded by each saved execution policy | +| `recoveryIntervalMs` | Recovery and scheduling cadence, 10–300000 ms | +| `recoveryMaximumRuns` | Maximum rows per recovery/scheduling page, 1–100 | +| `planTemplates` | Optional array of up to 8 uniquely named preparation templates; no templates are inferred | + +These are schema bounds, not recommended pod budgets. Choose values from measured +qualification evidence. The two database paths come from the application's open +SQLite databases, and the Node host comes from the fixed packaged build location; +neither can be overridden in this configuration. Missing configuration does not +start the service. Invalid configuration, nonpersistent databases or a missing +package leave it unavailable with a generic diagnostic, without exposing paths or +environment values. Setting valid JSON never creates qualification evidence: +every attempt must still match the installed registry and pass fresh admission. + +### Durable prospective scheduling + +Plans now include `candidateRevisionId`: a saved revision UUID or explicit `null` +for the shipped default. This identity is covered by the plan hash and idempotency +check. Legacy plans remain readable, but the installed execution boundary rejects +missing or mismatched revision identities. Profile resolution still validates the +saved configuration hashes and both acknowledged physical-channel baselines. + +The service scans bounded primary-key pages through the installed package's store +at its maintenance cadence, with a separate scheduling cursor. It selects only +pending executable plans with a saved revision whose observation window is open. +Future plans are revisited on later sweeps; terminal, running and legacy plans are +skipped. Each selected run goes through ordinary qualification, admission and lease +claim checks. A selection is not a reservation and cannot bypass cross-process +authority. Scheduled attempts are serialized locally while recovery continues. +Errors are observed and reported without disabling subsequent sweeps. + +After restart, pending plans remain schedulable from their durable identity. Running +attempts are never resumed by this queue; expired ones retain their samples and are +failed by recovery. Scan cadence and queue size may delay a start, so coverage is +measured from actual samples and missed windows are recorded explicitly. + +## Inspection and cancellation API + +The `biometricShadow` tRPC router and corresponding LAN REST endpoints expose: + +- `GET /api/biometric-tuning/shadow-runs`: required side, optional `afterId` + primary-key cursor and `maximumRuns` (1–50, default 20). +- `POST /api/biometric-tuning/shadow-runs`: a complete proposed `plan` and + `idempotencyKey`. The plan must include candidate revision (or explicit default), + both active generations/hashes, execution policy and prospective window. +- `GET /api/biometric-tuning/shadow-runs/{id}`: integrity-checked metadata. +- `POST /api/biometric-tuning/shadow-runs/{id}/stop`: durable stop followed by + awaited local attempt cancellation. Repeated calls preserve terminal evidence. +- `GET /api/biometric-tuning/shadow-runs/{id}/report`: aggregate health, coverage, + pairing, resource receipt and evidence hashes. +- `POST /api/biometric-tuning/shadow-runs/{id}/compare`: the same report with an + explicit `reference` import matching the saved sleeper and side. Submitted + reference measurements are compared without persisting an import. + +Inspection and stop use the application's database and remain available without +execution configuration, an installed worker, current health or qualification. +A store opened without storage policy cannot create, claim or append observations. +Metadata includes hashes, window, state, counts and saved candidate identity, but +not waveforms or deployment paths. Legacy missing revision identity is explicitly +distinguished from the shipped default through `candidateIdentitySaved`. + +Stopping clears database authority before waiting for local worker cleanup. The +response's `localAttemptSettled` indicates whether a locally owned attempt was +awaited; it does not claim that another process's worker was reaped. Retained +samples and completed/failed results are preserved. + +Reports execute in the shipped coordinator process's bounded read lane, leaving +the application's event loop free. They require a complete integrity-checked +snapshot of at most 100000 points and 32 MiB; larger evidence is refused rather +than sampled. The coordinator also enforces its 64 KiB request, 17 MiB response, +384 MiB Node heap and wall-clock limits. These bounds are not claims of measured +pod resource qualification. Reports require the installed coordinator package but +do not launch a waveform processor or require execution qualification. + +The public response has an explicit schema. Reference summaries include the +shared normalized import content hash; the report hash covers that summary, +snapshot identity and comparison results. No raw samples or submitted reference +annotations are returned. Agreement remains descriptive, never activation +authorization or an independent holdout claim. + +MCP uses these same router operations: `inspect_biometric_calibration` actions +`shadow_runs`, `shadow_run`, `shadow_report` and `shadow_compare` are read-only; +`stop_biometric_shadow` is an explicit idempotent mutation. +`schedule_biometric_shadow` submits a proposed plan through the same creation +operation. + +The calibration screen's Observations tab lists runs for the selected side, +shows UTC windows and saved identities, and can record a stop request. Detail +polling ends for terminal runs; list polling continues to discover new work. +Reports are opened on request and refreshed explicitly rather than recomputed +every status poll. They distinguish unavailable coverage/resource receipts from +zero values and show operational issues without implying validated improvement. +The report UI also compares pasted CSV/JSON reference records for the run's saved +side and sleeper identity. It checks the serialized 64 KiB coordinator request +bound, invalidates old results on edits, and prevents overlapping comparisons. +Results show accepted/rejected counts, outside-window exclusions, alignment +offset/tolerance, shared reference pairs, shared-support MAE, each output's +reference coverage and lost/added observation timestamps. Missing scores remain +unavailable; coverage loss and incompatible definitions remain visible. The +comparison is a fresh snapshot, identified separately from the earlier health +report, and does not persist a reference import. Browser verification and +translation-catalog completion remain release work. + +### Creation admission + +Submitted hashes, generations and policy are assertions, not authority. Creation +loads the verified installed package, checks its Python identity and unique +installed qualification, resolves the stored candidate and acknowledged active +profiles, and observes current hardware and health. It opens admission resources +without claiming a lease, launching a numerical worker or subscribing to sensor +frames. Final admission runs synchronously inside the creation transaction; +failure leaves no queued run. Handles and health sampling close on every exit. +Execution repeats these checks at the actual observation window. + +Both active generations are frozen in the plan. A same-hash reset/rollback on +either side invalidates subsequent profile resolution. Legacy plans lacking +candidate or generation identity remain inspectable but are not scheduled by the +installed execution path. Exact idempotent retries return existing evidence before +new Python/health probes; changed requests with the same key conflict. Creation +still requires explicit server deployment configuration and the installed package. +There are no caller-supplied executable paths, numerical configs or health flags. + +### Server-side plan preparation + +Deployment `planTemplates` supply a bounded ID/label, qualification hash, maximum +window, frozen execution and monitoring policy, joining limits, selected metrics, +optional IBI window and storage ceilings. Their maximum window plus final drain +must fit the worker wall budget; their point/byte ceilings must fit deployment +storage policy. These are explicit operator-installed values, not production +defaults or proof of qualification. No template ships enabled by default. + +`GET /api/biometric-tuning/shadow-templates` returns only template IDs, labels and +maximum windows. `POST /api/biometric-tuning/shadow-plans/prepare` accepts template +ID, side, sleeper identity, saved candidate revision (or explicit default) and +start/end times. The verified package checks the template against the installed +qualification registry and manifest, then captures acknowledged active generations +and candidate configuration from owned read-only database handles. Both physical +channel config hashes and the active baseline must match the template's evidence. + +The response contains the full frozen plan and hash with status +`prepared_not_admitted`. Preparation opens no waveform source, runs no Python +worker and queues nothing. Fresh installed-build, hardware and health admission +remain mandatory at creation and execution. Changed profiles or generations make +the prepared plan stale rather than silently replacing its inputs. + +MCP exposes `shadow_templates` and `shadow_prepare` through the read-only +`inspect_biometric_calibration` tool. Scheduling remains the separate explicit +`schedule_biometric_shadow` operation with its immutable retry key. + +The Observations UI now provides template selection, explicit default or saved +candidate selection with profile pagination, sleeper identity and UTC window +inputs. It preserves exact sleeper IDs and subsecond timestamps. Preparation is +a separate action from scheduling, and edits invalidate the prepared review. +The review shows the server-captured window, generations, candidate and plan hash. +No template is invented when deployment configuration is missing. + +Scheduling freezes the prepared plan and a random request key. After an uncertain +response, the open form keeps those inputs locked and retries the same request +without re-preparing it. Explicit bad-request or quota rejection unlocks editing; +confirmed saves expose a separate new-plan action and refresh the run list. A +saved request is never presented as proof that execution started. Retry state is +saved before sending in same-origin, per-side session storage, bounded to 64 KiB. +Reloading this tab restores the exact plan and key without preparing or sending +automatically, even if template/profile discovery is unavailable. Confirmed saves +and definite rejections clear the record. A failed storage write prevents sending; +invalid or inaccessible recovery blocks new scheduling while preserving the record. +Closing the tab, clearing browser data, or moving to another origin does not carry +recovery forward. The record contains plan metadata, not reference rows or waveforms. + + +### Unconfirmed local cleanup + +A process-group kill refusal or process cleanup confirmation deadline raises +`SHADOW_CLEANUP_UNCONFIRMED`, preserving the original worker failure as its cause. +This result takes precedence over an earlier cancellation or processing error; +the attempt writes `shadow_worker_cleanup_unconfirmed` when its fenced state +still permits a failure update. Existing terminal operator-stop evidence is not +rewritten. Reports label this failure as unconfirmed cleanup. + +The local service quarantines numerical scheduling after that result, while +continuing bounded database recovery. Local cancellation returns false instead +of claiming settlement; ordinary task failures with settled cleanup still release +the slot. The marker is recognized across the separately packaged host boundary. +This is local service state, not durable proof of OS cleanup: restarting a host +must not be treated as verification that old processes are gone. + +The supervisor now requires both the direct-child close event and `ESRCH` from +a signal-0 probe of the detached process group within a five-second deadline. +Surviving members, zombies, or persistent permission/observation failures leave +cleanup unconfirmed. Transient unknown observations are retried, never counted +as absence. No probe signals PID/group zero. These checks cover the original +detached group; they cannot prove cleanup of descendants that deliberately escape +into another group. Shipped numerical code must not create such descendants. +The shared kernel lock remains required across hosts. + + +### Browser demo + +The demo implements all eight observation procedures in browser memory. Its +explicitly labeled synthetic heart-rate template supports preparation, scheduling, +exact idempotent retries, side-specific paginated inspection, stopping, reporting +and reference comparison. Plans are accepted only after preparation in this demo +session, and changed profile generations invalidate an unsubmitted plan. Bounds +are 100 prepared plans, 20 runs and 120 samples per run over at most ten minutes. + +Synthetic active readings repeat 60/61/62 bpm; candidate readings are one bpm +lower and omit every fourth value. They do not execute the selected algorithm. +The demo shares production report calculations, normalization and reference +alignment, so missing coverage is penalized and incompatible records remain +unscored. Reports include a visible simulation issue, no worker resource receipt, +and no qualification or activation authorization. Synthetic plan identities are +not installed in a production registry. No waveform, hardware, DB or network +adapter is imported into the browser demo graph. + +A stop freezes saved synthetic samples. Repeated stop preserves terminal state; +no local process settlement is claimed because the demo has no process. Reloading +resets demo state. A scheduling retry from a previous browser session receives an +explicit bad-request result instead of silently creating a replacement run. + + +### Partial resource evidence + +After supervising cleanup, the child host reports a bounded snapshot even when +processing, input, cancellation or cleanup fails. The last protocol-validated +health reply is retained; it may be absent or stale and is not final CPU usage. +Input bytes count attempted serialized commands; output and diagnostic bytes +count observed transport chunks. RSS is sampled process-group RSS, not an exact +kernel peak; no successful sample is represented by null, never zero. Over-budget +observations remain evidence rather than failing completed-receipt validation. +The observation gap includes the unsampled cleanup tail. + +Completed attempts retain the original `biometric-shadow-receipt-v1` format. +Failed/stopped attempts use `biometric-shadow-partial-receipt-v1` in the same +hash-protected receipt columns, tied to run, plan and attempt. Attachment requires +the exact terminal attempt, cannot change its state/reason/end time, and cannot +replace evidence. An identical retry is idempotent. A later receipt timestamp is +allowed because an operator can stop before process cleanup settles. Old attempts +cannot annotate replacement attempts or completed runs. Missing/rejected writes +remain failures; a cleanup-unconfirmed error still quarantines the host even if +storing its receipt also fails. A host crash can leave no receipt; recovery does +not fabricate measurements or proof of cleanup. + +Reports expose partial receipts through the existing bounded report route and +show a partial-evidence notice; missing RSS is unavailable. Unconfirmed cleanup +is an issue even when an operator's stop reason remains authoritative. This does +not qualify a candidate, prove exact resource peaks, or authorize activation. + + +### Shared application background capacity + +`runInstalledShadowAttempt` and the coordinator supervisor reserve the same +`globalThis` work slot before starting probes, processes, scans or a shadow claim. +The coordinator's exhaustive task/lane map is shared with its queue: stage and +waveform execution, dataset construction/release, experiment creation, RAW +inspection and artifact cleanup are work. One separately reserved read slot keeps +reports available. The existing queue still bounds requests and wait times; +capacity conflicts return `RESOURCE_LIMIT` before starting work. A busy shadow run +remains pending and the service retries it on a later scan; no historical samples +are invented if its window is ultimately missed. + +Ownership lasts until the supervisor settles cleanup. Coordinator completion now +confirms disappearance of the original detached process group after child close, +using the same bounded signal-zero check as shadow cleanup. Unconfirmed cleanup +quarantines that lane instead of admitting replacement work. Ordinary errors +release ownership; cancellation alone does not. Read diagnostics remain available +when the work lane is quarantined. There is no API to clear quarantine: verify OS +cleanup before restarting the application. + +This gate bounds the application process across duplicate module bundles; it is +not an OS-wide lock. The existing private kernel flock still excludes numerical +replay and shadow workers across processes. Deployment must retain a single +application owner and retire old worker processes before startup. Independently +launched Node coordinators in another application process do not share this gate. +Production qualification must measure live processing plus one read process and +one admitted work task, including peak RSS, ingestion delay and storage reserves; +this concurrency bound does not replace per-worker or live-health limits. diff --git a/src/biometrics/shadow/active-database.ts b/src/biometrics/shadow/active-database.ts new file mode 100644 index 000000000..496b6d96a --- /dev/null +++ b/src/biometrics/shadow/active-database.ts @@ -0,0 +1,43 @@ +import Database from 'better-sqlite3' +import { lstatSync } from 'node:fs' +import { z } from 'zod' + +/** Trusted host path, never an RPC path. Keep ownership separate from source + * subscriptions so shutdown can unsubscribe before closing the connection. */ +function openOwnedShadowDatabase(path: string, options: { busyTimeoutMs: number }, readonly: boolean) { + const timeout = z.number().int().min(1).max(1000).parse(options.busyTimeoutMs) + const identity = () => { + const stat = lstatSync(path, { bigint: true }) + if (!stat.isFile() || stat.isSymbolicLink()) throw new Error('Shadow active database must be a regular file') + return `${stat.dev}:${stat.ino}` + } + const expected = identity() + const db = new Database(path, { readonly, fileMustExist: true, timeout }) + let closed = false + const close = () => { + if (closed) return + closed = true + db.close() + } + const assertCurrent = () => { + if (closed || !db.open || identity() !== expected) throw new Error('Shadow active database was closed or replaced') + } + try { + assertCurrent() + } + catch (error) { + close() + throw error + } + return { db, assertCurrent, close } +} + +export function openShadowActiveDatabase(path: string, options: { busyTimeoutMs: number }) { + return openOwnedShadowDatabase(path, options, true) +} + +/** The installed runner owns this writable handle for separate shadow tables. + * The store, not this connection wrapper, enforces the canonical-write boundary. */ +export function openShadowStateDatabase(path: string, options: { busyTimeoutMs: number }) { + return openOwnedShadowDatabase(path, options, false) +} diff --git a/src/biometrics/shadow/active-reader.ts b/src/biometrics/shadow/active-reader.ts new file mode 100644 index 000000000..c8084afcc --- /dev/null +++ b/src/biometrics/shadow/active-reader.ts @@ -0,0 +1,95 @@ +import type Database from 'better-sqlite3' +import { z } from 'zod' +import { biometricManifest, manifestHash, validateConfig } from '../algorithms/registry' +import { configHash, dataHash } from '../algorithms/canonical' +import { shadowActiveReadingSchema } from './worker-sink' + +const integer = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) +const optionsSchema = z.object({ side: z.enum(['left', 'right']), startMs: integer, endMs: integer, + activeConfigHash: z.string().regex(/^[a-f0-9]{64}$/), maximumRows: integer.min(1).max(100) }).strict() +const rowSchema = z.object({ id: integer.positive(), side: z.enum(['left', 'right']), timestamp: integer, + heart_rate: z.number().finite().nullable(), hrv: z.number().finite().nullable(), breathing_rate: z.number().finite().nullable(), + processing_run_id: z.string().uuid(), run_side: z.enum(['left', 'right']), kind: z.literal('live'), + config_hash: z.string(), bundle: z.string(), input_identity: z.string(), started_at_ms: integer, ended_at_ms: integer.nullable(), + quality_count: z.literal(1), quality_score: z.number().min(0).max(1), quality_side: z.enum(['left', 'right']), quality_timestamp: integer, +}) +const bundleSchema = z.object({ schemaVersion: z.literal(1), manifestHash: z.string(), resolved: z.unknown(), + generation: integer, revisionId: z.string().uuid().nullable(), components: z.unknown(), sourceHashes: z.unknown() }).strict() +const inputSchema = z.object({ sourceSide: z.enum(['left', 'right']), timestampBasis: z.literal('processing_clock'), + rawIdentity: z.literal('not_retained'), sensorCalibration: z.literal('not_consumed') }).strict() + +/** Poll an already-open read-only connection. Only observations inserted after + * construction are eligible; historical rows are never replayed as live output. + * The caller controls cadence and must fail its attempt on any reader error. */ +export class ShadowActiveReader { + private readonly options: z.infer + private cursor: number + private anchorHash: string | null + + constructor(private readonly db: Database.Database, options: unknown, private readonly now = Date.now) { + this.options = optionsSchema.parse(options) + if (this.options.startMs >= this.options.endMs || this.options.endMs - this.options.startMs > 7 * 86400000) throw new Error('Invalid active observation window') + if (!db.readonly && db.pragma('query_only', { simple: true }) !== 1) throw new Error('Active capture requires a read-only connection') + const anchor = this.latest() + this.cursor = anchor?.id ?? 0 + this.anchorHash = anchor ? dataHash(anchor) : null + } + + private latest() { + const row = this.db.prepare('SELECT id, side, timestamp, heart_rate, hrv, breathing_rate, processing_run_id FROM vitals ORDER BY id DESC LIMIT 1') + .get() as { id: number } | undefined + if (row) integer.positive().parse(row.id) + return row + } + + poll() { + return this.db.transaction(() => { + if (this.cursor) { + const anchor = this.db.prepare('SELECT id, side, timestamp, heart_rate, hrv, breathing_rate, processing_run_id FROM vitals WHERE id = ?').get(this.cursor) + if (!anchor || dataHash(anchor) !== this.anchorHash) throw new Error('Active observation cursor changed or was removed') + } + const latest = this.latest() + if (!latest || latest.id < this.cursor) { + if (this.cursor) throw new Error('Active observations reset') + return [] + } + const { side, startMs, endMs, maximumRows, activeConfigHash } = this.options + const rows = this.db.prepare(`SELECT v.*, r.side AS run_side, r.kind, r.config_hash, + CASE WHEN length(CAST(r.bundle AS BLOB)) <= 65536 THEN r.bundle END AS bundle, + CASE WHEN length(CAST(r.input_identity AS BLOB)) <= 4096 THEN r.input_identity END AS input_identity, + r.started_at_ms, r.ended_at_ms, + (SELECT COUNT(*) FROM vitals_quality q WHERE q.vitals_id = v.id) AS quality_count, + (SELECT quality_score FROM vitals_quality q WHERE q.vitals_id = v.id LIMIT 1) AS quality_score, + (SELECT side FROM vitals_quality q WHERE q.vitals_id = v.id LIMIT 1) AS quality_side, + (SELECT timestamp FROM vitals_quality q WHERE q.vitals_id = v.id LIMIT 1) AS quality_timestamp + FROM vitals v LEFT JOIN biometric_processing_runs r ON r.id = v.processing_run_id + WHERE v.id > ? AND v.id <= ? AND v.side = ? AND v.timestamp >= ? AND v.timestamp < ? + ORDER BY v.id LIMIT ?`).all(this.cursor, latest.id, side, Math.ceil(startMs / 1000), Math.ceil(endMs / 1000), maximumRows + 1) + if (rows.length > maximumRows) throw new Error('Active observation poll quota exceeded') + const at = this.now() + const readings = rows.map((raw) => { + const row = rowSchema.parse(raw), timestampMs = row.timestamp * 1000 + const bundle = bundleSchema.parse(JSON.parse(row.bundle)), input = inputSchema.parse(JSON.parse(row.input_identity)) + validateConfig(bundle.resolved) + if (!Number.isSafeInteger(timestampMs) || timestampMs > at || timestampMs + 999 < row.started_at_ms + || (row.ended_at_ms !== null && timestampMs >= row.ended_at_ms) + || row.side !== row.run_side || row.quality_side !== row.side || row.quality_timestamp !== row.timestamp + || row.config_hash !== activeConfigHash || configHash(bundle.resolved) !== activeConfigHash + || bundle.manifestHash !== manifestHash || configHash(bundle.components) !== configHash(biometricManifest.components) + || configHash(bundle.sourceHashes) !== configHash(biometricManifest.sourceHashes)) throw new Error('Active observation provenance mismatch') + return shadowActiveReadingSchema.parse({ side, timestampMs, configHash: activeConfigHash, inputHash: null, + provenance: { processingRunId: row.processing_run_id, sourceSide: input.sourceSide, generation: bundle.generation, manifestHash: bundle.manifestHash }, + readings: [ + { metric: 'heart_rate', unit: 'bpm', windowMs: null, value: row.heart_rate, quality: row.quality_score }, + { metric: 'breathing_rate', unit: 'breaths/min', windowMs: null, value: row.breathing_rate, quality: row.quality_score }, + { metric: 'window_ibi_variability_v1', unit: 'ms', windowMs: 300000, value: row.hrv, quality: row.quality_score }, + ] }) + }) + // Advance only after the entire bounded snapshot validates. Unknown, + // malformed or substituted provenance can never be skipped silently. + this.cursor = latest.id + this.anchorHash = dataHash(latest) + return readings + })() + } +} diff --git a/src/biometrics/shadow/admission.ts b/src/biometrics/shadow/admission.ts new file mode 100644 index 000000000..f7bd1f887 --- /dev/null +++ b/src/biometrics/shadow/admission.ts @@ -0,0 +1,91 @@ +import { constants, openSync, closeSync, fstatSync, readSync } from 'node:fs' +import { dirname, join } from 'node:path' +import { z } from 'zod' +import { dataHash } from '../algorithms/canonical' +import { manifestHash } from '../algorithms/registry' +import { admissionPolicySchema, evaluateAdmission } from '../backtest/admission' +import { shadowPlanSchema } from './store' + +const hash = z.string().regex(/^[a-f0-9]{64}$/) +const integer = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) +export const shadowQualificationSchema = z.object({ qualificationId: z.string().min(1).max(200), + manifestHash: hash, engineBuildHash: hash, pythonBuildHash: hash, hardwareId: z.string().min(1).max(200), + sampleRateHz: z.literal(500), activeConfigHash: hash, executionPolicyHash: hash, + fixtureEvidenceHash: hash, podEvidenceHash: hash, heldOutEvidenceHash: hash, + contextEvidence: z.literal('host_observations_not_live_processor_cache_attestation'), + maximumHardwareObservationAgeMs: integer.positive().max(300000), + maximumRuntimeHeartbeatAgeMs: integer.positive().max(300000), healthPolicy: admissionPolicySchema, +}).strict().refine(entry => entry.healthPolicy.qualificationId === entry.qualificationId, 'Qualification policy identity mismatch') +export const shadowQualificationRegistrySchema = z.object({ version: z.literal('biometric-shadow-qualifications-v1'), + entries: z.array(shadowQualificationSchema).max(32) }).strict() +const identitySchema = z.object({ engineBuildHash: hash, pythonBuildHash: hash, hardwareId: z.string().min(1).max(200), sampleRateHz: z.literal(500) }).strict() +const bindingSchema = z.object({ side: z.enum(['left', 'right']), generation: integer, resolvedHash: hash, + appliedHash: hash, appliedGeneration: integer, state: z.literal('applied'), heartbeatAtMs: integer }).strict() + +/** Read release-installed authority beside the trusted worker entrypoint. The + * file has no write API. Missing, ambiguous or changed evidence denies admission. + * Evidence hashes refer to reviewed artifacts; arbitrary supplied JSON is not + * a qualification and callers must never choose this filesystem location. */ +export function readInstalledShadowQualifications(workerPath: string) { + let fd: number | undefined + try { + fd = openSync(join(dirname(workerPath), 'shadow-qualifications.json'), constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK) + const before = fstatSync(fd) + if (!before.isFile() || before.size > 256 * 1024) throw new Error('Invalid shadow qualification file') + const bytes = Buffer.alloc(256 * 1024 + 1) + let length = 0 + while (length < bytes.length) { + const count = readSync(fd, bytes, length, bytes.length - length, null) + if (!count) break + length += count + } + const after = fstatSync(fd) + if (length > 256 * 1024 || before.size !== after.size || before.mtimeMs !== after.mtimeMs) throw new Error('Shadow qualification file changed or exceeded its bound') + return shadowQualificationRegistrySchema.parse(JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes.subarray(0, length)))) + } + catch { throw new Error('Installed shadow qualification unavailable') } + finally { if (fd !== undefined) closeSync(fd) } +} + +/** All observations come from trusted host collectors, never RPC fields. + * Snapshot generation at factory creation prevents a same-hash reset/rollback + * from silently continuing an attempt across a changed active binding. */ +export function installedShadowAdmitter(options: { + workerPath: string + expectedBinding: { side: 'left' | 'right', generation: number, resolvedHash: string } + identity: () => unknown + binding: () => unknown + health: () => unknown + now?: () => number +}) { + const expected = z.object({ side: bindingSchema.shape.side, generation: integer, resolvedHash: hash }).strict().parse(structuredClone(options.expectedBinding)) + const now = options.now ?? Date.now, workerPath = options.workerPath + const identityReader = options.identity, bindingReader = options.binding, healthReader = options.health + const snapshot = (read: () => unknown) => { + const value = read() + if (value && typeof value === 'object' && 'then' in value && typeof value.then === 'function') { + void Promise.resolve(value).catch(() => {}) + throw new Error('Shadow admission observations must be synchronous') + } + return value + } + return (rawPlan: unknown): void => { + const plan = shadowPlanSchema.parse(rawPlan) + if (!plan.execution) throw new Error('Shadow execution policy is missing') + const registry = readInstalledShadowQualifications(workerPath) + const entries = registry.entries.filter(entry => dataHash(entry) === plan.qualificationHash) + if (entries.length !== 1) throw new Error('Shadow qualification missing or ambiguous') + const entry = entries[0], identity = identitySchema.parse(snapshot(identityReader)), binding = bindingSchema.parse(snapshot(bindingReader)), at = now() + if (!Number.isSafeInteger(at) || at < 0) throw new Error('Invalid shadow admission clock') + if (entry.manifestHash !== manifestHash || entry.engineBuildHash !== plan.engineBuildHash + || entry.engineBuildHash !== identity.engineBuildHash || entry.pythonBuildHash !== identity.pythonBuildHash + || entry.hardwareId !== identity.hardwareId || entry.sampleRateHz !== identity.sampleRateHz + || entry.executionPolicyHash !== dataHash(plan.execution) || entry.activeConfigHash !== plan.activeConfigHash) throw new Error('Shadow qualification does not match installed execution') + if (binding.side !== plan.side || binding.side !== expected.side || binding.generation !== expected.generation + || binding.resolvedHash !== expected.resolvedHash || binding.resolvedHash !== plan.activeConfigHash + || binding.appliedHash !== binding.resolvedHash || binding.appliedGeneration !== binding.generation + || at < binding.heartbeatAtMs || at - binding.heartbeatAtMs > entry.maximumRuntimeHeartbeatAgeMs) throw new Error('Shadow active binding changed, unapplied or stale') + const decision = evaluateAdmission(snapshot(healthReader), entry.healthPolicy, at) + if (!decision.admitted) throw new Error(`Shadow host admission refused: ${decision.reasons.join(',')}`) + } +} diff --git a/src/biometrics/shadow/attempt.ts b/src/biometrics/shadow/attempt.ts new file mode 100644 index 000000000..bae608af6 --- /dev/null +++ b/src/biometrics/shadow/attempt.ts @@ -0,0 +1,235 @@ +import type { ShadowPartialReceipt } from './receipt' +import type { ShadowExecutionPolicy } from './execution-policy' +import { isShadowCleanupError, ShadowCleanupError } from './cleanup-error' +import { z } from 'zod' +import { configHash, dataHash } from '../algorithms/canonical' +import { validateConfig } from '../algorithms/registry' +import type { ShadowStore } from './store' +import { type ShadowLease } from './store' +import { ShadowStreamJoin } from './stream-join' +import { shadowActiveReadingSchema } from './worker-sink' +import { runShadowChild } from './worker-child' + +type Plan = ReturnType['plan'] +export interface ShadowSources { + frames: AsyncIterable + /** Poll committed active observations before sealing join timestamps. */ + flush?(): void + /** Unsubscribe both streams synchronously, including on failed startup. */ + close(): void +} +export interface ShadowAttemptHost { + location: { pythonPath: string, modulesPath: string } + configs: { left: unknown, right: unknown } + /** Trusted synchronous check of installed qualification/build, active binding, + * resource admission and source health. Rechecked throughout the attempt. */ + admit(plan: Plan): void + /** Must register bounded source adapters; the frame iterator is stopped at + * collection end while active readings may arrive during the drain window. */ + openSources(input: { execution: ShadowExecutionPolicy, signal: AbortSignal, collectionEndsAtMs: number, fail: () => void, active: (reading: unknown) => void }): ShadowSources +} + +/** Execute one fenced attempt. A restart is a new claim and fresh processor; + * historical pending frames are never synthesized. The host owns crash recovery + * and must terminate any orphan process before admitting a replacement. */ +export async function runShadowAttempt(store: ShadowStore, runId: string, host: ShadowAttemptHost, signal?: AbortSignal) { + const plan = store.get(runId).plan + if (!plan.joining || !plan.execution) throw new Error('Shadow attempt requires frozen join and execution policies') + const execution = structuredClone(plan.execution) + const joining = plan.joining + z.number().int().min(1000).max(3600000).parse(execution.leaseMs) + z.number().int().positive().max(Math.min(1000, Math.floor(execution.leaseMs / 3), plan.monitoring.heartbeatStaleMs)).parse(execution.heartbeatMs) + const configs = structuredClone(host.configs) + const configHashes = { left: configHash(validateConfig(configs.left)), right: configHash(validateConfig(configs.right)) } + if (configHashes[plan.side] !== plan.candidateConfigHash || dataHash(configHashes) !== dataHash(execution.configHashes)) throw new Error('Shadow candidate differs from frozen plan') + if (signal?.aborted) throw new Error('Shadow attempt cancelled before claim') + const admit = (frozen: Plan) => { + const result: unknown = host.admit(structuredClone(frozen)) + if (result !== undefined) { + void Promise.resolve(result).catch(() => {}) + throw new Error('Shadow admission must complete synchronously') + } + } + const lease: ShadowLease = store.claim(runId, execution.leaseMs, admit) + const controller = new AbortController() + let sources: ShadowSources | undefined, worker: ReturnType | undefined + let failure: Error | undefined, terminalState: 'failed' | 'stopped' = 'failed', accepting = true + let rejectFailure!: (error: Error) => void + const failed = new Promise((_resolve, reject) => { + rejectFailure = reject + }) + void failed.catch(() => {}) + const stop = (reason: string, state: 'failed' | 'stopped' = 'failed') => { + if (failure) return + failure = new Error(reason) + terminalState = state + accepting = false + controller.abort() + rejectFailure(failure) + } + const abort = () => stop('shadow_cancelled', 'stopped') + signal?.addEventListener('abort', abort, { once: true }) + if (signal?.aborted) abort() + let endCollection!: () => void, endDrain!: () => void + const collectionEnded = new Promise((resolve) => { + endCollection = resolve + }) + const drained = new Promise((resolve) => { + endDrain = resolve + }) + const collectionTimer = setTimeout(() => endCollection(), Math.max(0, plan.endMs - Date.now())) + const drainTimer = setTimeout(() => endDrain(), Math.max(0, plan.endMs + joining.maximumLatenessMs - Date.now())) + // Stop before the store's exclusive lease deadline so failure can be fenced. + const deadline = setTimeout(() => stop('shadow_drain_deadline'), Math.max(0, plan.endMs + joining.drainGraceMs - Date.now() - 1)) + let heartbeat: ReturnType | undefined + let resources: ShadowPartialReceipt | undefined + const recordResources = (cause: unknown) => { + try { + if (resources) store.recordPartialReceipt(lease, resources) + } + catch (error) { + const combined = new AggregateError([cause, error], 'Shadow partial resource evidence could not be stored') + if (isShadowCleanupError(cause)) throw new ShadowCleanupError(combined) + throw combined + } + } + let receipt: Awaited> | undefined + try { + const join = new ShadowStreamJoin(store, lease) + const check = () => { + if (failure) throw failure + admit(plan) + store.renew(lease, execution.leaseMs) + } + const advance = () => { + const flushed: unknown = sources?.flush?.() + if (flushed !== undefined) { + void Promise.resolve(flushed).catch(() => {}) + throw new Error('Shadow source flush must complete synchronously') + } + if (failure) throw failure + if (join.advance(Date.now()).stopped) stop('storage_quota', 'stopped') + } + heartbeat = setInterval(() => { + try { + check() + advance() + } + catch { stop('shadow_admission_or_lease_lost') } + }, execution.heartbeatMs) + sources = host.openSources({ execution: structuredClone(execution), signal: controller.signal, collectionEndsAtMs: plan.endMs, + fail: () => { + if (accepting) stop('shadow_source_failed') + }, + active: (reading) => { + if (!accepting) return + try { + check() + const active = shadowActiveReadingSchema.parse(reading) + if (active.timestampMs > Date.now()) throw new Error('shadow_future_active_reading') + join.active(active) + } + catch { stop('shadow_active_stream_failed') } + } }) + const source = sources.frames[Symbol.asyncIterator]() + async function* frames() { + let sequence = 0 + try { + while (Date.now() < plan.endMs) { + const next = await Promise.race([Promise.resolve().then(() => source.next()), collectionEnded.then(() => ({ done: true as const, value: undefined })), failed]) + if (next.done) { + if (Date.now() < plan.endMs) throw new Error('shadow_source_ended_early') + break + } + if (Date.now() >= plan.endMs) break + check() + const frame = z.object({ type: z.literal('frame'), observedAtMs: z.number().int().min(plan.startMs).max(plan.endMs - 1) }).passthrough().parse(next.value) + if (frame.observedAtMs > Date.now()) throw new Error('shadow_future_arrival') + yield { ...frame, sequence: sequence++ } + } + } + finally { + if (source.return) void Promise.resolve().then(() => source.return?.()).catch(() => {}) + } + } + worker = runShadowChild(host.location, { schema: 'biometric-shadow-worker-v1', ...lease, runId, + configs, configHashes, side: plan.side, startMs: plan.startMs, endMs: plan.endMs, + maximumRecords: execution.maximumRecords, maximumOutputs: execution.maximumOutputs, maximumCpuMs: execution.maximumCpuMs, + maximumInputBytes: execution.maximumInputBytes, maximumOutputBytes: execution.maximumOutputBytes }, frames(), (output) => { + check() + join.candidate(output) + }, { budgets: structuredClone(execution.budgets), signal: controller.signal, onResources: (value) => { resources = value } }) + receipt = await Promise.race([worker, failed]) + await Promise.race([drained, failed]) + check() + advance() + if (failure) throw failure + accepting = false + } + catch (error) { + stop(error instanceof Error ? error.message.slice(0, 500) : 'shadow_attempt_failed') + } + finally { + accepting = false + controller.abort() + clearInterval(heartbeat) + clearTimeout(collectionTimer) + clearTimeout(drainTimer) + clearTimeout(deadline) + signal?.removeEventListener('abort', abort) + try { + const result: unknown = sources?.close() + if (result !== undefined) { + void Promise.resolve(result).catch(() => {}) + stop('shadow_source_cleanup_must_be_synchronous') + } + } + catch { stop('shadow_source_cleanup_failed') } + // The subprocess supervisor bounds its own cleanup, including on abortion. + if (worker) await worker.catch((error: unknown) => { + if (isShadowCleanupError(error)) { + failure = error + terminalState = 'failed' + } + }) + } + if (!failure) { + try { + admit(plan) + } + catch { stop('shadow_admission_changed_before_completion') } + } + if (failure) { + try { + store.finish(lease, terminalState, failure.message) + } + catch { + // The supervisor has settled; cleanup may be unconfirmed. An expired attempt can be marked failed, + // but this compare-and-set cannot touch a replacement or terminal run. + try { + store.failExpired(lease, failure.message) + } + catch { /* Storage failure remains observable through the rejected run. */ } + } + recordResources(failure) + throw failure + } + try { + if (!receipt) throw new Error('Shadow worker receipt missing') + store.finish(lease, 'completed', 'observation_window_complete', receipt) + } + catch (error) { + // A synchronous final read/cleanup can outlast the remaining lease even + // when its admission check succeeds. Cleanup is complete; fence the failure. + try { + try { + store.finish(lease, 'failed', 'shadow_completion_failed') + } + catch { store.failExpired(lease, 'shadow_completion_lease_lost') } + } + catch { /* Preserve the original completion error. */ } + recordResources(error) + throw error + } + return { lease, receipt, run: store.get(runId) } +} diff --git a/src/biometrics/shadow/build.ts b/src/biometrics/shadow/build.ts new file mode 100644 index 000000000..351a86bfc --- /dev/null +++ b/src/biometrics/shadow/build.ts @@ -0,0 +1,36 @@ +import { execFile } from 'node:child_process' +import { tmpdir } from 'node:os' +import { promisify } from 'node:util' +import { z } from 'zod' +import { workerBuildHash } from '../backtest/child' +import { configHash } from '../algorithms/canonical' +import { waveformPythonBuildSchema } from '../backtest/waveform-build' + +const dependencyPaths = z.object({ record: z.string().min(1).max(4096), wheel: z.string().min(1).max(4096) }).strict() +export const shadowPythonBuildSchema = waveformPythonBuildSchema.extend({ + dependencyFiles: z.object({ numpy: dependencyPaths, scipy: dependencyPaths }).strict(), identity: waveformPythonBuildSchema.shape.identity.extend({ + version: z.literal('shadow-python-build-v1'), + }).refine(identity => ['shadow_build.py', 'shadow.py', 'shadow_worker.py', 'shadow_process.py'].every(name => + identity.sources[`common/biometrics/${name}`] !== undefined), 'Shadow source identity incomplete') }) + +/** Fixed shipped probe, exact worker interpreter, bounded output/deadline. Never + * accepts a module name, shell command or source code from a request. */ +export async function probeShadowBuild(location: { pythonPath: string, modulesPath: string }, signal?: AbortSignal) { + const { stdout } = await promisify(execFile)(location.pythonPath, ['-B', '-m', 'common.biometrics.shadow_build'], { + cwd: tmpdir(), env: { NODE_ENV: 'production', PYTHONPATH: location.modulesPath, LANG: 'C.UTF-8', TZ: 'UTC', + OPENBLAS_NUM_THREADS: '1', OMP_NUM_THREADS: '1', MKL_NUM_THREADS: '1', NUMEXPR_NUM_THREADS: '1' }, + encoding: 'utf8', maxBuffer: 65536, timeout: 10000, killSignal: 'SIGKILL', signal, + }).catch(() => { throw new Error('Shadow build identity is unavailable') }) + const result = shadowPythonBuildSchema.parse(JSON.parse(stdout)) + if (configHash(result.identity) !== result.hash) throw new Error('Shadow build identity integrity mismatch') + return result +} + +/** Bind the complete packaged host graph and prospective Python fingerprint. + * The service must execute this bundle, not hash an unrelated unused artifact. */ +export async function shadowEngineBuildIdentity(hostPath: string, location: { pythonPath: string, modulesPath: string }, signal?: AbortSignal) { + const [hostHash, python] = await Promise.all([workerBuildHash(hostPath), probeShadowBuild(location, signal)]) + const identity = { version: 'shadow-engine-build-v1', hostHash, pythonHash: python.hash, + node: { version: process.version, platform: process.platform, arch: process.arch, icu: process.versions.icu ?? null } } + return { hash: configHash(identity), identity, python } +} diff --git a/src/biometrics/shadow/cleanup-error.ts b/src/biometrics/shadow/cleanup-error.ts new file mode 100644 index 000000000..7031654e3 --- /dev/null +++ b/src/biometrics/shadow/cleanup-error.ts @@ -0,0 +1,11 @@ +/** Structural code survives the separately bundled installed host boundary. */ +export class ShadowCleanupError extends Error { + readonly code = 'SHADOW_CLEANUP_UNCONFIRMED' + constructor(cause?: unknown) { + super('shadow_worker_cleanup_unconfirmed', { cause }) + this.name = 'ShadowCleanupError' + } +} +export function isShadowCleanupError(error: unknown): error is Error & { code: 'SHADOW_CLEANUP_UNCONFIRMED' } { + return error instanceof Error && 'code' in error && error.code === 'SHADOW_CLEANUP_UNCONFIRMED' +} diff --git a/src/biometrics/shadow/context.ts b/src/biometrics/shadow/context.ts new file mode 100644 index 000000000..30b04658c --- /dev/null +++ b/src/biometrics/shadow/context.ts @@ -0,0 +1,96 @@ +import { shadowContextPolicySchema as policySchema } from './execution-policy' +import type Database from 'better-sqlite3' +import { constants, openSync, closeSync, fstatSync, readSync } from 'node:fs' +import { z } from 'zod' +import { activeSleeperSides } from '../../lib/singleSleeper' + +const sideSchema = z.object({ side: z.enum(['left', 'right']), away_mode: z.union([z.literal(0), z.literal(1)]) }) +const bedSchema = z.object({ bed_mode: z.enum(['two', 'solo-left', 'solo-right']) }) +const object = (value: unknown): Record | null => value !== null && typeof value === 'object' && !Array.isArray(value) ? value as Record : null +function presence(path: string, at: number, policy: z.infer) { + let fd: number | undefined + try { + fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK) + const before = fstatSync(fd) + if (!before.isFile() || before.size > policy.maximumPresenceBytes || at < before.mtimeMs || at - before.mtimeMs > policy.presenceStaleMs) return null + const bytes = Buffer.alloc(policy.maximumPresenceBytes + 1) + let length = 0 + while (length < bytes.length) { + const count = readSync(fd, bytes, length, bytes.length - length, null) + if (!count) break + length += count + } + const after = fstatSync(fd) + if (length > policy.maximumPresenceBytes || after.size !== before.size || after.mtimeMs !== before.mtimeMs) return null + const state = object(JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes.subarray(0, length)))) + return state?.version === 1 ? state : null + } + catch { return null } + finally { if (fd !== undefined) closeSync(fd) } +} +function occupied(state: Record | null, side: string, at: number, stale: number) { + const row = object(state?.[side]) + if (typeof row?.vitals_presence !== 'boolean' || typeof row.vitals_evidence_ts !== 'number') return null + const age = at - row.vitals_evidence_ts * 1000 + return Number.isFinite(age) && age >= 0 && age <= stale ? row.vitals_presence : null +} +function pumpEvidence(raw: unknown) { + const side = object(raw), nested = object(side?.pump) + if (!side) return false + const values = [...['pumpRpm', 'pump_rpm', 'pumpRPM', 'rpm', 'pumpDuty', 'pump_duty', 'duty'].map(key => side[key]), + ...['rpm', 'duty', 'power'].map(key => nested?.[key])] + return values.some(value => (typeof value === 'number' && Number.isFinite(value)) + || (typeof value === 'string' && value.trim() !== '' && value.length <= 32 && Number.isFinite(Number(value)))) +} + +/** Current committed evidence for this shadow attempt, not an attestation of + * the active Python process's independently cached view. Host qualification + * must account for that timing difference. Missing context stays unknown. */ +export class ShadowContextReader { + private readonly policy: z.infer + private pumpAt: { left: number | null, right: number | null } = { left: null, right: null } + private lastArrival = -1 + constructor(private readonly db: Database.Database, private readonly presencePath: string, + policy: unknown, private readonly assertDatabaseCurrent: () => void) { + this.policy = policySchema.parse(policy) + if (!db.readonly && db.pragma('query_only', { simple: true }) !== 1) throw new Error('Shadow context requires read-only configuration') + } + + private checkIdentity() { + const checked: unknown = this.assertDatabaseCurrent() + if (checked !== undefined) { + void Promise.resolve(checked).catch(() => {}) + throw new Error('Shadow context identity check must be synchronous') + } + } + + snapshot(frame: Record, observedAtMs: number) { + if (!Number.isSafeInteger(observedAtMs) || observedAtMs < 0 || observedAtMs < this.lastArrival) throw new Error('Shadow context arrival clock regressed') + this.lastArrival = observedAtMs + // Replacement is an attempt failure, not merely missing settings. + this.checkIdentity() + let activeSides: ('left' | 'right')[] | null = null + try { + activeSides = this.db.transaction(() => { + const sides = z.array(sideSchema).length(2).parse(this.db.prepare('SELECT side, away_mode FROM side_settings LIMIT 3').all()) + if (new Set(sides.map(row => row.side)).size !== 2) throw new Error('Ambiguous side settings') + const bed = bedSchema.parse(this.db.prepare('SELECT bed_mode FROM device_settings WHERE id = 1').get()) + return activeSleeperSides({ bedMode: bed.bed_mode, + left: { awayMode: sides.find(row => row.side === 'left')?.away_mode }, + right: { awayMode: sides.find(row => row.side === 'right')?.away_mode } }) + })() + } + catch { /* Missing/corrupt settings are unknown, never an invented two-sleeper default. */ } + const state = presence(this.presencePath, observedAtMs, this.policy) + this.checkIdentity() + if (frame.type === 'frzHealth') { + for (const side of ['left', 'right'] as const) { + // An incomplete update invalidates prior knowledge for that side. + this.pumpAt[side] = pumpEvidence(frame[side]) ? observedAtMs : null + } + } + return { activeSides, leftOccupied: occupied(state, 'left', observedAtMs, this.policy.presenceStaleMs), + rightOccupied: occupied(state, 'right', observedAtMs, this.policy.presenceStaleMs), + pumpContextKnown: Object.values(this.pumpAt).every(at => at !== null && observedAtMs - at <= this.policy.pumpStaleMs) } + } +} diff --git a/src/biometrics/shadow/descriptors.ts b/src/biometrics/shadow/descriptors.ts new file mode 100644 index 000000000..61bea5319 --- /dev/null +++ b/src/biometrics/shadow/descriptors.ts @@ -0,0 +1,25 @@ +import { z } from 'zod' +import type { ShadowStore } from './store' + +const integer = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) +const hash = z.string().regex(/^[a-f0-9]{64}$/) +export const shadowRunDescriptorSchema = z.object({ + id: z.string().uuid(), side: z.enum(['left', 'right']), subjectId: z.string(), + state: z.enum(['pending', 'running', 'completed', 'stopped', 'failed']), + planHash: hash, candidateConfigHash: hash, activeConfigHash: hash, + candidateRevisionId: z.string().uuid().nullable(), candidateIdentitySaved: z.boolean(), + startMs: integer, endMs: integer, createdAtMs: integer, endedAtMs: integer.nullable(), + heartbeatAtMs: integer.nullable(), reason: z.string().nullable(), pointCount: integer, byteCount: integer, + workerReceiptAvailable: z.boolean(), +}).strict() +export const shadowRunPageSchema = z.object({ items: z.array(shadowRunDescriptorSchema).max(50), nextCursor: z.string().uuid().nullable() }).strict() + +/** Public metadata excludes storage paths, waveforms and worker transport data. */ +export function describeShadowRun(run: ReturnType) { + return shadowRunDescriptorSchema.parse({ id: run.id, side: run.side, subjectId: run.plan.subjectId, state: run.state, + planHash: run.plan_hash, candidateConfigHash: run.plan.candidateConfigHash, activeConfigHash: run.plan.activeConfigHash, + candidateRevisionId: run.plan.candidateRevisionId ?? null, candidateIdentitySaved: run.plan.candidateRevisionId !== undefined, + startMs: run.plan.startMs, endMs: run.plan.endMs, createdAtMs: run.created_at_ms, endedAtMs: run.ended_at_ms, + heartbeatAtMs: run.heartbeat_at_ms, reason: run.reason, pointCount: run.point_count, byteCount: run.byte_count, + workerReceiptAvailable: run.workerReceipt !== null }) +} diff --git a/src/biometrics/shadow/device-identity.ts b/src/biometrics/shadow/device-identity.ts new file mode 100644 index 000000000..6c0b1338b --- /dev/null +++ b/src/biometrics/shadow/device-identity.ts @@ -0,0 +1,35 @@ +import { z } from 'zod' +import { getDacMonitorIfRunning } from '@/src/hardware/dacMonitor.instance' +import { shadowQualificationSchema } from './admission' +import type { inspectInstalledShadowHost } from './installed-host' + +/** Observe the existing successful status poll only; never start polling or send + * a DAC command. Preserve exact hardware revision without retaining serials. + * The general UI parser's unknown-label fallback is not qualification evidence. */ +export function observedShadowHardware(maximumAgeMs: number, now = Date.now()) { + z.number().int().positive().max(300000).parse(maximumAgeMs) + z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER).parse(now) + const monitor = getDacMonitorIfRunning() + if (!monitor || monitor.getStatus() !== 'running') throw new Error('Shadow hardware observation unavailable') + const observedAtMs = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER).parse(monitor.getLastPollAt()) + if (observedAtMs > now || now - observedAtMs > maximumAgeMs) throw new Error('Shadow hardware observation stale or future') + const status = z.object({ podVersion: z.enum(['H00', 'I00', 'J00']), sensorLabel: z.string().min(1).max(1000) }).parse(monitor.getLastStatus()) + const revision = status.sensorLabel.split('-').reverse().find(segment => /^[A-Z]\d{2}$/.test(segment)) + if (!revision || !/^[HIJ]\d{2}$/.test(revision) || status.podVersion !== `${revision[0]}00`) throw new Error('Shadow sensor hardware revision unavailable or inconsistent') + return { hardwareId: `pod-sensor-v1/${status.podVersion}/${revision}`, sampleRateHz: 500 as const } +} + +/** Called by the application, outside the passive packaged host. Every admission + * observes current hardware and rechecks the loaded Node/Python artifacts. */ +export function installedShadowIdentity(inspection: Awaited>, rawQualification: unknown) { + const qualification = shadowQualificationSchema.parse(rawQualification) + const engineBuildHash = inspection.build.hash, pythonBuildHash = inspection.build.python.hash + const assertCurrent = inspection.assertCurrent + if (engineBuildHash !== qualification.engineBuildHash || pythonBuildHash !== qualification.pythonBuildHash) throw new Error('Shadow installed build differs from qualification') + return () => { + assertCurrent() + const hardware = observedShadowHardware(qualification.maximumHardwareObservationAgeMs) + if (hardware.hardwareId !== qualification.hardwareId || hardware.sampleRateHz !== qualification.sampleRateHz) throw new Error('Shadow hardware differs from qualification') + return { engineBuildHash, pythonBuildHash, ...hardware } + } +} diff --git a/src/biometrics/shadow/execution-policy.ts b/src/biometrics/shadow/execution-policy.ts new file mode 100644 index 000000000..24e43e763 --- /dev/null +++ b/src/biometrics/shadow/execution-policy.ts @@ -0,0 +1,23 @@ +import { z } from 'zod' +const positive = z.number().int().positive().max(Number.MAX_SAFE_INTEGER) +const hash = z.string().regex(/^[a-f0-9]{64}$/) +export const shadowChildBudgetsSchema = z.object({ cpuSeconds: positive.max(3600), addressSpaceBytes: positive, + maximumRssBytes: positive, wallMs: positive.max(7 * 86400000 + 300000), + acknowledgmentMs: positive.max(300000), idleMs: positive.max(300000), + rssIntervalMs: positive.min(50).max(1000), rssTimeoutMs: positive.max(10000), +}).strict() +export const shadowFrameBudgetsSchema = z.object({ maximumFrames: positive.max(1000), maximumBytes: positive.max(32 * 1024 * 1024), + maximumEventAgeMs: positive.max(120000), maximumFutureSkewMs: z.number().int().min(0).max(5000) }).strict() +export const shadowContextPolicySchema = z.object({ presenceStaleMs: positive.max(300000), + pumpStaleMs: positive.max(300000), maximumPresenceBytes: positive.max(256 * 1024) }).strict() + +/** Explicit ceilings, never production defaults. The reviewed qualification + * must match the complete policy, including both physical channel configs. */ +export const shadowExecutionPolicySchema = z.object({ version: z.literal('shadow-execution-policy-v1'), + configHashes: z.object({ left: hash, right: hash }).strict(), + maximumRecords: positive, maximumOutputs: positive, maximumCpuMs: positive, maximumInputBytes: positive, maximumOutputBytes: positive, + budgets: shadowChildBudgetsSchema, leaseMs: positive.min(1000).max(3600000), heartbeatMs: positive.max(1000), + sources: z.object({ pollMs: positive.min(10).max(1000), maximumActiveRows: positive.max(100), + databaseBusyTimeoutMs: positive.max(1000), frames: shadowFrameBudgetsSchema, context: shadowContextPolicySchema }).strict(), +}).strict().refine(policy => policy.heartbeatMs * 3 <= policy.leaseMs, 'Heartbeat must fit the lease') +export type ShadowExecutionPolicy = z.infer diff --git a/src/biometrics/shadow/frame-source.ts b/src/biometrics/shadow/frame-source.ts new file mode 100644 index 000000000..c33a65c07 --- /dev/null +++ b/src/biometrics/shadow/frame-source.ts @@ -0,0 +1,149 @@ +import { shadowFrameBudgetsSchema as budgetsSchema } from './execution-policy' +import { z } from 'zod' +import { subscribeProspectiveFrames } from '../../streaming/prospectiveFrameTap' + +const contextSchema = z.object({ activeSides: z.array(z.enum(['left', 'right'])).max(2).nullable(), + leftOccupied: z.boolean().nullable(), rightOccupied: z.boolean().nullable(), pumpContextKnown: z.boolean(), +}).strict().refine(value => value.activeSides === null || new Set(value.activeSides).size === value.activeSides.length, 'Duplicate active side') +const pumpKeys = ['pumpRpm', 'pump_rpm', 'pumpRPM', 'rpm', 'pumpDuty', 'pump_duty', 'duty'] as const +function pumpValue(value: unknown) { + if (value === null) return null + if (typeof value === 'number' && Number.isFinite(value)) return value + if (typeof value === 'string' && value.length <= 32 && value.trim() !== '' && Number.isFinite(Number(value))) return value + throw new Error('Invalid prospective pump context') +} +function pumpSide(raw: unknown) { + if (raw === undefined) return undefined + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) throw new Error('Invalid prospective pump side') + const source = raw as Record, result: Record = {} + for (const key of pumpKeys) if (source[key] !== undefined) result[key] = pumpValue(source[key]) + if (source.pump !== undefined) { + if (!source.pump || typeof source.pump !== 'object' || Array.isArray(source.pump)) throw new Error('Invalid prospective nested pump') + const pump = source.pump as Record, selected: Record = {} + for (const key of ['rpm', 'duty', 'power']) if (pump[key] !== undefined) selected[key] = pumpValue(pump[key]) + result.pump = selected + } + return result +} +function channel(raw: unknown) { + if (!Array.isArray(raw) || raw.length > 5000) throw new Error('Invalid prospective waveform channel') + const bytes = Buffer.alloc(raw.length * 4) + for (let index = 0; index < raw.length; index++) { + const value = raw[index] + if (!Number.isInteger(value) || value < -2147483648 || value > 2147483647) throw new Error('Invalid prospective waveform sample') + bytes.writeInt32LE(value, index * 4) + } + return bytes.toString('base64') +} +function record(frame: Record) { + const result: Record = { type: frame.type } + if (frame.ts !== undefined) result.ts = z.number().finite().nonnegative().parse(frame.ts) + if (frame.type === 'piezo-dual') { + if (frame.freq !== undefined) result.freq = z.literal(500).parse(frame.freq) + result.left1 = channel(frame.left1) + result.right1 = channel(frame.right1) + } + else { + const left = pumpSide(frame.left), right = pumpSide(frame.right) + if (left !== undefined) result.left = left + if (right !== undefined) result.right = right + } + return result +} + +/** Subscribes to actual live decoded frames. Overflow fails the attempt rather + * than silently dropping evidence or allowing the worker to lag without bound. + * Returned JSON is a private snapshot; raw stream objects are never retained. */ +export function openShadowFrameSource(options: { + startMs: number + endMs: number + signal: AbortSignal + budgets: z.infer + context: (frame: Record, observedAtMs: number) => unknown + fail: () => void +}) { + const budgets = budgetsSchema.parse(options.budgets) + if (!Number.isSafeInteger(options.startMs) || !Number.isSafeInteger(options.endMs) || options.startMs < 0 || options.endMs <= options.startMs || options.endMs - options.startMs > 7 * 86400000 || options.endMs - Date.now() > 7 * 86400000) throw new Error('Invalid prospective source window') + let closed = false, used = false, bytes = 0, previousArrival = options.startMs + const queue: { text: string, bytes: number }[] = [] + let waiter: ((value: IteratorResult) => void) | undefined + let unsubscribe: (() => void) | undefined + const close = () => { + if (closed) return + closed = true + unsubscribe?.() + options.signal.removeEventListener('abort', close) + clearTimeout(endTimer) + queue.length = 0 + bytes = 0 + waiter?.({ done: true, value: undefined }) + waiter = undefined + } + const notifyFailure = () => { + try { + options.fail() + } + catch { /* Isolate consumer errors from the live server's timers. */ } + } + const finishCollection = () => { + const lost = queue.length > 0 + close() + if (lost) notifyFailure() + } + const endTimer = setTimeout(finishCollection, Math.max(0, options.endMs - Date.now())) + if (options.signal.aborted) close() + else { + options.signal.addEventListener('abort', close, { once: true }) + try { + unsubscribe = subscribeProspectiveFrames({ failed: () => { + close() + notifyFailure() + }, frame: (frame, observedAtMs) => { + if (closed) return + if (!Number.isSafeInteger(observedAtMs) || observedAtMs > Date.now()) throw new Error('Invalid prospective arrival clock') + if (observedAtMs < options.startMs || observedAtMs >= options.endMs) return + if (observedAtMs < previousArrival) throw new Error('Prospective arrival clock regressed') + const eventMs = z.number().finite().nonnegative().parse(frame.ts) * 1000 + if (!Number.isFinite(eventMs) || eventMs > Number.MAX_SAFE_INTEGER || observedAtMs - eventMs > budgets.maximumEventAgeMs || eventMs - observedAtMs > budgets.maximumFutureSkewMs) throw new Error('Prospective event time outside freshness policy') + if (queue.length >= budgets.maximumFrames) throw new Error('Prospective frame queue full') + const value = { type: 'frame', observedAtMs, record: record(frame), context: contextSchema.parse(options.context(frame, observedAtMs)) } + const text = JSON.stringify(value), size = Buffer.byteLength(text) + if (size > 128 * 1024 || bytes + size > budgets.maximumBytes) throw new Error('Prospective frame byte quota exceeded') + previousArrival = observedAtMs + if (waiter) { + const resolve = waiter + waiter = undefined + resolve({ done: false, value: JSON.parse(text) }) + } + else { + queue.push({ text, bytes: size }) + bytes += size + } + } }) + } + catch (error) { + close() + throw error + } + } + return { close, frames: { [Symbol.asyncIterator]() { + if (used) throw new Error('Prospective frame source has one consumer') + used = true + return { next: (): Promise> => { + if (closed) return Promise.resolve({ done: true, value: undefined }) + if (waiter) return Promise.reject(new Error('Concurrent prospective frame reads')) + const row = queue.shift() + if (row) { + bytes -= row.bytes + return Promise.resolve({ done: false, value: JSON.parse(row.text) }) + } + return new Promise((resolve) => { + waiter = resolve + }) + }, return: async (): Promise> => { + if (Date.now() >= options.endMs) finishCollection() + else close() + return { done: true, value: undefined } + } } + } } } +} diff --git a/src/biometrics/shadow/health-monitor.ts b/src/biometrics/shadow/health-monitor.ts new file mode 100644 index 000000000..1595779f9 --- /dev/null +++ b/src/biometrics/shadow/health-monitor.ts @@ -0,0 +1,64 @@ +import { admissionPolicySchema, collectAdmissionSample, evaluateAdmission } from '../backtest/admission' +import type { AdmissionSample } from '../backtest/admission' + +/** One collection in flight, with a deadline derived from the installed policy. + * Timeout is terminal: uncancellable filesystem I/O must not accumulate behind + * retries. The attempt observes null on its next synchronous admission check. */ +export async function startShadowHealthMonitor(collect: () => Promise, rawPolicy: unknown, signal?: AbortSignal) { + const policy = admissionPolicySchema.parse(rawPolicy) + const cadenceMs = Math.min(1000, Math.floor(policy.maximumSampleAgeMs / 2)) + if (cadenceMs < 1) throw new Error('Shadow health freshness policy is too short') + let closed = false, sample: AdmissionSample | null = null + let refreshTimer: ReturnType | undefined, deadline: ReturnType | undefined + let rejectPending: ((error: Error) => void) | undefined + const close = () => { + closed = true + sample = null + clearTimeout(refreshTimer) + clearTimeout(deadline) + signal?.removeEventListener('abort', close) + rejectPending?.(new Error('Shadow health monitor closed')) + } + const refresh = async () => { + if (closed) throw new Error('Shadow health monitor closed') + try { + const collected = await Promise.race([Promise.resolve().then(() => { + if (closed) throw new Error('Shadow health monitor closed') + return collect() + }), new Promise((_resolve, reject) => { + rejectPending = reject + deadline = setTimeout(() => reject(new Error('Shadow health collection deadline exceeded')), cadenceMs) + })]) + if (closed) throw new Error('Shadow health monitor closed') + sample = collected ? structuredClone(collected) : null + } + catch (error) { + close() + throw error + } + finally { + clearTimeout(deadline) + rejectPending = undefined + } + } + const schedule = () => { + if (closed) return + refreshTimer = setTimeout(() => { + void refresh().then(schedule).catch(() => {}) + }, cadenceMs) + } + signal?.addEventListener('abort', close, { once: true }) + if (signal?.aborted) close() + await refresh() + schedule() + return { + read: () => !closed && evaluateAdmission(sample, policy).admitted ? structuredClone(sample) : null, + close, + } +} + +/** Production collector uses host filesystem health and the actual live piezo + * receive clock. Paths and qualification policy are supplied by the host only. */ +export async function openShadowHealthMonitor(databaseDirectory: string, policy: unknown, piezoReceivedAt: () => number | undefined, signal?: AbortSignal) { + return startShadowHealthMonitor(() => collectAdmissionSample(databaseDirectory, piezoReceivedAt), policy, signal) +} diff --git a/src/biometrics/shadow/host-entry.ts b/src/biometrics/shadow/host-entry.ts new file mode 100644 index 000000000..7094f4d51 --- /dev/null +++ b/src/biometrics/shadow/host-entry.ts @@ -0,0 +1,8 @@ +// Shipped, fingerprinted host API. Importing this module opens no database, +// starts no service, subscribes to no source and performs no hardware command. +export { runProfiledShadowAttempt, openProfiledShadowHost } from './host' +export { prepareShadowPlan } from './preparation' +export { ShadowStore, shadowPlanSchema, shadowStoragePolicySchema } from './store' +export { shadowEngineBuildIdentity } from './build' +export { readInstalledShadowQualifications } from './admission' +export const shadowHostApiVersion = 'shadow-host-api-v1' as const diff --git a/src/biometrics/shadow/host.ts b/src/biometrics/shadow/host.ts new file mode 100644 index 000000000..e91b6f5e1 --- /dev/null +++ b/src/biometrics/shadow/host.ts @@ -0,0 +1,139 @@ +import { resolveShadowProfiles } from './profiles' +import { dirname } from 'node:path' +import { dataHash } from '../algorithms/canonical' +import { shadowPlanSchema, type ShadowStore } from './store' +import { installedShadowAdmitter, readInstalledShadowQualifications } from './admission' +import { openShadowActiveDatabase } from './active-database' +import { openShadowHealthMonitor } from './health-monitor' +import { shadowRuntimeBindingReader } from './runtime-binding' +import { plannedShadowSourceFactory } from './planned-sources' +import { runShadowAttempt, type ShadowAttemptHost } from './attempt' + +export interface ShadowHostInstallation { + workerPath: string + pythonPath: string + modulesPath: string + configurationDatabase: string + biometricsDatabase: string + presenceFile: string + configs: { left: unknown, right: unknown } + expectedBindings: Record<'left' | 'right', { generation: number, resolvedHash: string }> + /** Trusted installed-build/hardware collector; must detect changes, never RPC data. */ + identity: () => unknown + piezoReceivedAt: () => number | undefined +} + +/** Own admission connections and health sampling for exactly one saved plan. + * Qualification lookup precedes I/O sampling; no source subscription or worker + * launch occurs until the attempt controller has claimed its fenced lease. */ +export async function openPlannedShadowHost(rawPlan: unknown, installation: ShadowHostInstallation, signal?: AbortSignal) { + const plan = shadowPlanSchema.parse(rawPlan), planHash = dataHash(plan) + if (!plan.execution || !plan.joining) throw new Error('Shadow host requires a saved execution policy') + const { identity, piezoReceivedAt, ...rawLocations } = installation + const locations = structuredClone(rawLocations) + const controller = new AbortController() + let closed = false + let config: ReturnType | undefined, bio: ReturnType | undefined + let health: Awaited> | undefined + const close = () => { + if (closed) return + closed = true + signal?.removeEventListener('abort', close) + controller.abort() + try { + health?.close() + } + finally { + try { + config?.close() + } + finally { bio?.close() } + } + } + signal?.addEventListener('abort', close, { once: true }) + if (signal?.aborted) close() + try { + if (closed) throw new Error('Shadow host cancelled before opening') + const entries = readInstalledShadowQualifications(locations.workerPath).entries.filter(entry => dataHash(entry) === plan.qualificationHash) + if (entries.length !== 1) throw new Error('Shadow qualification missing or ambiguous') + const databaseOptions = { busyTimeoutMs: plan.execution.sources.databaseBusyTimeoutMs } + config = openShadowActiveDatabase(locations.configurationDatabase, databaseOptions) + bio = openShadowActiveDatabase(locations.biometricsDatabase, databaseOptions) + const configuration = config, biometrics = bio + const binding = shadowRuntimeBindingReader(configuration.db, biometrics.db, { side: plan.side, expected: locations.expectedBindings, + assertCurrent: () => { + configuration.assertCurrent() + biometrics.assertCurrent() + } }) + health = await openShadowHealthMonitor(dirname(locations.biometricsDatabase), entries[0].healthPolicy, piezoReceivedAt, controller.signal) + if (closed) { + health.close() + throw new Error('Shadow host cancelled while opening') + } + const admission = installedShadowAdmitter({ workerPath: locations.workerPath, + expectedBinding: { side: plan.side, ...locations.expectedBindings[plan.side] }, identity, binding, health: health.read }) + const host: ShadowAttemptHost = { + location: { pythonPath: locations.pythonPath, modulesPath: locations.modulesPath }, configs: locations.configs, + admit: (candidate) => { + if (closed || dataHash(candidate) !== planHash) throw new Error('Shadow host closed or saved plan changed') + admission(candidate) + }, + openSources: plannedShadowSourceFactory(plan, locations), + } + host.admit(plan) + return { host, signal: controller.signal, close } + } + catch (error) { + close() + throw error + } +} + +/** Own cleanup even when claim fails, the worker fails, or a caller cancels. */ +export async function runPlannedShadowAttempt(store: ShadowStore, runId: string, installation: ShadowHostInstallation, signal?: AbortSignal) { + const owned = await openPlannedShadowHost(store.get(runId).plan, installation, signal) + try { + return await runShadowAttempt(store, runId, owned.host, owned.signal) + } + finally { owned.close() } +} + +/** Public service integration supplies a saved revision ID, never resolved + * numerical configs or asserted active generations. Admission rechecks the + * captured generations after these snapshot handles have been closed. */ +export async function runProfiledShadowAttempt(store: ShadowStore, runId: string, + installation: Omit, revisionId: string | null, signal?: AbortSignal) { + const owned = await openProfiledShadowHost(store.get(runId).plan, installation, revisionId, signal) + try { + return await runShadowAttempt(store, runId, owned.host, owned.signal) + } + finally { owned.close() } +} + +/** Admission-only preparation for a proposed plan. Does not launch a worker, + * claim a lease or open sensor sources. Caller owns the returned resources. */ +export async function openProfiledShadowHost(rawPlan: unknown, + installation: Omit, revisionId: string | null, signal?: AbortSignal) { + const plan = shadowPlanSchema.parse(rawPlan) + if (!plan.execution) throw new Error('Shadow profile resolution requires execution policy') + if (signal?.aborted) throw new Error('Shadow attempt cancelled before profile resolution') + const options = { busyTimeoutMs: plan.execution.sources.databaseBusyTimeoutMs } + const configuration = openShadowActiveDatabase(installation.configurationDatabase, options) + let biometrics: ReturnType | undefined + let resolved: ReturnType + try { + biometrics = openShadowActiveDatabase(installation.biometricsDatabase, options) + const bio = biometrics + resolved = resolveShadowProfiles(configuration.db, bio.db, plan, revisionId, () => { + configuration.assertCurrent() + bio.assertCurrent() + }) + } + finally { + try { + configuration.close() + } + finally { biometrics?.close() } + } + return openPlannedShadowHost(plan, { ...installation, configs: resolved.configs, expectedBindings: resolved.expectedBindings }, signal) +} diff --git a/src/biometrics/shadow/installed-attempt.ts b/src/biometrics/shadow/installed-attempt.ts new file mode 100644 index 000000000..61c4311fd --- /dev/null +++ b/src/biometrics/shadow/installed-attempt.ts @@ -0,0 +1,87 @@ +import { withBiometricBackgroundSlot } from '../background-admission' +import { z } from 'zod' +import { dataHash } from '../algorithms/canonical' +import { getSensorFrameTimes } from '@/src/streaming/piezoStream' +import type { ShadowStore } from './store' +import { inspectInstalledShadowHost, loadInstalledShadowHost } from './installed-host' +import { installedShadowIdentity } from './device-identity' +import { openShadowStateDatabase } from './active-database' +import type { ShadowHostInstallation } from './host' + +type Installation = Pick + +/** Application-side execution boundary. Paths/storage policy are trusted service + * configuration. User operations identify only an existing run and revision. + * Executes the inspected package, never the source-copy attempt controller. */ +async function runAdmittedShadowAttempt(runId: string, revisionId: string | null, installation: Installation, + storagePolicy: unknown, busyTimeoutMs: number, signal?: AbortSignal) { + z.string().uuid().parse(runId) + z.string().uuid().nullable().parse(revisionId) + z.number().int().min(1).max(1000).parse(busyTimeoutMs) + const paths = structuredClone(installation), policy = structuredClone(storagePolicy) + if (signal?.aborted) throw new Error('Installed shadow attempt cancelled') + const inspected = await inspectInstalledShadowHost(paths.workerPath, paths, signal) + if (signal?.aborted) throw new Error('Installed shadow attempt cancelled') + inspected.assertCurrent() + const database = openShadowStateDatabase(paths.biometricsDatabase, { busyTimeoutMs }) + try { + const store = new inspected.api.ShadowStore(database.db, policy) + const plan = store.get(runId).plan + if (!plan.expectedBindings) throw new Error('Shadow active generations missing from saved plan') + if (plan.candidateRevisionId === undefined || plan.candidateRevisionId !== revisionId) throw new Error('Shadow candidate revision differs from saved plan') + if (!plan.execution || busyTimeoutMs > plan.execution.sources.databaseBusyTimeoutMs) throw new Error('Shadow store timeout exceeds saved execution policy') + const entries = inspected.api.readInstalledShadowQualifications(paths.workerPath).entries.filter(entry => dataHash(entry) === plan.qualificationHash) + if (entries.length !== 1) throw new Error('Shadow qualification missing or ambiguous') + const observedIdentity = installedShadowIdentity(inspected, entries[0]) + const identity = () => { + database.assertCurrent() + return observedIdentity() + } + identity() + return await inspected.api.runProfiledShadowAttempt(store, runId, { ...paths, identity, + piezoReceivedAt: () => getSensorFrameTimes()['piezo-dual'] }, revisionId, signal) + } + finally { database.close() } +} + +/** Reserve before build probes, database ownership, source subscription or claim. + * Busy pending runs remain pending for the service's next scheduling pass. */ +export function runInstalledShadowAttempt(...args: Parameters) { + return withBiometricBackgroundSlot('work', () => runAdmittedShadowAttempt(...args)) +} + +/** Queue inspection runs the verified package with an owned state connection. */ +export function pendingInstalledShadowRun(installation: Pick, + storagePolicy: unknown, busyTimeoutMs: number, options: Parameters[0]) { + z.number().int().min(1).max(1000).parse(busyTimeoutMs) + const loaded = loadInstalledShadowHost(installation.workerPath) + loaded.assertCurrent() + const database = openShadowStateDatabase(installation.biometricsDatabase, { busyTimeoutMs }) + try { + database.assertCurrent() + const result = new loaded.api.ShadowStore(database.db, storagePolicy).pending(options) + database.assertCurrent() + loaded.assertCurrent() + return result + } + finally { database.close() } +} + +/** Recovery needs the installed store implementation but never starts a Python + * probe/worker, subscribes to sensors or requires an admission qualification. */ +export function reconcileInstalledShadowRuns(installation: Pick, + storagePolicy: unknown, busyTimeoutMs: number, options: Parameters[0]) { + z.number().int().min(1).max(1000).parse(busyTimeoutMs) + const loaded = loadInstalledShadowHost(installation.workerPath) + loaded.assertCurrent() + const database = openShadowStateDatabase(installation.biometricsDatabase, { busyTimeoutMs }) + try { + database.assertCurrent() + const store = new loaded.api.ShadowStore(database.db, storagePolicy) + const result = store.reconcileExpired(options) + database.assertCurrent() + loaded.assertCurrent() + return result + } + finally { database.close() } +} diff --git a/src/biometrics/shadow/installed-host.ts b/src/biometrics/shadow/installed-host.ts new file mode 100644 index 000000000..e2d86baba --- /dev/null +++ b/src/biometrics/shadow/installed-host.ts @@ -0,0 +1,81 @@ +import { guardShadowPythonInstallation } from './python-installation' +import { createHash } from 'node:crypto' +import { constants, openSync, closeSync, fstatSync, lstatSync, readSync, realpathSync } from 'node:fs' +import { createRequire } from 'node:module' +import { resolve } from 'node:path' +import type * as HostAPI from './host-entry' + +type API = typeof HostAPI +const maximumBytes = 16 * 1024 * 1024 +function signature(path: string) { + const stat = lstatSync(path, { bigint: true }) + if (!stat.isFile() || stat.isSymbolicLink() || stat.size > BigInt(maximumBytes)) throw new Error('Invalid installed shadow host artifact') + return `${stat.dev}:${stat.ino}:${stat.size}:${stat.mtimeNs}:${stat.ctimeNs}` +} +interface InstalledHost { path: string, signature: string, hostHash: string, api: API } +const state = globalThis as typeof globalThis & { __sleepypodInstalledShadowHostV1?: InstalledHost } + +/** Trusted deployment path only. Load one exact release artifact per process; + * changed installations require restart rather than reusing Node's stale cache. + * This guards the Node bundle, not Python binaries or dependency wheel contents. */ +export function loadInstalledShadowHost(hostPath: string) { + const requestedPath = resolve(hostPath) + signature(requestedPath) // reject a leaf symlink before canonicalizing parent paths + const path = realpathSync(requestedPath) + let cached = state.__sleepypodInstalledShadowHostV1 + if (cached) { + if (cached.path !== path || signature(path) !== cached.signature) throw new Error('Installed shadow host changed; restart required') + } + else { + const before = signature(path) + const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK) + let hostHash: string + try { + const opened = fstatSync(fd, { bigint: true }) + if (`${opened.dev}:${opened.ino}:${opened.size}:${opened.mtimeNs}:${opened.ctimeNs}` !== before) throw new Error('Shadow host changed before hashing') + const digest = createHash('sha256'), buffer = Buffer.alloc(65536) + let length = 0 + while (true) { + const count = readSync(fd, buffer, 0, buffer.length, null) + if (!count) break + length += count + if (length > maximumBytes) throw new Error('Shadow host artifact exceeds its bound') + digest.update(buffer.subarray(0, count)) + } + hostHash = digest.digest('hex') + } + finally { closeSync(fd) } + if (signature(path) !== before) throw new Error('Shadow host changed during hashing') + const require = createRequire(path) + if (require.cache[path]) throw new Error('Shadow host was loaded without installed identity verification') + const api = require(path) as Partial + if (signature(path) !== before) throw new Error('Shadow host changed while loading') + if (api.shadowHostApiVersion !== 'shadow-host-api-v1' || ['runProfiledShadowAttempt', 'openProfiledShadowHost', 'prepareShadowPlan', 'ShadowStore', 'shadowEngineBuildIdentity', 'readInstalledShadowQualifications'] + .some(name => typeof api[name as keyof API] !== 'function') || !api.shadowPlanSchema || !api.shadowStoragePolicySchema) throw new Error('Invalid installed shadow host API') + cached = { path, signature: before, hostHash, api: api as API } + state.__sleepypodInstalledShadowHostV1 = cached + } + const installed = cached + return { api: installed.api, hostHash: installed.hostHash, + assertCurrent: () => { + if (signature(path) !== installed.signature || signature(requestedPath) !== installed.signature) throw new Error('Installed shadow host changed; restart required') + } } +} + +/** Probe through the loaded package and prove the fingerprint describes the + * artifact actually loaded, including rechecking it after the async Python probe. */ +export async function inspectInstalledShadowHost(hostPath: string, location: { pythonPath: string, modulesPath: string }, signal?: AbortSignal) { + if (signal?.aborted) throw new Error('Shadow build inspection cancelled') + const target = { ...location } + const loaded = loadInstalledShadowHost(hostPath) + const build = await loaded.api.shadowEngineBuildIdentity(resolve(hostPath), target, signal) + loaded.assertCurrent() + if (build.identity.hostHash !== loaded.hostHash) throw new Error('Loaded shadow host differs from inspected artifact') + const pythonCurrent = guardShadowPythonInstallation(target, build.python) + const assertCurrent = () => { + loaded.assertCurrent() + pythonCurrent() + } + assertCurrent() + return { ...loaded, build, assertCurrent } +} diff --git a/src/biometrics/shadow/installed-preparation.ts b/src/biometrics/shadow/installed-preparation.ts new file mode 100644 index 000000000..8bd49e4e9 --- /dev/null +++ b/src/biometrics/shadow/installed-preparation.ts @@ -0,0 +1,44 @@ +import type { z } from 'zod' +import type { runInstalledShadowAttempt } from './installed-attempt' +import { openShadowActiveDatabase } from './active-database' +import { loadInstalledShadowHost } from './installed-host' +import { prepareShadowRequestSchema, shadowPlanTemplateSchema, shadowTemplatePageSchema } from './preparation-contracts' +import { configuredShadowService } from './startup' + +export function configuredShadowTemplates() { + return shadowTemplatePageSchema.parse({ items: (configuredShadowService()?.planTemplates ?? []).map(({ id, label, maximumWindowMs }) => ({ id, label, maximumWindowMs })) }) +} + +/** Fixed server paths and package implementation; no Python probe or sources. + * A prepared response cannot be submitted without ordinary creation admission. */ +export function prepareInstalledShadow(raw: z.input, + installation: Parameters[2], busyTimeoutMs: number) { + const request = prepareShadowRequestSchema.parse(raw) + const templates = configuredShadowService()?.planTemplates ?? [] + const matches = templates.filter(template => template.id === request.templateId) + if (matches.length !== 1) throw new Error('Shadow template unavailable') + const template = shadowPlanTemplateSchema.parse(matches[0]) + if (busyTimeoutMs > template.execution.sources.databaseBusyTimeoutMs) throw new Error('Shadow store timeout exceeds template policy') + const loaded = loadInstalledShadowHost(installation.workerPath) + loaded.assertCurrent() + const configuration = openShadowActiveDatabase(installation.configurationDatabase, { busyTimeoutMs }) + let biometrics: ReturnType | undefined + try { + biometrics = openShadowActiveDatabase(installation.biometricsDatabase, { busyTimeoutMs }) + const bio = biometrics + const guard = () => { + loaded.assertCurrent() + configuration.assertCurrent() + bio.assertCurrent() + } + const prepared = loaded.api.prepareShadowPlan(configuration.db, bio.db, request, template, installation.workerPath, guard) + guard() + return prepared + } + finally { + try { + configuration.close() + } + finally { biometrics?.close() } + } +} diff --git a/src/biometrics/shadow/live-sources.ts b/src/biometrics/shadow/live-sources.ts new file mode 100644 index 000000000..66c78e60b --- /dev/null +++ b/src/biometrics/shadow/live-sources.ts @@ -0,0 +1,86 @@ +import type Database from 'better-sqlite3' +import { z } from 'zod' +import { ShadowActiveReader } from './active-reader' +import { openShadowFrameSource } from './frame-source' +import type { ShadowSources } from './attempt' + +/** Compose the real read-only active reader with the live waveform tap. The + * owner retains the database connection and must detect path/inode replacement. + * Polling and final-drain flush use the same bounded, synchronous snapshot. */ +export function openShadowLiveSources(options: { + db: Database.Database + side: 'left' | 'right' + startMs: number + endMs: number + activeConfigHash: string + maximumActiveRows: number + maximumLatenessMs: number + pollMs: number + frameBudgets: Parameters[0]['budgets'] + context: Parameters[0]['context'] + assertDatabaseCurrent: () => void + signal: AbortSignal + active: (reading: unknown) => void + fail: () => void +}): ShadowSources { + z.number().int().min(1).max(120000).parse(options.maximumLatenessMs) + z.number().int().min(10).max(Math.min(1000, options.maximumLatenessMs - 1)).parse(options.pollMs) + function synchronous(action: () => void) { + const result: unknown = action() + if (result !== undefined) { + void Promise.resolve(result).catch(() => {}) + throw new Error('Shadow live source callbacks must complete synchronously') + } + } + synchronous(options.assertDatabaseCurrent) + // Establish the active watermark first; never miss a commit while enabling + // candidate input, and never substitute historical active rows. + const reader = new ShadowActiveReader(options.db, { side: options.side, startMs: options.startMs, endMs: options.endMs, + activeConfigHash: options.activeConfigHash, maximumRows: options.maximumActiveRows }) + let closed = false, waveform: ReturnType | undefined + let timer: ReturnType | undefined + const close = () => { + if (closed) return + closed = true + clearInterval(timer) + waveform?.close() + options.signal.removeEventListener('abort', close) + } + const fail = () => { + if (closed) return + close() + try { + options.fail() + } + catch { /* Isolate controller errors from the live server's timer. */ } + } + const flush = () => { + if (closed) throw new Error('Shadow live sources are closed') + synchronous(options.assertDatabaseCurrent) + const readings = reader.poll() + synchronous(options.assertDatabaseCurrent) + for (const reading of readings) { + if (closed || options.signal.aborted) throw new Error('Shadow active delivery cancelled') + synchronous(() => options.active(reading)) + } + } + try { + waveform = openShadowFrameSource({ startMs: options.startMs, endMs: options.endMs, budgets: options.frameBudgets, + signal: options.signal, context: options.context, fail }) + if (options.signal.aborted) close() + else { + options.signal.addEventListener('abort', close, { once: true }) + timer = setInterval(() => { + try { + flush() + } + catch { fail() } + }, options.pollMs) + } + return { frames: waveform.frames, flush, close } + } + catch (error) { + close() + throw error + } +} diff --git a/src/biometrics/shadow/plan-contracts.ts b/src/biometrics/shadow/plan-contracts.ts new file mode 100644 index 000000000..c39c7ef84 --- /dev/null +++ b/src/biometrics/shadow/plan-contracts.ts @@ -0,0 +1,34 @@ +import { z } from 'zod' +import { shadowExecutionPolicySchema } from './execution-policy' + +const integer = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) +const positive = integer.positive() +const hash = z.string().regex(/^[a-f0-9]{64}$/) +const metric = z.enum(['heart_rate', 'breathing_rate', 'window_ibi_variability_v1']) +const binding = z.object({ generation: positive, resolvedHash: hash }).strict() +export const shadowExpectedBindingsSchema = z.object({ left: binding, right: binding }).strict() +export const shadowMonitoringSchema = z.object({ + expectedIntervalMs: positive, maximumGapMs: positive, heartbeatStaleMs: positive, + minimumCandidateCoverage: z.number().positive().max(1), maximumCoverageLoss: z.number().min(0).max(1), minimumQuality: z.number().min(0).max(1), + alignment: z.object({ version: z.literal('reference-alignment-v1'), pointToleranceMs: integer.max(300000), + clockOffsetMs: z.number().int().min(-3600000).max(3600000), offsetSource: z.enum(['fixed', 'training']) }).strict(), +}).strict() +export const shadowPlanSchema = z.object({ schema: z.literal('biometric-shadow-plan-v1'), side: z.enum(['left', 'right']), + execution: shadowExecutionPolicySchema.optional(), + // Absent on legacy evidence; null explicitly selects the shipped default. + candidateRevisionId: z.string().uuid().nullable().optional(), + expectedBindings: shadowExpectedBindingsSchema.optional(), + subjectId: z.string().min(1).max(200), monitoring: shadowMonitoringSchema, + windowIbiWindowMs: positive.max(86400000).optional(), + joining: z.object({ maximumPendingPoints: positive.max(10000), maximumLatenessMs: positive.max(120000), + drainGraceMs: positive.max(300000) }).strict().refine(value => value.drainGraceMs > value.maximumLatenessMs, 'Drain grace must exceed lateness').optional(), + candidateConfigHash: hash, activeConfigHash: hash, qualificationHash: hash, engineBuildHash: hash, + startMs: integer, endMs: integer, maximumPoints: positive, maximumBytes: positive, + metrics: z.array(metric).min(1).max(3), +}).strict().refine(plan => plan.endMs > plan.startMs && plan.endMs - plan.startMs <= 7 * 86400000 + && Number.isSafeInteger(plan.endMs + (plan.joining?.drainGraceMs ?? 0)) + && (!plan.execution || (plan.joining !== undefined && plan.execution.sources.pollMs < plan.joining.maximumLatenessMs + && plan.execution.heartbeatMs <= plan.monitoring.heartbeatStaleMs + && plan.execution.configHashes[plan.side] === plan.candidateConfigHash)) + && new Set(plan.metrics).size === plan.metrics.length + && (!plan.metrics.includes('window_ibi_variability_v1') || plan.windowIbiWindowMs !== undefined), 'Invalid shadow window or metrics') diff --git a/src/biometrics/shadow/planned-sources.ts b/src/biometrics/shadow/planned-sources.ts new file mode 100644 index 000000000..1f7083186 --- /dev/null +++ b/src/biometrics/shadow/planned-sources.ts @@ -0,0 +1,83 @@ +import { dataHash } from '../algorithms/canonical' +import { shadowPlanSchema } from './store' +import { openShadowActiveDatabase } from './active-database' +import { ShadowContextReader } from './context' +import { openShadowLiveSources } from './live-sources' +import type { ShadowAttemptHost, ShadowSources } from './attempt' + +/** Trusted installation paths only. This factory owns its read-only connections + * and closes them after unsubscribing on every failure/cancellation/finish path. + * Admission and worker launch remain the attempt controller's responsibility. */ +export function plannedShadowSourceFactory(rawPlan: unknown, locations: { + biometricsDatabase: string + configurationDatabase: string + presenceFile: string +}): ShadowAttemptHost['openSources'] { + const plan = shadowPlanSchema.parse(structuredClone(rawPlan)) + if (!plan.execution || !plan.joining) throw new Error('Planned shadow sources require execution and joining policies') + const execution = structuredClone(plan.execution), joining = { ...plan.joining }, paths = { ...locations } + const executionHash = dataHash(execution) + return (input) => { + if (dataHash(input.execution) !== executionHash || input.collectionEndsAtMs !== plan.endMs) throw new Error('Shadow source policy differs from saved plan') + if (input.signal.aborted) throw new Error('Shadow sources cancelled before opening') + let biometrics: ReturnType | undefined + let configuration: ReturnType | undefined + let live: ShadowSources | undefined, closed = false + const close = () => { + if (closed) return + closed = true + input.signal.removeEventListener('abort', close) + try { + live?.close() + } + finally { + try { + configuration?.close() + } + finally { biometrics?.close() } + } + } + try { + const databaseOptions = { busyTimeoutMs: execution.sources.databaseBusyTimeoutMs } + biometrics = openShadowActiveDatabase(paths.biometricsDatabase, databaseOptions) + configuration = openShadowActiveDatabase(paths.configurationDatabase, databaseOptions) + const bio = biometrics, config = configuration + const assertCurrent = () => { + bio.assertCurrent() + config.assertCurrent() + } + const context = new ShadowContextReader(config.db, paths.presenceFile, execution.sources.context, assertCurrent) + live = openShadowLiveSources({ db: bio.db, side: plan.side, startMs: plan.startMs, endMs: plan.endMs, + activeConfigHash: plan.activeConfigHash, maximumActiveRows: execution.sources.maximumActiveRows, + maximumLatenessMs: joining.maximumLatenessMs, pollMs: execution.sources.pollMs, + frameBudgets: execution.sources.frames, context: (frame, at) => context.snapshot(frame, at), + assertDatabaseCurrent: assertCurrent, signal: input.signal, active: input.active, + fail: () => { + try { + close() + } + finally { input.fail() } + } }) + input.signal.addEventListener('abort', close, { once: true }) + if (input.signal.aborted) { + close() + throw new Error('Shadow sources cancelled while opening') + } + const source = live + return { frames: source.frames, flush: () => { + if (closed) throw new Error('Planned shadow sources are closed') + try { + source.flush?.() + } + catch (error) { + close() + throw error + } + }, close } + } + catch (error) { + close() + throw error + } + } +} diff --git a/src/biometrics/shadow/preparation-contracts.ts b/src/biometrics/shadow/preparation-contracts.ts new file mode 100644 index 000000000..38f54fe5a --- /dev/null +++ b/src/biometrics/shadow/preparation-contracts.ts @@ -0,0 +1,22 @@ +import { z } from 'zod' +import { shadowPlanSchema } from './plan-contracts' +import { shadowExecutionPolicySchema } from './execution-policy' +import { submitShadowSchema } from './submission-contracts' + +const time = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) +export const shadowPlanTemplateSchema = z.object({ + id: z.string().min(1).max(80), label: z.string().min(1).max(200), + qualificationHash: z.string().regex(/^[a-f0-9]{64}$/), + maximumWindowMs: time.positive().max(7 * 86400000), + execution: shadowExecutionPolicySchema, monitoring: shadowPlanSchema.shape.monitoring, + joining: shadowPlanSchema.shape.joining.unwrap(), metrics: shadowPlanSchema.shape.metrics, + windowIbiWindowMs: shadowPlanSchema.shape.windowIbiWindowMs, + maximumPoints: shadowPlanSchema.shape.maximumPoints, maximumBytes: shadowPlanSchema.shape.maximumBytes, +}).strict().refine(template => template.maximumWindowMs + template.joining.drainGraceMs <= template.execution.budgets.wallMs, + 'Shadow template window and drain exceed worker wall budget') +export const prepareShadowRequestSchema = z.object({ templateId: z.string().min(1).max(80), side: z.enum(['left', 'right']), + subjectId: z.string().min(1).max(200), candidateRevisionId: z.string().uuid().nullable(), startMs: time, endMs: time }).strict() +export const preparedShadowPlanSchema = z.object({ status: z.literal('prepared_not_admitted'), plan: submitShadowSchema.shape.plan, + planHash: z.string().regex(/^[a-f0-9]{64}$/) }).strict() +export const shadowTemplatePageSchema = z.object({ items: z.array(z.object({ id: shadowPlanTemplateSchema.shape.id, + label: shadowPlanTemplateSchema.shape.label, maximumWindowMs: shadowPlanTemplateSchema.shape.maximumWindowMs }).strict()).max(8) }).strict() diff --git a/src/biometrics/shadow/preparation.ts b/src/biometrics/shadow/preparation.ts new file mode 100644 index 000000000..52c745d2a --- /dev/null +++ b/src/biometrics/shadow/preparation.ts @@ -0,0 +1,35 @@ +import type Database from 'better-sqlite3' +import type { z } from 'zod' +import { configHash, dataHash } from '../algorithms/canonical' +import { manifestHash } from '../algorithms/registry' +import { captureShadowProfiles } from './profiles' +import { readInstalledShadowQualifications } from './admission' +import { prepareShadowRequestSchema, preparedShadowPlanSchema, shadowPlanTemplateSchema } from './preparation-contracts' +import { submitShadowSchema } from './submission-contracts' + +/** A server-owned template provides reviewed ceilings, never caller defaults. + * Preparation proves saved input consistency, not current hardware admission. */ +export function prepareShadowPlan(configuration: Database.Database, biometrics: Database.Database, + rawRequest: z.input, rawTemplate: unknown, workerPath: string, + assertCurrent: () => void, now = Date.now()) { + const request = prepareShadowRequestSchema.parse(rawRequest), template = shadowPlanTemplateSchema.parse(rawTemplate) + if (template.id !== request.templateId) throw new Error('Shadow template changed') + const duration = request.endMs - request.startMs + if (request.startMs < now || duration <= 0 || duration > template.maximumWindowMs + || duration + template.joining.drainGraceMs > template.execution.budgets.wallMs) throw new Error('Shadow window exceeds template limits or is not prospective') + const entries = readInstalledShadowQualifications(workerPath).entries.filter(entry => dataHash(entry) === template.qualificationHash) + if (entries.length !== 1) throw new Error('Shadow template qualification missing or ambiguous') + const entry = entries[0] + if (entry.manifestHash !== manifestHash || entry.executionPolicyHash !== dataHash(template.execution)) throw new Error('Shadow template differs from installed qualification') + const captured = captureShadowProfiles(configuration, biometrics, request.side, request.candidateRevisionId, assertCurrent, now) + if (captured.activeConfigHash !== entry.activeConfigHash + || dataHash({ left: configHash(captured.configs.left), right: configHash(captured.configs.right) }) !== dataHash(template.execution.configHashes)) throw new Error('Shadow template does not match candidate and active profiles') + const policy = { qualificationHash: template.qualificationHash, execution: template.execution, monitoring: template.monitoring, + joining: template.joining, metrics: template.metrics, maximumPoints: template.maximumPoints, maximumBytes: template.maximumBytes, + ...(template.windowIbiWindowMs === undefined ? {} : { windowIbiWindowMs: template.windowIbiWindowMs }) } + const plan = submitShadowSchema.shape.plan.parse({ ...policy, schema: 'biometric-shadow-plan-v1', + side: request.side, subjectId: request.subjectId, candidateRevisionId: request.candidateRevisionId, + startMs: request.startMs, endMs: request.endMs, expectedBindings: captured.expectedBindings, + candidateConfigHash: captured.candidateConfigHash, activeConfigHash: captured.activeConfigHash, engineBuildHash: entry.engineBuildHash }) + return preparedShadowPlanSchema.parse({ status: 'prepared_not_admitted', plan, planHash: dataHash(plan) }) +} diff --git a/src/biometrics/shadow/process-cleanup.ts b/src/biometrics/shadow/process-cleanup.ts new file mode 100644 index 000000000..167481e19 --- /dev/null +++ b/src/biometrics/shadow/process-cleanup.ts @@ -0,0 +1,28 @@ +import { setTimeout as delay } from 'node:timers/promises' + +/** Confirm both the owned child's close event and disappearance of its detached + * process group. Signal 0 observes existence; only ESRCH proves absence. + * Zombies and permission failures remain unconfirmed. Never signal PID/group 0. + * This cannot account for descendants that escape into a different group. */ +export async function confirmShadowProcessCleanup(pid: number | undefined, isClosed: () => boolean, timeoutMs = 5000) { + if (pid !== undefined && (!Number.isSafeInteger(pid) || pid < 1)) return false + if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 5000) return false + const deadline = performance.now() + timeoutMs + while (true) { + let absent = pid === undefined + if (pid !== undefined) { + try { + process.kill(-pid, 0) + } + catch (error) { + // Some kernels briefly return EPERM while killed children are reaped. + // Retry unknown observations within the deadline; never call them absent. + absent = (error as NodeJS.ErrnoException).code === 'ESRCH' + } + } + if (absent && isClosed()) return true + const remaining = deadline - performance.now() + if (remaining <= 0) return false + await delay(Math.min(25, remaining)) + } +} diff --git a/src/biometrics/shadow/profiles.ts b/src/biometrics/shadow/profiles.ts new file mode 100644 index 000000000..e045bbd3c --- /dev/null +++ b/src/biometrics/shadow/profiles.ts @@ -0,0 +1,63 @@ +import type Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { z } from 'zod' +import * as schema from '@/src/db/schema' +import { configHash, dataHash } from '../algorithms/canonical' +import { defaultConfig, validateConfig } from '../algorithms/registry' +import { ProfileStore } from '../profiles/store' +import { runtimePayloadSchema } from '../profiles/runtime' +import { shadowRuntimeBindingReader } from './runtime-binding' +import { shadowPlanSchema } from './store' + +/** Capture saved candidate and acknowledged baselines without activation or + * repair. Both physical channels must have adopted their desired generation. */ +export function captureShadowProfiles(configuration: Database.Database, biometrics: Database.Database, + rawSide: 'left' | 'right', rawRevisionId: string | null, assertCurrent: () => void, now = Date.now()) { + const side = z.enum(['left', 'right']).parse(rawSide), revisionId = z.string().uuid().nullable().parse(rawRevisionId) + for (const db of [configuration, biometrics]) { + if (!db.open || (!db.readonly && db.pragma('query_only', { simple: true }) !== 1)) throw new Error('Shadow profile resolution requires read-only databases') + } + const check = () => { + const value: unknown = assertCurrent() + if (value !== undefined) { + void Promise.resolve(value).catch(() => {}) + throw new Error('Shadow profile identity guard must be synchronous') + } + } + check() + const result = configuration.transaction(() => biometrics.transaction(() => { + const profiles = new ProfileStore(drizzle(configuration, { schema })) + const candidate = revisionId === null ? structuredClone(defaultConfig) : profiles.validateRevision(revisionId, side).resolved + const capture = (side: 'left' | 'right') => { + const desired = z.object({ generation: z.number().int().positive().max(Number.MAX_SAFE_INTEGER), resolvedHash: z.string().regex(/^[a-f0-9]{64}$/) }) + .parse(configuration.prepare(`SELECT b.generation, e.next_hash AS resolvedHash FROM biometric_profile_bindings b + JOIN biometric_activation_events e ON e.side = b.side AND e.generation = b.generation WHERE b.side = ?`).get(side)) + const row = biometrics.prepare(`SELECT CASE WHEN length(CAST(payload AS BLOB)) <= 65536 THEN payload ELSE NULL END AS payload + FROM biometric_runtime_configs WHERE side = ? AND generation = ?`).get(side, desired.generation) as { payload: string | null } | undefined + if (!row?.payload) throw new Error('Shadow baseline projection missing or oversized') + const payload = runtimePayloadSchema.parse(JSON.parse(row.payload)) + return { expected: desired, resolved: validateConfig(payload.resolved) } + } + const left = capture('left'), right = capture('right') + const expectedBindings = { left: left.expected, right: right.expected } + const binding = shadowRuntimeBindingReader(configuration, biometrics, { side, expected: expectedBindings, assertCurrent: check, now: () => now })() + const configs = { left: left.resolved, right: right.resolved, [side]: candidate } + return { revisionId, configs, expectedBindings, heartbeatAtMs: binding.heartbeatAtMs, candidateConfigHash: configHash(candidate), activeConfigHash: binding.resolvedHash } + })())() + check() + return structuredClone(result) +} + +/** Validate a frozen plan against a current capture. */ +export function resolveShadowProfiles(configuration: Database.Database, biometrics: Database.Database, rawPlan: unknown, + rawRevisionId: string | null, assertCurrent: () => void, now = Date.now()) { + const plan = shadowPlanSchema.parse(rawPlan), revisionId = z.string().uuid().nullable().parse(rawRevisionId) + if (!plan.execution) throw new Error('Shadow profile resolution requires execution policy') + if (plan.candidateRevisionId !== undefined && plan.candidateRevisionId !== revisionId) throw new Error('Shadow candidate revision differs from saved plan') + const result = captureShadowProfiles(configuration, biometrics, plan.side, revisionId, assertCurrent, now) + if (result.candidateConfigHash !== plan.candidateConfigHash) throw new Error('Shadow candidate differs from saved plan') + if (plan.expectedBindings && dataHash(plan.expectedBindings) !== dataHash(result.expectedBindings)) throw new Error('Shadow active generations differ from saved plan') + if (result.activeConfigHash !== plan.activeConfigHash) throw new Error('Shadow active baseline differs from saved plan') + if (dataHash({ left: configHash(result.configs.left), right: configHash(result.configs.right) }) !== dataHash(plan.execution.configHashes)) throw new Error('Shadow physical-channel configs differ from saved plan') + return { revisionId: result.revisionId, configs: result.configs, expectedBindings: result.expectedBindings, heartbeatAtMs: result.heartbeatAtMs } +} diff --git a/src/biometrics/shadow/python-installation.ts b/src/biometrics/shadow/python-installation.ts new file mode 100644 index 000000000..159da29cb --- /dev/null +++ b/src/biometrics/shadow/python-installation.ts @@ -0,0 +1,66 @@ +import { createHash } from 'node:crypto' +import { constants, openSync, closeSync, fstatSync, lstatSync, readSync, realpathSync } from 'node:fs' +import { isAbsolute, resolve } from 'node:path' +import { configHash } from '../algorithms/canonical' +import { shadowPythonBuildSchema } from './build' + +const maximumBytes = 16 * 1024 * 1024 +function signature(path: string) { + const stat = lstatSync(path, { bigint: true }) + return `${stat.dev}:${stat.ino}:${stat.mode}:${stat.size}:${stat.mtimeNs}:${stat.ctimeNs}` +} + +/** Capture known source and dependency metadata artifacts after the Python + * probe, validating their content against that probe before retaining cheap + * synchronous file-identity checks. Does not attest every installed binary. */ +export function guardShadowPythonInstallation(location: { pythonPath: string, modulesPath: string }, rawBuild: unknown) { + const build = shadowPythonBuildSchema.parse(rawBuild) + if (configHash(build.identity) !== build.hash) throw new Error('Shadow Python build integrity mismatch') + const files: { path: string, hash: string, metadata: boolean }[] = [] + for (const [relative, hash] of Object.entries(build.identity.sources)) { + if (relative.split('/').some(part => !part || part === '.' || part === '..')) throw new Error('Invalid Python source identity path') + files.push({ path: resolve(location.modulesPath, relative), hash, metadata: false }) + } + for (const name of ['numpy', 'scipy'] as const) { + for (const kind of ['record', 'wheel'] as const) { + const path = build.dependencyFiles[name][kind] + if (!isAbsolute(path)) throw new Error('Dependency metadata path must be absolute') + files.push({ path, hash: build.identity.dependencies[name][`${kind}_sha256`], metadata: true }) + } + } + const watched: { path: string, canonical: string, requestedSignature: string, canonicalSignature: string }[] = [] + const watch = (path: string) => { + const requestedSignature = signature(path), canonical = realpathSync(path), canonicalSignature = signature(canonical) + watched.push({ path, canonical, requestedSignature, canonicalSignature }) + return canonical + } + const assertCurrent = () => { + for (const file of watched) { + if (signature(file.path) !== file.requestedSignature || realpathSync(file.path) !== file.canonical + || signature(file.canonical) !== file.canonicalSignature) throw new Error('Shadow Python installation changed') + } + } + for (const file of files) { + const canonical = watch(file.path), fd = openSync(canonical, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK) + try { + const stat = fstatSync(fd) + if (!stat.isFile() || stat.size > maximumBytes) throw new Error('Invalid Python identity artifact') + const buffer = Buffer.alloc(65536), chunks: Buffer[] = [] + let length = 0 + while (true) { + const count = readSync(fd, buffer, 0, buffer.length, null) + if (!count) break + length += count + if (length > maximumBytes) throw new Error('Python identity artifact exceeds its bound') + chunks.push(Buffer.from(buffer.subarray(0, count))) + } + const bytes = Buffer.concat(chunks, length) + const content = file.metadata ? new TextDecoder('utf-8', { fatal: true }).decode(bytes).replace(/\r\n?/g, '\n') : bytes + if (createHash('sha256').update(content).digest('hex') !== file.hash) throw new Error('Python identity artifact differs from probe') + } + finally { closeSync(fd) } + } + watch(resolve(location.pythonPath)) // interpreter links and target replacement + assertCurrent() + return assertCurrent +} diff --git a/src/biometrics/shadow/receipt.ts b/src/biometrics/shadow/receipt.ts new file mode 100644 index 000000000..b4a0e2f5d --- /dev/null +++ b/src/biometrics/shadow/receipt.ts @@ -0,0 +1,46 @@ +import { z } from 'zod' +import { dataHash } from '../algorithms/canonical' +import { shadowChildBudgetsSchema } from './execution-policy' +import { shadowWorkerHealthSchema } from './worker-protocol' +import type { ShadowExecutionPolicy } from './execution-policy' +const integer = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) +export const shadowWorkerReceiptSchema = z.object({ health: shadowWorkerHealthSchema, + inputBytes: integer, outputBytes: integer, peakObservedRssBytes: integer.positive(), rssSamples: integer.positive(), + maximumObservationGapMs: z.number().finite().nonnegative(), budgets: shadowChildBudgetsSchema, + memoryScope: z.literal('sum_rss_of_shadow_process_group'), exactKernelPeak: z.literal(false), +}).strict() +export function validateShadowWorkerReceipt(raw: unknown, execution: ShadowExecutionPolicy) { + const receipt = shadowWorkerReceiptSchema.parse(raw) + const h = receipt.health + if (h.state !== 'completed' || h.reason !== 'window_complete' || h.records !== h.counts.records || h.outputs > h.counts.scored_outputs + || h.counts.accepted_frames + h.counts.empty_channel_frames + h.counts.pump_gated_frames > h.records + || h.counts.scored_outputs + h.counts.preroll_outputs > 2 * h.counts.accepted_frames + || h.records > execution.maximumRecords || h.outputs > execution.maximumOutputs || h.cpu_ms > execution.maximumCpuMs + || receipt.inputBytes > execution.maximumInputBytes || receipt.outputBytes > execution.maximumOutputBytes + || receipt.peakObservedRssBytes > execution.budgets.maximumRssBytes + || dataHash(receipt.budgets) !== dataHash(execution.budgets)) throw new Error('Shadow worker receipt does not match execution policy') + return receipt +} +const completedEnvelopeSchema = z.object({ schema: z.literal('biometric-shadow-receipt-v1'), + runId: z.string().uuid(), attemptId: z.string().uuid(), planHash: z.string().regex(/^[a-f0-9]{64}$/), + recordedAtMs: integer, receipt: shadowWorkerReceiptSchema }).strict() + +/** Supervisor observations from an incomplete attempt. Quota excess is evidence, + * not a reason to discard the receipt. CPU/health is only the last valid reply. */ +export const shadowPartialReceiptSchema = z.object({ + kind: z.literal('partial_supervisor_observation'), health: shadowWorkerHealthSchema.nullable(), + inputBytes: integer, outputBytes: integer, diagnosticBytes: integer, + peakObservedRssBytes: integer.positive().nullable(), rssSamples: integer, + maximumObservationGapMs: z.number().finite().nonnegative(), budgets: shadowChildBudgetsSchema, + cleanupConfirmed: z.boolean(), memoryScope: z.literal('sum_rss_of_shadow_process_group'), exactKernelPeak: z.literal(false), +}).strict().refine(value => (value.rssSamples === 0) === (value.peakObservedRssBytes === null), 'RSS availability mismatch') +export type ShadowPartialReceipt = z.infer +export function validateShadowPartialReceipt(raw: unknown, execution: ShadowExecutionPolicy) { + const receipt = shadowPartialReceiptSchema.parse(raw) + if (dataHash(receipt.budgets) !== dataHash(execution.budgets)) throw new Error('Shadow partial receipt policy mismatch') + return receipt +} +export const shadowPartialEnvelopeSchema = z.object({ schema: z.literal('biometric-shadow-partial-receipt-v1'), + runId: z.string().uuid(), attemptId: z.string().uuid(), planHash: z.string().regex(/^[a-f0-9]{64}$/), + recordedAtMs: integer, receipt: shadowPartialReceiptSchema }).strict() +export const shadowReceiptEnvelopeSchema = z.discriminatedUnion('schema', [completedEnvelopeSchema, shadowPartialEnvelopeSchema]) diff --git a/src/biometrics/shadow/report-contracts.ts b/src/biometrics/shadow/report-contracts.ts new file mode 100644 index 000000000..2d452942a --- /dev/null +++ b/src/biometrics/shadow/report-contracts.ts @@ -0,0 +1,40 @@ +import { z } from 'zod' +import { previewSummarySchema, referenceImportSchema, referenceMetricSchema } from '../references/contracts' +import { shadowReceiptEnvelopeSchema } from './receipt' +import { shadowMonitoringSchema } from './plan-contracts' + +const count = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) +const finite = z.number().finite() +const nullable = finite.nullable() +const hash = z.string().regex(/^[a-f0-9]{64}$/) +const score = z.object({ status: z.enum(['no_reference', 'compared', 'no_overlap', 'incompatible']), + metrics: z.object({ mae: finite, medianAbsoluteError: finite, p95AbsoluteError: finite, signedBias: finite }).strict().nullable(), + pairedCount: count, referenceCoverage: nullable, validObservationCount: count, unmatchedReferences: count, unmatchedObservations: count }).strict() +const support = z.object({ baseline: score, candidate: score }).strict() +const coverage = z.object({ usableCount: count, usableSlots: count, coverage: nullable }).strict() +const comparison = z.object({ version: z.literal('paired-numeric-support-v1'), metric: referenceMetricSchema, + baseline: z.enum(['default', 'current', 'candidate']), candidate: z.enum(['default', 'current', 'candidate']), baselineHash: hash, candidateHash: hash, + commonSupportScope: z.literal('all_configurations'), supportDefinition: z.literal('identical_unambiguous_observation_timestamps'), + commonObservationCount: count, ownSupport: support, commonSupport: support, + differences: z.object({ commonMae: nullable, commonMedianAbsoluteError: nullable, commonP95AbsoluteError: nullable, + commonSignedBias: nullable, ownMae: nullable, referenceCoverage: nullable }).strict(), + lostObservationTimestamps: count, addedObservationTimestamps: count, + ambiguousObservationTimestamps: z.object({ baseline: count, candidate: count }).strict(), coverageReduced: z.boolean(), + interpretation: z.literal('descriptive_numeric_agreement_not_validated_improvement'), +}).strict() +export const shadowReportRequestSchema = z.object({ id: z.string().uuid(), reference: referenceImportSchema.optional() }).strict() +export const shadowReportSchema = z.object({ + schema: z.literal('biometric-shadow-report-v1'), runId: z.string().uuid(), planHash: hash, snapshotHash: hash, reportHash: hash, + capturedAtMs: count, state: z.enum(['pending', 'running', 'completed', 'stopped', 'failed']), reason: z.string().nullable(), + health: z.enum(['issues_detected', 'waiting', 'within_observed_limits']), issues: z.array(z.string()), + expectedSlots: count, observedSamples: count, maximumGapMs: count, attemptsObserved: count, + alignment: shadowMonitoringSchema.shape.alignment, + metrics: z.array(z.object({ metric: z.enum(['heart_rate', 'breathing_rate', 'window_ibi_variability_v1']), windowMs: count.nullable(), + current: coverage, candidate: coverage, coverageDifference: nullable, issues: z.array(z.string()), + referenceExclusions: z.record(z.string(), count), comparison }).strict()).max(3), + pairing: z.object({ activeOnly: count, candidateOnly: count, unknownInputIdentity: count }).strict(), + workerReceipt: shadowReceiptEnvelopeSchema.nullable(), executionPolicyHash: hash.nullable(), + contextEvidence: z.literal('host_observations_not_live_processor_cache_attestation'), limitations: z.array(z.string()), + referenceSummary: previewSummarySchema.nullable(), excludedBoundaryReferences: count, + interpretation: z.literal('prospective_observation_and_reference_agreement_not_activation_authorization'), +}).strict() diff --git a/src/biometrics/shadow/report-core.ts b/src/biometrics/shadow/report-core.ts new file mode 100644 index 000000000..a9786b3d7 --- /dev/null +++ b/src/biometrics/shadow/report-core.ts @@ -0,0 +1,87 @@ +import { compareNumericConfigurations } from '../backtest/numeric-comparison' +import { referenceCompatibility } from '../references/alignment' +import type { previewReferenceImport } from '../references/normalize' +import type { ShadowStore } from './store' + +type Snapshot = ReturnType +export type ShadowReportSnapshot = Pick & { + run: Pick +} +/** Pure report calculations shared by persisted production evidence and the + * explicitly simulated browser demo. Storage integrity remains the caller's job. */ +export function summarizeShadowSnapshot(snapshot: ShadowReportSnapshot, reference: ReturnType | null, executionPolicyHash: string | null) { + const { run, samples, capturedAtMs } = snapshot + const { plan } = run, policy = plan.monitoring + const until = Math.max(plan.startMs, Math.min(plan.endMs, capturedAtMs)) + const expectedSlots = capturedAtMs < plan.startMs + ? 0 + : capturedAtMs < plan.endMs + ? Math.floor((capturedAtMs - plan.startMs) / policy.expectedIntervalMs) + 1 + : Math.ceil((plan.endMs - plan.startMs) / policy.expectedIntervalMs) + const windowSamples = samples.filter(sample => sample.timestampMs <= capturedAtMs && sample.timestampMs < plan.endMs) + const clockSkew = samples.length !== windowSamples.length || (run.heartbeat_at_ms !== null && run.heartbeat_at_ms > capturedAtMs) + let maximumGapMs = 0, previous = plan.startMs + for (const sample of windowSamples) { + maximumGapMs = Math.max(maximumGapMs, sample.timestampMs - previous) + previous = sample.timestampMs + } + maximumGapMs = Math.max(maximumGapMs, until - previous) + const measurements = reference?.measurements.filter(row => row.startMs >= plan.startMs && row.startMs < plan.endMs && row.endMs <= plan.endMs && row.endMs <= until) ?? [] + const metrics = plan.metrics.map((metric) => { + const windowMs = metric === 'window_ibi_variability_v1' ? plan.windowIbiWindowMs ?? null : null + const outputs = (['current', 'candidate'] as const).map((label) => { + const slots = new Set() + const observations = windowSamples.map((sample) => { + const row = sample.readings.find(row => row.metric === metric) + if (!row) throw new Error('Missing shadow metric') + const reading = label === 'current' ? row.active : row.candidate + const value = reading.quality >= policy.minimumQuality ? reading.value : null + if (value !== null) slots.add(Math.floor((sample.timestampMs - plan.startMs) / policy.expectedIntervalMs)) + return { timestampMs: sample.timestampMs, value } + }) + return { label, configHash: label === 'current' ? plan.activeConfigHash : plan.candidateConfigHash, observations, + usableCount: observations.filter(row => row.value !== null).length, usableSlots: slots.size, + coverage: expectedSlots ? Math.min(1, slots.size / expectedSlots) : null } + }) + const current = outputs[0], candidate = outputs[1] + const coverageDifference = current.coverage === null || candidate.coverage === null ? null : candidate.coverage - current.coverage + const issues = [] + if (candidate.coverage !== null && candidate.coverage < policy.minimumCandidateCoverage) issues.push('candidate_coverage_below_minimum') + if (coverageDifference !== null && coverageDifference < -policy.maximumCoverageLoss) issues.push('coverage_regression') + const exclusions: Record = {} + for (const row of measurements) { + const reason = referenceCompatibility(metric, windowMs, row) + if (reason) exclusions[reason] = (exclusions[reason] ?? 0) + 1 + } + return { metric, windowMs, current: { usableCount: current.usableCount, usableSlots: current.usableSlots, coverage: current.coverage }, + candidate: { usableCount: candidate.usableCount, usableSlots: candidate.usableSlots, coverage: candidate.coverage }, coverageDifference, issues, + referenceExclusions: exclusions, comparison: compareNumericConfigurations(metric, windowMs, outputs, measurements, policy.alignment)[0] } + }) + const issues: string[] = [] + if (run.workerReceipt?.schema === 'biometric-shadow-partial-receipt-v1' && !run.workerReceipt.receipt.cleanupConfirmed) issues.push('worker_cleanup_unconfirmed') + if (clockSkew) issues.push('clock_skew') + if (run.reason?.startsWith('join_')) issues.push('join_failed') + if (run.reason === 'shadow_observation_window_missed') issues.push('observation_window_missed') + else if (run.state === 'failed') issues.push('processor_failed') + if (!issues.includes('worker_cleanup_unconfirmed') && (run.reason === 'shadow_lease_expired_cleanup_unconfirmed' || run.reason === 'shadow_worker_cleanup_unconfirmed')) issues.push('worker_cleanup_unconfirmed') + if (run.reason === 'storage_quota') issues.push('storage_quota') + if (run.state === 'running' && ((run.lease_until_ms ?? 0) <= capturedAtMs || run.heartbeat_at_ms === null + || capturedAtMs - run.heartbeat_at_ms > policy.heartbeatStaleMs)) issues.push('worker_stale') + if (maximumGapMs > policy.maximumGapMs) issues.push('observation_gap') + if (!windowSamples.length && capturedAtMs >= plan.startMs) issues.push('no_observations') + if (metrics.some(metric => metric.issues.length)) issues.push('coverage_degraded') + const report = { schema: 'biometric-shadow-report-v1' as const, runId: run.id, planHash: run.plan_hash, snapshotHash: snapshot.snapshotHash, + capturedAtMs, state: run.state, reason: run.reason, health: issues.length ? 'issues_detected' as const : capturedAtMs < plan.startMs ? 'waiting' as const : 'within_observed_limits' as const, + issues, expectedSlots, observedSamples: windowSamples.length, maximumGapMs, alignment: policy.alignment, + attemptsObserved: new Set(samples.map(sample => sample.attemptId)).size, metrics, + pairing: { activeOnly: samples.filter(sample => sample.pairing?.activePresent && !sample.pairing.candidatePresent).length, + candidateOnly: samples.filter(sample => sample.pairing?.candidatePresent && !sample.pairing.activePresent).length, + unknownInputIdentity: samples.filter(sample => sample.inputHash === null).length }, + workerReceipt: run.workerReceipt, + executionPolicyHash: executionPolicyHash, + contextEvidence: 'host_observations_not_live_processor_cache_attestation' as const, + limitations: [...new Set(samples.flatMap(sample => sample.candidateProvenance?.limitations ?? []))].sort(), + referenceSummary: reference?.summary ?? null, excludedBoundaryReferences: reference ? reference.measurements.length - measurements.length : 0, + interpretation: 'prospective_observation_and_reference_agreement_not_activation_authorization' as const } + return report +} diff --git a/src/biometrics/shadow/report-operation.ts b/src/biometrics/shadow/report-operation.ts new file mode 100644 index 000000000..187ee839e --- /dev/null +++ b/src/biometrics/shadow/report-operation.ts @@ -0,0 +1,25 @@ +import type Database from 'better-sqlite3' +import type { z } from 'zod' +import { ReferenceInputError } from '../references/time' +import { ShadowStore } from './store' +import { reportShadowRun } from './report' +import { shadowReportRequestSchema, shadowReportSchema } from './report-contracts' + +/** Coordinator-only bounded computation; no execution policy or worker needed. + * Failure on oversized evidence is explicit, never a downsampled report. */ +export function readShadowReport(db: Database.Database, raw: z.input) { + const input = shadowReportRequestSchema.parse(raw) + const store = new ShadowStore(db) + try { + const plan = store.get(input.id).plan + if (input.reference && (input.reference.side !== plan.side || input.reference.subjectId !== plan.subjectId)) { + throw new ReferenceInputError('SUBJECT_MISMATCH', 'Shadow reference must match the saved sleeper and side') + } + return shadowReportSchema.parse(reportShadowRun(store, input.id, { maximumPoints: 100000, maximumBytes: 32 * 1024 * 1024 }, input.reference)) + } + catch (error) { + if (error instanceof Error && error.message === 'Unknown shadow run') throw new ReferenceInputError('NOT_FOUND', 'Shadow run not found') + if (error instanceof Error && error.message === 'Shadow report input limit exceeded') throw new ReferenceInputError('RESOURCE_LIMIT', 'Shadow report exceeds the bounded evidence limit') + throw error + } +} diff --git a/src/biometrics/shadow/report.ts b/src/biometrics/shadow/report.ts new file mode 100644 index 000000000..08c6cc593 --- /dev/null +++ b/src/biometrics/shadow/report.ts @@ -0,0 +1,15 @@ +import { dataHash } from '../algorithms/canonical' +import { referenceImportSchema } from '../references/contracts' +import { previewReferenceImport } from '../references/normalize' +import type { ShadowStore } from './store' +import { summarizeShadowSnapshot } from './report-core' + +/** Complete, integrity-checked storage snapshot; never canonical vitals. */ +export function reportShadowRun(store: ShadowStore, runId: string, limits: Parameters[1], rawReference?: unknown) { + const snapshot = store.snapshot(runId, limits), { plan } = snapshot.run + const referenceInput = rawReference === undefined ? null : referenceImportSchema.parse(rawReference) + if (referenceInput && (referenceInput.subjectId !== plan.subjectId || referenceInput.side !== plan.side)) throw new Error('Shadow reference subject or side mismatch') + const reference = referenceInput ? previewReferenceImport(referenceInput) : null + const report = summarizeShadowSnapshot(snapshot, reference, plan.execution ? dataHash(plan.execution) : null) + return { ...report, reportHash: dataHash(report) } +} diff --git a/src/biometrics/shadow/runtime-binding.ts b/src/biometrics/shadow/runtime-binding.ts new file mode 100644 index 000000000..2dcd0c713 --- /dev/null +++ b/src/biometrics/shadow/runtime-binding.ts @@ -0,0 +1,71 @@ +import type Database from 'better-sqlite3' +import { z } from 'zod' +import { configHash } from '../algorithms/canonical' +import { manifestHash, validateConfig } from '../algorithms/registry' +import { runtimePayloadSchema } from '../profiles/runtime' + +const integer = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) +const expectedSchema = z.object({ generation: integer.positive(), resolvedHash: z.string().regex(/^[a-f0-9]{64}$/) }).strict() +const receiptSchema = z.object({ generation: integer, applied_hash: z.string(), state: z.literal('applied'), + effective_sample_time_ms: integer, heartbeat_at_ms: integer }) +const sides = ['left', 'right'] as const +const components = ['heart_rate', 'breathing_rate', 'hrv_index'] as const + +/** Read-only observation of both physical-channel baselines. No default-config + * fallback: an unacknowledged channel cannot establish prospective admission. + * The aggregate uses the oldest heartbeat so the installed qualification's age + * limit applies to every numerical component on both sides. */ +export function shadowRuntimeBindingReader(configDb: Database.Database, bioDb: Database.Database, options: { + side: 'left' | 'right' + expected: Record<'left' | 'right', { generation: number, resolvedHash: string }> + assertCurrent: () => void + now?: () => number +}) { + const expected = z.object({ left: expectedSchema, right: expectedSchema }).strict().parse(options.expected) + const side = z.enum(sides).parse(options.side), now = options.now ?? Date.now, guard = options.assertCurrent + const check = () => { + for (const db of [configDb, bioDb]) { + if (!db.open || (!db.readonly && db.pragma('query_only', { simple: true }) !== 1)) throw new Error('Shadow runtime observation requires read-only databases') + } + const result: unknown = guard() + if (result !== undefined) { + void Promise.resolve(result).catch(() => {}) + throw new Error('Shadow database identity guard must be synchronous') + } + } + check() + return () => { + check() + const at = integer.parse(now()) + const heartbeats: number[] = [] + configDb.transaction(() => bioDb.transaction(() => { + for (const channel of sides) { + const target = expected[channel] + const binding = configDb.prepare('SELECT generation FROM biometric_profile_bindings WHERE side = ?').get(channel) as { generation: number } | undefined + if (binding?.generation !== target.generation) throw new Error('Shadow runtime desired generation changed') + const event = configDb.prepare('SELECT next_hash FROM biometric_activation_events WHERE side = ? AND generation = ?').get(channel, target.generation) as { next_hash: string } | undefined + if (event?.next_hash !== target.resolvedHash) throw new Error('Shadow runtime desired configuration changed') + const projection = bioDb.prepare(`SELECT resolved_hash, boundary, CASE WHEN length(CAST(payload AS BLOB)) <= 65536 THEN payload ELSE NULL END AS payload + FROM biometric_runtime_configs WHERE side = ? AND generation = ?`).get(channel, target.generation) as { resolved_hash: string, boundary: string, payload: string | null } | undefined + if (!projection?.payload) throw new Error('Shadow runtime projection missing or oversized') + const payload = runtimePayloadSchema.parse(JSON.parse(projection.payload)) + if (payload.side !== channel || payload.generation !== target.generation || payload.manifestHash !== manifestHash + || payload.resolvedHash !== target.resolvedHash || projection.resolved_hash !== target.resolvedHash + || payload.boundary !== projection.boundary || configHash(validateConfig(payload.resolved)) !== target.resolvedHash) throw new Error('Shadow runtime projection mismatch') + for (const component of components) { + // Read latest intent, including pending/failed generations; never fall + // back to an older applied receipt after a replacement is announced. + const receipt = receiptSchema.parse(bioDb.prepare(`SELECT generation, applied_hash, state, effective_sample_time_ms, heartbeat_at_ms + FROM biometric_runtime_receipts WHERE side = ? AND component = ? ORDER BY generation DESC LIMIT 1`).get(channel, component)) + if (receipt.generation !== target.generation || receipt.applied_hash !== target.resolvedHash + || receipt.effective_sample_time_ms > at || receipt.heartbeat_at_ms > at + || receipt.heartbeat_at_ms < receipt.effective_sample_time_ms) throw new Error('Shadow runtime receipt mismatch or clock skew') + heartbeats.push(receipt.heartbeat_at_ms) + } + } + })())() + check() + return { side, ...expected[side], appliedHash: expected[side].resolvedHash, + appliedGeneration: expected[side].generation, state: 'applied' as const, heartbeatAtMs: Math.min(...heartbeats) } + } +} diff --git a/src/biometrics/shadow/service-host.ts b/src/biometrics/shadow/service-host.ts new file mode 100644 index 000000000..cb5dfbc37 --- /dev/null +++ b/src/biometrics/shadow/service-host.ts @@ -0,0 +1,53 @@ +import { dataHash } from '../algorithms/canonical' +import { runInstalledShadowAttempt, reconcileInstalledShadowRuns, pendingInstalledShadowRun } from './installed-attempt' +import { shadowStoragePolicySchema } from './store' +import { ShadowService } from './service' + +interface InstallationOptions { + installation: Parameters[2] + storagePolicy: unknown + busyTimeoutMs: number + recoveryIntervalMs: number + recoveryMaximumRuns: number +} +const shared = globalThis as typeof globalThis & { + __biometricShadowService?: { service: ShadowService, configurationHash: string } +} + +/** Explicit deployment configuration only; no unqualified execution defaults. */ +export function startShadowService(raw: InstallationOptions) { + const options = structuredClone(raw) + options.storagePolicy = shadowStoragePolicySchema.parse(options.storagePolicy) + if (!Number.isSafeInteger(options.busyTimeoutMs) || options.busyTimeoutMs < 1 || options.busyTimeoutMs > 1000) throw new Error('Invalid shadow database timeout') + const configurationHash = dataHash(options) + const existing = shared.__biometricShadowService + if (existing) { + if (existing.configurationHash !== configurationHash) throw new Error('Shadow service configuration changed; restart required') + return existing.service + } + const service = new ShadowService({ + recoveryIntervalMs: options.recoveryIntervalMs, + recoveryMaximumRuns: options.recoveryMaximumRuns, + recover: cursor => reconcileInstalledShadowRuns(options.installation, options.storagePolicy, options.busyTimeoutMs, cursor), + pending: cursor => pendingInstalledShadowRun(options.installation, options.storagePolicy, options.busyTimeoutMs, cursor), + execute: (runId, revisionId, signal) => runInstalledShadowAttempt(runId, revisionId, options.installation, + options.storagePolicy, options.busyTimeoutMs, signal), + onRecoveryError: () => console.warn('[biometrics] Shadow recovery failed; service will retry'), + onExecutionError: () => console.warn('[biometrics] Shadow scheduling or execution failed'), + }) + shared.__biometricShadowService = { service, configurationHash } + service.start() + return service +} + +export async function stopShadowService() { + const current = shared.__biometricShadowService + if (!current) return + await current.service.stop() + if (shared.__biometricShadowService === current) delete shared.__biometricShadowService +} + +/** Durable stop must be recorded before invoking this local cleanup helper. */ +export async function cancelShadowServiceAttempt(runId: string) { + return await shared.__biometricShadowService?.service.cancel(runId) ?? false +} diff --git a/src/biometrics/shadow/service.ts b/src/biometrics/shadow/service.ts new file mode 100644 index 000000000..e3bc4bfb1 --- /dev/null +++ b/src/biometrics/shadow/service.ts @@ -0,0 +1,113 @@ +import { isShadowCleanupError } from './cleanup-error' +import { z } from 'zod' +import type { ShadowStore } from './store' + +interface ShadowServiceOptions { + execute: (runId: string, revisionId: string | null, signal: AbortSignal) => Promise + recover: (options: Parameters[0]) => ReturnType + recoveryIntervalMs: number + recoveryMaximumRuns: number + onRecoveryError?: () => void + pending?: (options: Parameters[0]) => ReturnType + onExecutionError?: () => void +} + +/** Owns local attempts and bounded maintenance. The store lease and worker flock + * remain authoritative across processes. An aborted attempt keeps its slot until + * execution has finished cleanup. Scheduling reads saved pending plans only. */ +export class ShadowService { + private stopped = false + private cleanupUnconfirmed = false + private timer: ReturnType | null = null + private cursor: string | undefined + private pendingCursor: string | undefined + private active: { runId: string, controller: AbortController, done: Promise } | null = null + private readonly options: ShadowServiceOptions + + constructor(options: ShadowServiceOptions) { + z.number().int().min(10).max(300000).parse(options.recoveryIntervalMs) + z.number().int().min(1).max(100).parse(options.recoveryMaximumRuns) + this.options = { ...options } + } + + start() { + if (this.stopped || this.timer) return + this.reconcile() + } + + private reconcile() { + try { + const result = this.options.recover({ maximumRuns: this.options.recoveryMaximumRuns, + ...(this.cursor ? { afterId: this.cursor } : {}) }) + this.cursor = result.nextCursor ?? undefined + } + catch { + try { + this.options.onRecoveryError?.() + } + catch { /* Logging must not disable recovery. */ } + } + if (!this.stopped && !this.cleanupUnconfirmed && !this.active && this.options.pending) { + try { + const result = this.options.pending({ maximumRuns: this.options.recoveryMaximumRuns, + ...(this.pendingCursor ? { afterId: this.pendingCursor } : {}) }) + this.pendingCursor = result.nextCursor ?? undefined + if (result.run) void this.run(result.run.runId, result.run.revisionId).catch(() => this.reportExecutionError()) + } + catch { this.reportExecutionError() } + } + if (!this.stopped) { + this.timer = setTimeout(() => { + this.timer = null + this.reconcile() + }, this.options.recoveryIntervalMs) + this.timer.unref() + } + } + + private reportExecutionError() { + try { + this.options.onExecutionError?.() + } + catch { /* Logging must not interrupt scheduling or cause rejection. */ } + } + + run(runId: string, revisionId: string | null) { + z.string().uuid().parse(runId) + z.string().uuid().nullable().parse(revisionId) + if (this.stopped) throw new Error('Shadow service stopped') + if (this.cleanupUnconfirmed) throw new Error('Shadow service worker cleanup unconfirmed') + if (this.active) throw new Error('Shadow service busy') + const controller = new AbortController() + // Defer execution so even synchronous failure cannot leave a stale slot. + const done = Promise.resolve().then(() => { + if (controller.signal.aborted) throw new Error('Shadow attempt cancelled') + return this.options.execute(runId, revisionId, controller.signal) + }).catch((error: unknown) => { + if (isShadowCleanupError(error)) this.cleanupUnconfirmed = true + throw error + }).finally(() => { + if (this.active?.done === done) this.active = null + }) + this.active = { runId, controller, done } + return done + } + + async cancel(runId: string) { + z.string().uuid().parse(runId) + const active = this.active + if (!active || active.runId !== runId) return false + active.controller.abort() + await active.done.catch(() => {}) + return !this.cleanupUnconfirmed + } + + async stop() { + this.stopped = true + if (this.timer) clearTimeout(this.timer) + this.timer = null + const active = this.active + active?.controller.abort() + await active?.done.catch(() => {}) + } +} diff --git a/src/biometrics/shadow/startup.ts b/src/biometrics/shadow/startup.ts new file mode 100644 index 000000000..d3646c990 --- /dev/null +++ b/src/biometrics/shadow/startup.ts @@ -0,0 +1,65 @@ +import { isAbsolute } from 'node:path' +import { sql } from 'drizzle-orm' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import type * as configSchema from '@/src/db/schema' +import type * as biometricsSchema from '@/src/db/biometrics-schema' +import { z } from 'zod' +import { packagedShadowHostPath } from '../backtest/location' +import { shadowStoragePolicySchema } from './store' +import { startShadowService } from './service-host' +import { shadowPlanTemplateSchema } from './preparation-contracts' + +const absolutePath = z.string().min(1).max(4096).refine(value => isAbsolute(value) && !value.includes('\0'), 'Expected absolute installation path') +export const shadowServiceConfigurationSchema = z.object({ + version: z.literal('shadow-service-configuration-v1'), + pythonPath: absolutePath, + modulesPath: absolutePath, + presenceFile: absolutePath, + storagePolicy: shadowStoragePolicySchema, + busyTimeoutMs: z.number().int().min(1).max(1000), + recoveryIntervalMs: z.number().int().min(10).max(300000), + recoveryMaximumRuns: z.number().int().min(1).max(100), + planTemplates: z.array(shadowPlanTemplateSchema).max(8).refine(items => new Set(items.map(item => item.id)).size === items.length, 'Duplicate shadow template IDs').optional(), +}).strict().refine(value => (value.planTemplates ?? []).every(template => template.maximumPoints <= value.storagePolicy.maximumPoints + && template.maximumBytes <= value.storagePolicy.maximumBytes), 'Shadow templates exceed deployment storage policy') + +/** Environment configuration is an operator-owned resource policy, not evidence + * of qualification. Every execution still verifies the installed qualification + * registry, current build, acknowledged bindings and observed pod health. */ +export function configuredShadowService() { + const raw = process.env.SLEEPYPOD_BIOMETRICS_SHADOW_SERVICE + if (raw === undefined) return null + if (Buffer.byteLength(raw) > 16384) throw new Error('Shadow service configuration too large') + return shadowServiceConfigurationSchema.parse(JSON.parse(raw)) +} + +/** Resolve databases from the running app, never from an API/request or an + * independently configured database URL. Missing configuration leaves no timer, + * database connection, Python probe or sensor subscription behind. */ +export function startConfiguredShadowService(configuration: BetterSQLite3Database, + biometrics: BetterSQLite3Database) { + try { + const options = configuredShadowServiceOptions(configuration, biometrics) + return options ? startShadowService(options) : null + } + catch { + // Avoid exposing deployment paths or arbitrary environment content in logs. + console.warn('[biometrics] Shadow service unavailable; check deployment configuration and installed package') + return null + } +} + +/** Resolve server-owned paths for submission without starting background work. */ +export function configuredShadowServiceOptions(configuration: BetterSQLite3Database, + biometrics: BetterSQLite3Database) { + const options = configuredShadowService() + if (!options) return null + const configurationDatabase = absolutePath.parse(configuration.all<{ name: string, file: string }>(sql`PRAGMA database_list`).find(row => row.name === 'main')?.file) + const biometricsDatabase = absolutePath.parse(biometrics.all<{ name: string, file: string }>(sql`PRAGMA database_list`).find(row => row.name === 'main')?.file) + if (configurationDatabase === biometricsDatabase) throw new Error('Shadow databases must be separate') + return { installation: { workerPath: packagedShadowHostPath(), + pythonPath: options.pythonPath, modulesPath: options.modulesPath, presenceFile: options.presenceFile, + configurationDatabase, biometricsDatabase }, storagePolicy: options.storagePolicy, + busyTimeoutMs: options.busyTimeoutMs, recoveryIntervalMs: options.recoveryIntervalMs, + recoveryMaximumRuns: options.recoveryMaximumRuns } +} diff --git a/src/biometrics/shadow/store.ts b/src/biometrics/shadow/store.ts new file mode 100644 index 000000000..aff965682 --- /dev/null +++ b/src/biometrics/shadow/store.ts @@ -0,0 +1,357 @@ +import { shadowReceiptEnvelopeSchema, shadowPartialEnvelopeSchema, validateShadowPartialReceipt, validateShadowWorkerReceipt } from './receipt' +import { shadowPlanSchema } from './plan-contracts' +export { shadowPlanSchema, shadowMonitoringSchema, shadowExpectedBindingsSchema } from './plan-contracts' +import type Database from 'better-sqlite3' +import { randomUUID } from 'node:crypto' +import { z } from 'zod' +import { canonicalJson, dataHash } from '../algorithms/canonical' + +const integer = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) +const positive = integer.positive() +const hash = z.string().regex(/^[a-f0-9]{64}$/) +const metric = z.enum(['heart_rate', 'breathing_rate', 'window_ibi_variability_v1']) +const reading = z.object({ value: z.number().finite().nullable(), quality: z.number().min(0).max(1) }).strict() +export const shadowActiveProvenanceSchema = z.object({ processingRunId: z.string().uuid(), sourceSide: z.enum(['left', 'right']), + generation: integer, manifestHash: hash }).strict() +export const shadowSampleSchema = z.object({ sequence: integer, side: z.enum(['left', 'right']), timestampMs: integer, inputHash: hash.nullable(), + candidateConfigHash: hash, activeConfigHash: hash, + readings: z.array(z.object({ metric, candidate: reading, active: reading }).strict()).min(1).max(3), + pairing: z.object({ policy: z.literal('exact_timestamp_union_v1'), candidatePresent: z.boolean(), activePresent: z.boolean() }).strict().optional(), + inputIdentityKind: z.enum(['candidate_prefix', 'active_provenance', 'unavailable']).optional(), + activeProvenance: shadowActiveProvenanceSchema.optional(), + candidateProvenance: z.object({ sourceSide: z.enum(['left', 'right']), outputSequence: integer, + flags: z.array(z.string().max(200)).max(32).nullable(), limitations: z.array(z.string().max(200)).max(33) }).strict().optional(), +}).strict() +export const shadowStoragePolicySchema = z.object({ maximumRuns: positive, maximumPoints: positive, maximumBytes: positive }).strict() +type Plan = z.infer +interface RunRow { + worker_receipt: string | null + worker_receipt_hash: string | null + id: string + plan_hash: string + plan: string + side: 'left' | 'right' + state: 'pending' | 'running' | 'completed' | 'stopped' | 'failed' + attempt_id: string | null + lease_until_ms: number | null + heartbeat_at_ms: number | null + created_at_ms: number + ended_at_ms: number | null + reason: string | null + point_count: number + byte_count: number +} +export interface ShadowLease { runId: string, attemptId: string } + +/** Storage only: accepts observations from a separately admitted shipped + * processor. No algorithm imports, canonical writes, activation or hardware. + * The host must verify qualification/build and enforce CPU/RSS before claim. */ +export class ShadowStore { + private readonly policy: z.infer | null + constructor(private readonly db: Database.Database, policy?: unknown, private readonly now = Date.now) { + this.policy = policy === undefined ? null : shadowStoragePolicySchema.parse(policy) + } + + private executionPolicy() { + if (!this.policy) throw new Error('Shadow execution storage policy required') + return this.policy + } + + list(options: { side: 'left' | 'right', maximumRuns: number, afterId?: string }) { + const input = z.object({ side: z.enum(['left', 'right']), maximumRuns: positive.max(50), afterId: z.string().uuid().optional() }).strict().parse(options) + return this.db.transaction(() => { + const rows = this.db.prepare('SELECT id FROM biometric_shadow_runs WHERE side = ? AND id > ? ORDER BY id LIMIT ?') + .all(input.side, input.afterId ?? '', input.maximumRuns + 1) as { id: string }[] + const items = rows.slice(0, input.maximumRuns).map(row => this.get(row.id)) + return { items, nextCursor: rows.length > input.maximumRuns ? items.at(-1)?.id ?? null : null } + }).deferred() + } + + get(id: string) { + const row = this.db.prepare('SELECT * FROM biometric_shadow_runs WHERE id = ?').get(id) as RunRow | undefined + if (!row) throw new Error('Unknown shadow run') + const plan = shadowPlanSchema.parse(JSON.parse(row.plan)) + if (dataHash(plan) !== row.plan_hash || row.side !== plan.side) throw new Error('Shadow plan integrity mismatch') + let workerReceipt = null + if (row.worker_receipt !== null) { + if (Buffer.byteLength(row.worker_receipt) > 16384) throw new Error('Shadow receipt exceeds storage limit') + workerReceipt = shadowReceiptEnvelopeSchema.parse(JSON.parse(row.worker_receipt)) + if (!plan.execution || dataHash(workerReceipt) !== row.worker_receipt_hash || workerReceipt.runId !== row.id + || workerReceipt.attemptId !== row.attempt_id || workerReceipt.planHash !== row.plan_hash) throw new Error('Shadow receipt integrity mismatch') + if (workerReceipt.schema === 'biometric-shadow-receipt-v1') { + if (row.state !== 'completed' || workerReceipt.recordedAtMs !== row.ended_at_ms) throw new Error('Shadow receipt integrity mismatch') + validateShadowWorkerReceipt(workerReceipt.receipt, plan.execution) + } + else { + if (!['failed', 'stopped'].includes(row.state) || row.ended_at_ms === null || workerReceipt.recordedAtMs < row.ended_at_ms) throw new Error('Shadow receipt integrity mismatch') + validateShadowPartialReceipt(workerReceipt.receipt, plan.execution) + } + } + else if (row.worker_receipt_hash !== null) throw new Error('Shadow receipt integrity mismatch') + return { ...row, plan, workerReceipt } + } + + /** Complete bounded snapshot for scoring. Refuse oversize data rather than + * silently scoring a paginated/downsampled subset. */ + snapshot(id: string, limits: { maximumPoints: number, maximumBytes: number }) { + z.object({ maximumPoints: positive.max(100000), maximumBytes: positive.max(32 * 1024 * 1024) }).strict().parse(limits) + return this.db.transaction(() => { + const run = this.get(id) + const totals = this.db.prepare(`SELECT COUNT(*) AS points, COALESCE(SUM(byte_length), 0) AS bytes, + COALESCE(SUM(length(CAST(payload AS BLOB))), 0) AS actual_bytes FROM biometric_shadow_samples WHERE run_id = ?`).get(id) as { points: number, bytes: number, actual_bytes: number } + if (totals.points !== run.point_count || totals.bytes !== run.byte_count || totals.bytes !== totals.actual_bytes) throw new Error('Shadow sample totals integrity mismatch') + if (totals.points > limits.maximumPoints || totals.actual_bytes > limits.maximumBytes) throw new Error('Shadow report input limit exceeded') + const rows = this.db.prepare('SELECT sequence, timestamp_ms, content_hash, payload, byte_length FROM biometric_shadow_samples WHERE run_id = ? ORDER BY sequence').all(id) as { sequence: number, timestamp_ms: number, content_hash: string, payload: string, byte_length: number }[] + const capturedAtMs = this.now() + let previousTime = -1 + const samples = rows.map((row, index) => { + const envelope = shadowSampleSchema.extend({ attemptId: z.string().uuid() }).parse(JSON.parse(row.payload)) + const { attemptId, ...sample } = envelope + if (dataHash(sample) !== row.content_hash || row.byte_length !== Buffer.byteLength(row.payload) + || row.sequence !== index || sample.sequence !== index || sample.timestampMs !== row.timestamp_ms + || sample.timestampMs <= previousTime || sample.timestampMs < run.plan.startMs || sample.timestampMs >= run.plan.endMs + || sample.side !== run.side || sample.candidateConfigHash !== run.plan.candidateConfigHash || sample.activeConfigHash !== run.plan.activeConfigHash + || sample.readings.length !== run.plan.metrics.length || new Set(sample.readings.map(row => row.metric)).size !== run.plan.metrics.length + || sample.readings.some(row => !run.plan.metrics.includes(row.metric))) throw new Error('Shadow sample integrity mismatch') + previousTime = sample.timestampMs + return { ...sample, attemptId } + }) + const snapshot = { schema: 'biometric-shadow-snapshot-v1' as const, run, capturedAtMs, samples } + return { ...snapshot, snapshotHash: dataHash(snapshot) } + }).deferred() + } + + cursor(id: string) { + return this.db.transaction(() => { + const run = this.get(id) + const last = this.db.prepare('SELECT sequence, timestamp_ms FROM biometric_shadow_samples WHERE run_id = ? ORDER BY sequence DESC LIMIT 1').get(id) as { sequence: number, timestamp_ms: number } | undefined + if ((last ? last.sequence + 1 : 0) !== run.point_count) throw new Error('Shadow cursor integrity mismatch') + return { nextSequence: run.point_count, lastTimestampMs: last?.timestamp_ms ?? null } + }).deferred() + } + + findCreated(raw: unknown, idempotencyKey: string) { + const plan = shadowPlanSchema.parse(raw) + z.string().min(1).max(200).parse(idempotencyKey) + const previous = this.db.prepare('SELECT id, plan_hash FROM biometric_shadow_runs WHERE idempotency_key = ?').get(idempotencyKey) as { id: string, plan_hash: string } | undefined + if (!previous) return null + if (previous.plan_hash !== dataHash(plan)) throw new Error('Shadow idempotency conflict') + return this.get(previous.id) + } + + create(raw: unknown, idempotencyKey: string, admit?: (plan: Plan) => void) { + const policy = this.executionPolicy() + const plan = shadowPlanSchema.parse(raw), planHash = dataHash(plan) + z.string().min(1).max(200).parse(idempotencyKey) + return this.db.transaction(() => { + const previous = this.findCreated(plan, idempotencyKey) + if (previous) { + return previous + } + const now = this.now() + if (plan.startMs < now) throw new Error('Shadow window must be prospective') + if (plan.maximumPoints > policy.maximumPoints || plan.maximumBytes > policy.maximumBytes) throw new Error('Shadow plan exceeds storage policy') + const count = (this.db.prepare('SELECT COUNT(*) AS count FROM biometric_shadow_runs').get() as { count: number }).count + if (count >= policy.maximumRuns) throw new Error('Shadow run quota exceeded') + const admitted: unknown = admit?.(structuredClone(plan)) + if (admitted !== undefined) { + void Promise.resolve(admitted).catch(() => {}) + throw new Error('Shadow admission must complete synchronously') + } + if (plan.startMs < this.now()) throw new Error('Shadow window must be prospective') + const id = randomUUID() + this.db.prepare(`INSERT INTO biometric_shadow_runs + (id, idempotency_key, plan_hash, plan, side, state, created_at_ms) VALUES (?, ?, ?, ?, ?, 'pending', ?)`).run(id, idempotencyKey, planHash, canonicalJson(plan), plan.side, now) + return this.get(id) + }).immediate() + } + + /** Read-only bounded queue scan. It never resumes a running attempt after a + * restart: recovery must revoke that authority, retaining incomplete evidence. + * The caller still has to claim and revalidate the selected plan. */ + pending(options: { maximumRuns: number, afterId?: string }) { + const input = z.object({ maximumRuns: positive.max(100), afterId: z.string().uuid().optional() }).strict().parse(options) + return this.db.transaction(() => { + const now = integer.parse(this.now()) + const rows = this.db.prepare('SELECT id, state FROM biometric_shadow_runs WHERE id > ? ORDER BY id LIMIT ?') + .all(input.afterId ?? '', input.maximumRuns + 1) as { id: string, state: RunRow['state'] }[] + const batch = rows.slice(0, input.maximumRuns) + for (const row of batch) { + if (row.state !== 'pending') continue + const run = this.get(row.id) + if (run.attempt_id !== null || run.lease_until_ms !== null) throw new Error('Invalid pending shadow authority') + if (!run.plan.execution || !run.plan.expectedBindings || run.plan.candidateRevisionId === undefined || now < run.plan.startMs || now >= run.plan.endMs) continue + return { run: { runId: run.id, revisionId: run.plan.candidateRevisionId }, nextCursor: row.id } + } + return { run: null, nextCursor: rows.length > input.maximumRuns ? batch.at(-1)?.id ?? null : null } + }).deferred() + } + + /** Admission is a trusted synchronous host check, never a client boolean. */ + claim(id: string, leaseMs: number, admit: (plan: Plan) => void): ShadowLease { + this.executionPolicy() + z.number().int().min(1000).max(3600000).parse(leaseMs) + return this.db.transaction(() => { + const run = this.get(id), now = this.now() + if (!['pending', 'running'].includes(run.state) || (run.state === 'running' && (run.lease_until_ms ?? 0) > now)) throw new Error('Shadow run unavailable') + if (now < run.plan.startMs || now >= run.plan.endMs) throw new Error('Outside shadow window') + if (this.db.prepare(`SELECT id FROM biometric_shadow_runs WHERE side = ? AND state = 'running' AND lease_until_ms > ? AND id != ?`).get(run.side, now, id)) throw new Error('Side already has an active shadow worker') + // Pass a copy: an admission callback cannot mutate the frozen plan. + const admitted: unknown = admit(structuredClone(run.plan)) + if (admitted !== undefined) { + void Promise.resolve(admitted).catch(() => undefined) + throw new Error('Shadow admission must complete synchronously') + } + const attemptId = randomUUID() + this.db.prepare('UPDATE biometric_shadow_runs SET state = \'running\', attempt_id = ?, lease_until_ms = ?, heartbeat_at_ms = ? WHERE id = ?') + .run(attemptId, Math.min(run.plan.endMs + (run.plan.joining?.drainGraceMs ?? 0), now + leaseMs), now, id) + return { runId: id, attemptId } + }).immediate() + } + + private current(lease: ShadowLease) { + const run = this.get(lease.runId), now = this.now() + if (run.state !== 'running' || run.attempt_id !== lease.attemptId || (run.lease_until_ms ?? 0) <= now || now >= run.plan.endMs + (run.plan.joining?.drainGraceMs ?? 0)) throw new Error('Stale shadow lease') + return { run, now } + } + + append(lease: ShadowLease, raw: unknown) { + const policy = this.executionPolicy() + const sample = shadowSampleSchema.parse(raw), contentHash = dataHash(sample) + return this.db.transaction(() => { + const { run, now } = this.current(lease) + if (sample.side !== run.side || sample.candidateConfigHash !== run.plan.candidateConfigHash || sample.activeConfigHash !== run.plan.activeConfigHash + || sample.timestampMs < run.plan.startMs || sample.timestampMs >= run.plan.endMs || sample.timestampMs > now + || sample.readings.length !== run.plan.metrics.length || new Set(sample.readings.map(row => row.metric)).size !== run.plan.metrics.length + || sample.readings.some(row => !run.plan.metrics.includes(row.metric))) throw new Error('Shadow sample differs from frozen plan') + const previous = this.db.prepare('SELECT content_hash FROM biometric_shadow_samples WHERE run_id = ? AND sequence = ?').get(run.id, sample.sequence) as { content_hash: string } | undefined + if (previous) { + if (previous.content_hash !== contentHash) throw new Error('Shadow sample idempotency conflict') + return { state: 'already_recorded' as const } + } + const last = this.db.prepare('SELECT sequence, timestamp_ms FROM biometric_shadow_samples WHERE run_id = ? ORDER BY sequence DESC LIMIT 1').get(run.id) as { sequence: number, timestamp_ms: number } | undefined + if (sample.sequence !== (last ? last.sequence + 1 : 0) || (last && sample.timestampMs <= last.timestamp_ms)) throw new Error('Out-of-order shadow sample') + const payload = canonicalJson({ ...sample, attemptId: lease.attemptId }), bytes = Buffer.byteLength(payload) + const totals = this.db.prepare('SELECT COALESCE(SUM(point_count), 0) AS points, COALESCE(SUM(byte_count), 0) AS bytes FROM biometric_shadow_runs').get() as { points: number, bytes: number } + if (run.point_count + 1 > run.plan.maximumPoints || run.byte_count + bytes > run.plan.maximumBytes + || totals.points + 1 > policy.maximumPoints || totals.bytes + bytes > policy.maximumBytes) { + this.db.prepare('UPDATE biometric_shadow_runs SET state = \'stopped\', reason = \'storage_quota\', ended_at_ms = ?, lease_until_ms = NULL WHERE id = ?').run(now, run.id) + return { state: 'quota_reached' as const } + } + this.db.prepare('INSERT INTO biometric_shadow_samples VALUES (?, ?, ?, ?, ?, ?)').run(run.id, sample.sequence, sample.timestampMs, contentHash, payload, bytes) + this.db.prepare('UPDATE biometric_shadow_runs SET point_count = point_count + 1, byte_count = byte_count + ?, heartbeat_at_ms = ? WHERE id = ?').run(bytes, now, run.id) + return { state: 'recorded' as const } + }).immediate() + } + + /** Host/operator cancellation also works after a worker has disappeared or + * the observation window has ended. Terminal evidence is preserved. */ + stop(id: string, reason: string) { + z.string().min(1).max(500).parse(reason) + return this.db.transaction(() => { + this.get(id) + this.db.prepare('UPDATE biometric_shadow_runs SET state = \'stopped\', reason = ?, ended_at_ms = ?, lease_until_ms = NULL WHERE id = ? AND state IN (\'pending\', \'running\')') + .run(reason, this.now(), id) + return this.get(id) + }).immediate() + } + + renew(lease: ShadowLease, leaseMs: number) { + z.number().int().min(1000).max(3600000).parse(leaseMs) + this.db.transaction(() => { + const { run, now } = this.current(lease) + this.db.prepare('UPDATE biometric_shadow_runs SET lease_until_ms = ?, heartbeat_at_ms = ? WHERE id = ?').run(Math.min(run.plan.endMs + (run.plan.joining?.drainGraceMs ?? 0), now + leaseMs), now, run.id) + }).immediate() + } + + /** Bounded keyset maintenance. Revoke expired authority, not a claim that an + * OS process was reaped. A resumed worker cannot append/renew after recovery; + * the kernel execution slot still excludes overlap until that worker exits. */ + reconcileExpired(options: { maximumRuns: number, afterId?: string }) { + const input = z.object({ maximumRuns: positive.max(100), afterId: z.string().uuid().optional() }).strict().parse(options) + return this.db.transaction(() => { + const now = integer.parse(this.now()) + const rows = this.db.prepare('SELECT id, state FROM biometric_shadow_runs WHERE id > ? ORDER BY id LIMIT ?') + .all(input.afterId ?? '', input.maximumRuns + 1) as { id: string, state: RunRow['state'] }[] + const batch = rows.slice(0, input.maximumRuns) + const recovered: { runId: string, attemptId: string | null, reason: string }[] = [] + for (const row of batch) { + if (row.state !== 'pending' && row.state !== 'running') continue + const run = this.get(row.id) + let reason: string + if (run.state === 'pending') { + if (now < run.plan.endMs) continue + if (run.attempt_id !== null || run.lease_until_ms !== null) throw new Error('Invalid pending shadow authority') + reason = 'shadow_observation_window_missed' + } + else { + z.string().uuid().parse(run.attempt_id) + integer.parse(run.lease_until_ms) + if ((run.lease_until_ms ?? 0) > now && now < run.plan.endMs + (run.plan.joining?.drainGraceMs ?? 0)) continue + reason = 'shadow_lease_expired_cleanup_unconfirmed' + } + const updated = this.db.prepare(`UPDATE biometric_shadow_runs SET state = 'failed', reason = ?, ended_at_ms = ?, lease_until_ms = NULL + WHERE id = ? AND state = ? AND attempt_id IS ? AND lease_until_ms IS ?`) + .run(reason, now, run.id, run.state, run.attempt_id, run.lease_until_ms) + if (updated.changes !== 1) throw new Error('Shadow recovery authority changed') + recovered.push({ runId: run.id, attemptId: run.attempt_id, reason }) + } + return { scanned: batch.length, recovered, nextCursor: rows.length > input.maximumRuns ? batch.at(-1)?.id ?? null : null } + }).immediate() + } + + /** After the host has reaped its worker, record an expired attempt's failure. + * A newer attempt or an already terminal run is never changed. */ + failExpired(lease: ShadowLease, reason: string) { + z.string().min(1).max(500).parse(reason) + return this.db.transaction(() => { + const run = this.get(lease.runId), now = this.now() + if (run.state !== 'running' || run.attempt_id !== lease.attemptId + || ((run.lease_until_ms ?? 0) > now && now < run.plan.endMs + (run.plan.joining?.drainGraceMs ?? 0))) return false + this.db.prepare('UPDATE biometric_shadow_runs SET state = \'failed\', reason = ?, ended_at_ms = ?, lease_until_ms = NULL WHERE id = ?') + .run(reason, now, run.id) + return true + }).immediate() + } + + /** Attach cleanup evidence without changing terminal authority, even when an + * operator or recovery already ended this exact attempt. Never replace evidence. */ + recordPartialReceipt(lease: ShadowLease, raw: unknown) { + return this.db.transaction(() => { + const run = this.get(lease.runId), now = this.now() + if (run.attempt_id !== lease.attemptId || !['failed', 'stopped'].includes(run.state)) return false + if (!run.plan.execution || run.ended_at_ms === null || now < run.ended_at_ms) throw new Error('Shadow partial receipt requires a terminal executable attempt') + const receipt = validateShadowPartialReceipt(raw, run.plan.execution) + if (run.workerReceipt) { + if (run.workerReceipt.schema === 'biometric-shadow-partial-receipt-v1' && dataHash(run.workerReceipt.receipt) === dataHash(receipt)) return true + throw new Error('Shadow receipt already recorded') + } + const envelope = shadowPartialEnvelopeSchema.parse({ schema: 'biometric-shadow-partial-receipt-v1', runId: run.id, + attemptId: lease.attemptId, planHash: run.plan_hash, recordedAtMs: now, receipt }) + const encoded = canonicalJson(envelope) + if (Buffer.byteLength(encoded) > 16384) throw new Error('Shadow receipt exceeds storage limit') + this.db.prepare('UPDATE biometric_shadow_runs SET worker_receipt = ?, worker_receipt_hash = ? WHERE id = ?') + .run(encoded, dataHash(envelope), run.id) + return true + }).immediate() + } + + finish(lease: ShadowLease, state: 'completed' | 'stopped' | 'failed', reason: string, rawReceipt?: unknown) { + z.enum(['completed', 'stopped', 'failed']).parse(state) + z.string().min(1).max(500).parse(reason) + this.db.transaction(() => { + const { run, now } = this.current(lease) + let receipt = null, receiptHash = null + if (state === 'completed' && run.plan.execution) { + receipt = canonicalJson(shadowReceiptEnvelopeSchema.parse({ schema: 'biometric-shadow-receipt-v1', + runId: run.id, attemptId: lease.attemptId, planHash: run.plan_hash, recordedAtMs: now, + receipt: validateShadowWorkerReceipt(rawReceipt, run.plan.execution) })) + if (Buffer.byteLength(receipt) > 16384) throw new Error('Shadow receipt exceeds storage limit') + receiptHash = dataHash(JSON.parse(receipt)) + } + else if (rawReceipt !== undefined) throw new Error('Worker receipt requires a completed executable plan') + this.db.prepare('UPDATE biometric_shadow_runs SET state = ?, reason = ?, ended_at_ms = ?, lease_until_ms = NULL, worker_receipt = ?, worker_receipt_hash = ? WHERE id = ?') + .run(state, reason, now, receipt, receiptHash, run.id) + }).immediate() + } +} diff --git a/src/biometrics/shadow/stream-join.ts b/src/biometrics/shadow/stream-join.ts new file mode 100644 index 000000000..e1534295c --- /dev/null +++ b/src/biometrics/shadow/stream-join.ts @@ -0,0 +1,105 @@ +import { dataHash } from '../algorithms/canonical' +import type { ShadowLease, ShadowStore } from './store' +import { shadowWorkerOutputSchema } from './worker-protocol' +import { shadowActiveReadingSchema } from './worker-sink' +import type { z } from 'zod' + +type Candidate = z.infer +type Active = z.infer +interface Pair { candidate?: Candidate, active?: Active } + +/** Exact timestamp union, with bounded buffering and a frozen lateness policy. + * The supervisor advances the clock and validates worker batches beforehand. + * No nearby timestamp is relabelled to make a match. */ +export class ShadowStreamJoin { + private readonly plan: ReturnType['plan'] + private readonly policy: NonNullable['plan']['joining']> + private readonly pending = new Map() + private sealedThrough: number + private lastClock = -1 + private nextSequence: number + private closed = false + + constructor(private readonly store: ShadowStore, private readonly lease: ShadowLease) { + const run = store.get(lease.runId) + if (run.state !== 'running' || run.attempt_id !== lease.attemptId || !run.plan.joining) throw new Error('Shadow join requires a live attempt and frozen join policy') + this.plan = structuredClone(run.plan) + this.policy = { ...run.plan.joining } + const cursor = store.cursor(lease.runId) + this.nextSequence = cursor.nextSequence + this.sealedThrough = cursor.lastTimestampMs ?? run.plan.startMs - 1 + } + + private fail(reason: string): never { + this.closed = true + // This is a worker failure, not an operator stop. Never stop a replacement. + try { + this.store.finish(this.lease, 'failed', reason) + } + catch { /* A lost lease already fences this attempt. */ } + throw new Error(reason) + } + + private push(kind: 'candidate' | 'active', row: Candidate | Active) { + if (this.closed) throw new Error('Shadow join is closed') + if (row.side !== this.plan.side || row.configHash !== (kind === 'candidate' ? this.plan.candidateConfigHash : this.plan.activeConfigHash) + || row.timestampMs < this.plan.startMs || row.timestampMs >= this.plan.endMs) return this.fail('join_identity_mismatch') + if (row.timestampMs <= this.sealedThrough) return this.fail('join_late_arrival') + for (const metric of this.plan.metrics) { + const reading = row.readings.find(reading => reading.metric === metric) + if (!reading || reading.windowMs !== (metric === 'window_ibi_variability_v1' ? this.plan.windowIbiWindowMs : null)) return this.fail('join_metric_mismatch') + } + const previous = this.pending.get(row.timestampMs) + if (previous?.[kind]) { + if (dataHash(previous[kind]) !== dataHash(row)) return this.fail('join_conflicting_duplicate') + return + } + if (!previous && this.pending.size >= this.policy.maximumPendingPoints) return this.fail('join_queue_quota') + this.pending.set(row.timestampMs, { ...previous, [kind]: row }) + } + + candidate(raw: unknown) { this.push('candidate', shadowWorkerOutputSchema.parse(raw)) } + + active(raw: unknown) { this.push('active', shadowActiveReadingSchema.parse(raw)) } + + advance(nowMs: number) { + if (this.closed) throw new Error('Shadow join is closed') + if (!Number.isSafeInteger(nowMs) || nowMs < this.plan.startMs || nowMs < this.lastClock) return this.fail('join_clock_regression') + this.lastClock = nowMs + const through = Math.min(this.plan.endMs - 1, nowMs - this.policy.maximumLatenessMs) + let emitted = 0 + for (const timestampMs of [...this.pending.keys()].filter(time => time <= through).sort((a, b) => a - b)) { + const pair = this.pending.get(timestampMs) + if (!pair) throw new Error('Missing shadow join point') + const { candidate, active } = pair + const readings = this.plan.metrics.map((metric) => { + const next = candidate?.readings.find(reading => reading.metric === metric) + const previous = active?.readings.find(reading => reading.metric === metric) + return { metric, candidate: next ? { value: next.value, quality: next.quality } : { value: null, quality: 0 }, + active: previous ? { value: previous.value, quality: previous.quality } : { value: null, quality: 0 } } + }) + const inputHash = candidate?.inputHash ?? active?.inputHash ?? null + const result = this.store.append(this.lease, { sequence: this.nextSequence, timestampMs, side: this.plan.side, + candidateConfigHash: this.plan.candidateConfigHash, activeConfigHash: this.plan.activeConfigHash, inputHash, + inputIdentityKind: candidate ? 'candidate_prefix' : inputHash ? 'active_provenance' : 'unavailable', + ...(active?.provenance ? { activeProvenance: active.provenance } : {}), + pairing: { policy: 'exact_timestamp_union_v1', candidatePresent: Boolean(candidate), activePresent: Boolean(active) }, readings, + ...(candidate + ? { candidateProvenance: { sourceSide: candidate.sourceSide, outputSequence: candidate.sequence, flags: candidate.flags, + limitations: active ? candidate.limitations : [...candidate.limitations, 'missing_contemporaneous_active_reading'] } } + : {}) }) + if (result.state === 'quota_reached') { + this.closed = true + return { emitted, stopped: true } + } + // Keep the point and sequence unchanged until storage acknowledges it; + // this permits an exact retry after an uncertain committed write. + this.pending.delete(timestampMs) + this.nextSequence++ + this.sealedThrough = timestampMs + emitted++ + } + this.sealedThrough = Math.max(this.sealedThrough, through) + return { emitted, stopped: false } + } +} diff --git a/src/biometrics/shadow/submission-contracts.ts b/src/biometrics/shadow/submission-contracts.ts new file mode 100644 index 000000000..e43a840cb --- /dev/null +++ b/src/biometrics/shadow/submission-contracts.ts @@ -0,0 +1,10 @@ +import { z } from 'zod' +import { shadowPlanSchema, shadowExpectedBindingsSchema } from './plan-contracts' +import { shadowExecutionPolicySchema } from './execution-policy' + +export const submitShadowSchema = z.object({ + plan: shadowPlanSchema.safeExtend({ candidateRevisionId: z.string().uuid().nullable(), + expectedBindings: shadowExpectedBindingsSchema, execution: shadowExecutionPolicySchema }), + idempotencyKey: z.string().min(1).max(200), +}).strict() + diff --git a/src/biometrics/shadow/submission.ts b/src/biometrics/shadow/submission.ts new file mode 100644 index 000000000..0a34dd128 --- /dev/null +++ b/src/biometrics/shadow/submission.ts @@ -0,0 +1,57 @@ +import { z } from 'zod' +import { dataHash } from '../algorithms/canonical' +import { submitShadowSchema } from './submission-contracts' +import { loadInstalledShadowHost, inspectInstalledShadowHost } from './installed-host' +import { openShadowStateDatabase } from './active-database' +import { installedShadowIdentity } from './device-identity' +import { getSensorFrameTimes } from '@/src/streaming/piezoStream' +import type { runInstalledShadowAttempt } from './installed-attempt' + +export { submitShadowSchema } from './submission-contracts' + +/** Assertions in a proposed plan are checked against installed qualification and + * fresh observations. A request cannot supply paths, resolved configs or health. + * Exact retries return durable evidence before any new Python/health work. */ +export async function submitInstalledShadow(raw: z.input, + installation: Parameters[2], storagePolicy: unknown, busyTimeoutMs: number, signal?: AbortSignal) { + const input = submitShadowSchema.parse(raw), paths = structuredClone(installation), policy = structuredClone(storagePolicy) + z.number().int().min(1).max(1000).parse(busyTimeoutMs) + if (busyTimeoutMs > input.plan.execution.sources.databaseBusyTimeoutMs) throw new Error('Shadow store timeout exceeds saved execution policy') + if (signal?.aborted) throw new Error('Shadow submission cancelled') + const loaded = loadInstalledShadowHost(paths.workerPath) + loaded.assertCurrent() + const database = openShadowStateDatabase(paths.biometricsDatabase, { busyTimeoutMs }) + let owned: Awaited> | undefined + try { + database.assertCurrent() + const store = new loaded.api.ShadowStore(database.db, policy) + const previous = store.findCreated(input.plan, input.idempotencyKey) + if (previous) return previous + const inspected = await inspectInstalledShadowHost(paths.workerPath, paths, signal) + if (signal?.aborted) throw new Error('Shadow submission cancelled') + inspected.assertCurrent() + database.assertCurrent() + const entries = inspected.api.readInstalledShadowQualifications(paths.workerPath).entries.filter(entry => dataHash(entry) === input.plan.qualificationHash) + if (entries.length !== 1) throw new Error('Shadow qualification missing or ambiguous') + const observed = installedShadowIdentity(inspected, entries[0]) + const identity = () => { + database.assertCurrent() + return observed() + } + owned = await inspected.api.openProfiledShadowHost(input.plan, { ...paths, identity, + piezoReceivedAt: () => getSensorFrameTimes()['piezo-dual'] }, input.plan.candidateRevisionId, signal) + if (signal?.aborted) throw new Error('Shadow submission cancelled') + const host = owned.host + return store.create(input.plan, input.idempotencyKey, (plan) => { + inspected.assertCurrent() + database.assertCurrent() + host.admit(plan) + }) + } + finally { + try { + owned?.close() + } + finally { database.close() } + } +} diff --git a/src/biometrics/shadow/tests/active-reader.test.ts b/src/biometrics/shadow/tests/active-reader.test.ts new file mode 100644 index 000000000..1cd55b6e8 --- /dev/null +++ b/src/biometrics/shadow/tests/active-reader.test.ts @@ -0,0 +1,350 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { randomUUID } from 'node:crypto' +import { mkdtempSync, rmSync, renameSync, copyFileSync, symlinkSync, writeFileSync, utimesSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join, dirname } from 'node:path' +import { fileURLToPath } from 'node:url' +import { afterEach, expect, it, vi } from 'vitest' +import { biometricManifest, defaultConfig, manifestHash } from '../../algorithms/registry' +import { configHash } from '../../algorithms/canonical' +import { plannedShadowSourceFactory } from '../planned-sources' +import { shadowPlanSchema } from '../store' +import { openShadowActiveDatabase, openShadowStateDatabase } from '../active-database' +import { openShadowLiveSources } from '../live-sources' +import { publishProspectiveFrame } from '../../../streaming/prospectiveFrameTap' +import { ShadowActiveReader } from '../active-reader' +import { ShadowStore } from '../store' +import { ShadowStreamJoin } from '../stream-join' +const start = 1700000000000, hash = configHash(defaultConfig) +const cleanups: (() => void)[] = [] +afterEach(() => { + cleanups.splice(0).forEach(cleanup => cleanup()) + vi.useRealTimers() +}) +function setup(maximumRows = 10) { + const directory = mkdtempSync(join(tmpdir(), 'shadow-active-')), path = join(directory, 'biometrics.db') + const writer = new Database(path) + writer.pragma('journal_mode = WAL') + migrate(drizzle(writer), { migrationsFolder: fileURLToPath(new URL('../../../db/biometrics-migrations/', import.meta.url)) }) + const readonly = new Database(path, { readonly: true, fileMustExist: true }) + cleanups.push(() => { + readonly.close() + writer.close() + rmSync(directory, { recursive: true, force: true }) + }) + const runId = randomUUID() + const bundle = { schemaVersion: 1, manifestHash, resolved: defaultConfig, generation: 0, revisionId: null, + components: biometricManifest.components, sourceHashes: biometricManifest.sourceHashes } + writer.prepare('INSERT INTO biometric_processing_runs (id, side, kind, bundle, config_hash, calibration_hash, input_identity, started_at_ms) VALUES (?, ?, ?, ?, ?, NULL, ?, ?)') + .run(runId, 'left', 'live', JSON.stringify(bundle), hash, JSON.stringify({ sourceSide: 'right', timestampBasis: 'processing_clock', rawIdentity: 'not_retained', sensorCalibration: 'not_consumed' }), start) + const options = { side: 'left', startMs: start, endMs: start + 10000, activeConfigHash: hash, maximumRows } + let now = start + const reader = () => new ShadowActiveReader(readonly, options, () => now) + const insert = (seconds = 0, side = 'left', provenance: string | null = runId) => { + const id = Number(writer.prepare('INSERT INTO vitals (side,timestamp,heart_rate,hrv,breathing_rate,processing_run_id) VALUES (?,?,60,NULL,12,?)').run(side, start / 1000 + seconds, provenance).lastInsertRowid) + writer.prepare('INSERT INTO vitals_quality (vitals_id,side,timestamp,quality_score,created_at) VALUES (?,?,?,0.9,?)').run(id, side, start / 1000 + seconds, start / 1000 + seconds) + return id + } + return { writer, readonly, path, runId, bundle, options, reader, insert, advance: (milliseconds: number) => { + now += milliseconds + } } +} +it('captures only newly inserted rows with original values, quality and verified live provenance', () => { + const { reader, insert, advance, runId } = setup() + insert() + const capture = reader() + expect(capture.poll()).toEqual([]) + advance(1000) + insert(1) + const rows = capture.poll() + expect(rows).toHaveLength(1) + expect(rows[0]).toMatchObject({ timestampMs: start + 1000, configHash: hash, inputHash: null, + provenance: { processingRunId: runId, sourceSide: 'right', generation: 0, manifestHash }, + readings: [{ metric: 'heart_rate', value: 60, quality: 0.9 }, { metric: 'breathing_rate', value: 12 }, { metric: 'window_ibi_variability_v1', value: null, windowMs: 300000 }] }) + expect(capture.poll()).toEqual([]) +}) +it('requires read-only access and never writes through the capture connection', () => { + const { writer, readonly, options, reader, insert } = setup() + expect(() => new ShadowActiveReader(writer, options)).toThrow(/read-only/) + const capture = reader() + insert() + expect(capture.poll()).toHaveLength(1) + expect(() => readonly.prepare('DELETE FROM vitals').run()).toThrow(/readonly/) +}) +it('advances past other-side and out-of-window inserts without labelling them active', () => { + const { reader, insert } = setup() + const capture = reader() + insert(0, 'right') + insert(-1) + expect(capture.poll()).toEqual([]) + insert() + expect(capture.poll()).toHaveLength(1) +}) +it('refuses a batch above the poll quota without silently taking a prefix', () => { + const { reader, insert, advance } = setup(1) + const capture = reader() + insert() + advance(1000) + insert(1) + expect(() => capture.poll()).toThrow(/quota/) + expect(() => capture.poll()).toThrow(/quota/) +}) +it.each(['legacy', 'quality_missing', 'quality_duplicate', 'quality_side', 'quality_timestamp', 'quality_invalid', 'config', 'bundle', 'kind', 'source', 'future'])('fails closed for %s evidence', (mode) => { + const { reader, insert, writer, runId } = setup() + const capture = reader() + const id = insert(mode === 'future' ? 1 : 0, 'left', mode === 'legacy' ? null : runId) + if (mode === 'quality_missing') writer.prepare('DELETE FROM vitals_quality').run() + if (mode === 'quality_duplicate') writer.prepare('INSERT INTO vitals_quality (vitals_id,side,timestamp,quality_score,created_at) SELECT vitals_id,side,timestamp,quality_score,created_at FROM vitals_quality').run() + if (mode === 'quality_side') writer.prepare('UPDATE vitals_quality SET side=\'right\' WHERE vitals_id=?').run(id) + if (mode === 'quality_timestamp') writer.prepare('UPDATE vitals_quality SET timestamp=timestamp+1').run() + if (mode === 'quality_invalid') writer.prepare('UPDATE vitals_quality SET quality_score=2').run() + if (mode === 'config') writer.prepare('UPDATE biometric_processing_runs SET config_hash=?').run('b'.repeat(64)) + if (mode === 'bundle') writer.prepare('UPDATE biometric_processing_runs SET bundle=?').run('x'.repeat(65537)) + if (mode === 'kind') writer.prepare('UPDATE biometric_processing_runs SET kind=\'replay\'').run() + if (mode === 'source') writer.prepare('UPDATE biometric_processing_runs SET input_identity=?').run('{}') + expect(() => capture.poll()).toThrow() + expect(() => capture.poll()).toThrow() +}) +it('detects deletion or replacement of its cursor instead of resuming through a reset', () => { + const { reader, insert, writer } = setup() + insert() + const capture = reader() + writer.prepare('UPDATE vitals SET heart_rate=99').run() + expect(() => capture.poll()).toThrow(/cursor changed/) + writer.prepare('DELETE FROM vitals').run() + expect(() => capture.poll()).toThrow(/cursor changed/) +}) +it('retains active run provenance when the stream join persists an active-only point', () => { + const { reader, insert, writer, runId } = setup() + const capture = reader() + const store = new ShadowStore(writer, { maximumRuns: 10, maximumPoints: 10, maximumBytes: 100000 }, () => start) + const run = store.create({ schema: 'biometric-shadow-plan-v1', subjectId: 'test', side: 'left', candidateConfigHash: hash, activeConfigHash: hash, + qualificationHash: hash, engineBuildHash: hash, startMs: start, endMs: start + 1000, maximumPoints: 10, maximumBytes: 10000, metrics: ['heart_rate'], + joining: { maximumPendingPoints: 10, maximumLatenessMs: 1, drainGraceMs: 100 }, monitoring: { expectedIntervalMs: 1000, maximumGapMs: 1000, heartbeatStaleMs: 1000, + minimumCandidateCoverage: 0.8, maximumCoverageLoss: 0.1, minimumQuality: 0.8, alignment: { version: 'reference-alignment-v1', pointToleranceMs: 0, clockOffsetMs: 0, offsetSource: 'fixed' } } }, 'test') + const lease = store.claim(run.id, 1000, () => {}) + const stream = new ShadowStreamJoin(store, lease) + insert() + stream.active(capture.poll()[0]) + stream.advance(start + 1) + expect(store.snapshot(run.id, { maximumPoints: 10, maximumBytes: 10000 }).samples[0]).toMatchObject({ + inputHash: null, inputIdentityKind: 'unavailable', activeProvenance: { processingRunId: runId, sourceSide: 'right' }, pairing: { activePresent: true, candidatePresent: false } }) +}) +it.each(['manifestHash', 'components', 'sourceHashes', 'resolved'])('rejects substituted %s in a live run bundle', (field) => { + const { reader, insert, writer, bundle } = setup() + const capture = reader() + insert() + writer.prepare('UPDATE biometric_processing_runs SET bundle=?').run(JSON.stringify({ ...bundle, [field]: field === 'manifestHash' ? 'a'.repeat(64) : {} })) + expect(() => capture.poll()).toThrow() + writer.prepare('UPDATE biometric_processing_runs SET bundle=?').run(JSON.stringify(bundle)) + // Failed validation did not advance the cursor or hide this observation. + expect(capture.poll()).toHaveLength(1) +}) +function liveSetup() { + vi.useFakeTimers() + vi.setSystemTime(start) + const fixture = setup() + const controller = new AbortController(), active = vi.fn(), fail = vi.fn(), databaseCurrent = vi.fn() + const options = { db: fixture.readonly, side: 'left' as const, startMs: start, endMs: start + 10000, activeConfigHash: hash, + maximumActiveRows: 10, maximumLatenessMs: 100, pollMs: 50, + frameBudgets: { maximumFrames: 2, maximumBytes: 10000, maximumEventAgeMs: 1000, maximumFutureSkewMs: 0 }, + context: () => ({ activeSides: ['left'], leftOccupied: true, rightOccupied: false, pumpContextKnown: false }), + assertDatabaseCurrent: databaseCurrent, signal: controller.signal, active, fail } + const open = () => { + const source = openShadowLiveSources(options) + // Close subscribers before the owning database connections in afterEach. + cleanups.unshift(source.close) + return source + } + return { ...fixture, controller, active, fail, databaseCurrent, options, open } +} +it('composes real waveform capture and read-only active polling with an explicit final flush', async () => { + const { insert, open, active, fail } = liveSetup() + insert() + const source = open(), frames = source.frames[Symbol.asyncIterator]() + const frame = frames.next() + publishProspectiveFrame({ type: 'piezo-dual', ts: start / 1000, freq: 500, left1: [1, 2], right1: [3, 4] }, false, start) + expect((await frame).value).toMatchObject({ observedAtMs: start, context: { activeSides: ['left'] } }) + expect(active).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(1000) + insert(1) + source.flush?.() + expect(active).toHaveBeenCalledOnce() + expect(active.mock.calls[0][0]).toMatchObject({ timestampMs: start + 1000 }) + await vi.advanceTimersByTimeAsync(50) + expect(active).toHaveBeenCalledOnce() + expect(fail).not.toHaveBeenCalled() +}) +it('keeps active polling available during final drain after waveform collection ends', async () => { + const { insert, open, active, fail } = liveSetup() + const source = open(), frames = source.frames[Symbol.asyncIterator]() + const done = frames.next() + await vi.advanceTimersByTimeAsync(10000) + expect((await done).done).toBe(true) + insert(9) + source.flush?.() + expect(active.mock.calls[0][0]).toMatchObject({ timestampMs: start + 9000 }) + expect(fail).not.toHaveBeenCalled() +}) +it('closes both captures on a database identity failure without delivering old-connection data', async () => { + const { insert, open, active, fail, databaseCurrent } = liveSetup() + const source = open(), frames = source.frames[Symbol.asyncIterator]() + const pending = frames.next() + insert() + databaseCurrent.mockImplementation(() => { + throw new Error('database inode changed') + }) + await vi.advanceTimersByTimeAsync(50) + expect((await pending).done).toBe(true) + expect(active).not.toHaveBeenCalled() + expect(fail).toHaveBeenCalledOnce() + expect(vi.getTimerCount()).toBe(0) +}) +it('clears polling and subscriptions on abort and refuses asynchronous delivery', async () => { + const fixture = liveSetup() + const source = fixture.open() + fixture.controller.abort() + expect(vi.getTimerCount()).toBe(0) + expect(() => source.flush?.()).toThrow(/closed/) + expect(fixture.fail).not.toHaveBeenCalled() + const next = liveSetup() + next.active.mockImplementation(async () => {}) + next.open() + next.insert() + await vi.advanceTimersByTimeAsync(50) + expect(next.fail).toHaveBeenCalledOnce() + expect(vi.getTimerCount()).toBe(0) +}) +it('opens a bounded read-only active connection and detects path replacement', () => { + const { path } = setup() + const connection = openShadowActiveDatabase(path, { busyTimeoutMs: 50 }) + cleanups.unshift(connection.close) + expect(connection.db.readonly).toBe(true) + expect(connection.db.pragma('busy_timeout', { simple: true })).toBe(50) + connection.assertCurrent() + renameSync(path, path + '.old') + copyFileSync(path + '.old', path) + expect(() => connection.assertCurrent()).toThrow(/replaced/) + connection.close() + expect(() => connection.assertCurrent()).toThrow(/closed/) +}) +it('refuses missing files, symlinks and unbounded busy timeouts', () => { + const { path } = setup() + expect(() => openShadowActiveDatabase(path + '.missing', { busyTimeoutMs: 50 })).toThrow() + symlinkSync(path, path + '.link') + expect(() => openShadowActiveDatabase(path + '.link', { busyTimeoutMs: 50 })).toThrow(/regular file/) + expect(() => openShadowActiveDatabase(path, { busyTimeoutMs: 1001 })).toThrow() +}) +it('rechecks database identity after the snapshot and before active delivery', async () => { + const { insert, open, active, fail, databaseCurrent } = liveSetup() + databaseCurrent.mockImplementationOnce(() => {}).mockImplementationOnce(() => {}).mockImplementation(() => { + throw new Error('database replaced during snapshot') + }) + open() + insert() + await vi.advanceTimersByTimeAsync(50) + expect(active).not.toHaveBeenCalled() + expect(fail).toHaveBeenCalledOnce() +}) +function plannedSetup() { + vi.useFakeTimers() + vi.setSystemTime(start) + const fixture = setup() + const configPath = join(dirname(fixture.path), 'configuration.db'), presencePath = join(dirname(fixture.path), 'presence.json') + const config = new Database(configPath) + config.exec('CREATE TABLE side_settings(side TEXT, away_mode INTEGER); INSERT INTO side_settings VALUES(\'left\',0),(\'right\',1); CREATE TABLE device_settings(id INTEGER, bed_mode TEXT); INSERT INTO device_settings VALUES(1,\'two\');') + cleanups.unshift(() => config.close()) + writeFileSync(presencePath, JSON.stringify({ version: 1, left: { vitals_presence: true, vitals_evidence_ts: start / 1000 } })) + utimesSync(presencePath, start / 1000, start / 1000) + const plan = shadowPlanSchema.parse({ schema: 'biometric-shadow-plan-v1', subjectId: 'test', side: 'left', startMs: start, endMs: start + 10000, + candidateConfigHash: hash, activeConfigHash: hash, qualificationHash: hash, engineBuildHash: hash, maximumPoints: 100, maximumBytes: 100000, metrics: ['heart_rate'], + joining: { maximumPendingPoints: 10, maximumLatenessMs: 100, drainGraceMs: 500 }, + monitoring: { expectedIntervalMs: 1000, maximumGapMs: 1000, heartbeatStaleMs: 1000, minimumCandidateCoverage: 0.8, maximumCoverageLoss: 0.1, minimumQuality: 0.8, + alignment: { version: 'reference-alignment-v1', pointToleranceMs: 0, clockOffsetMs: 0, offsetSource: 'fixed' } }, + execution: { version: 'shadow-execution-policy-v1', configHashes: { left: hash, right: hash }, + maximumRecords: 10, maximumOutputs: 10, maximumCpuMs: 1000, maximumInputBytes: 100000, maximumOutputBytes: 100000, + budgets: { cpuSeconds: 2, addressSpaceBytes: 100000000, maximumRssBytes: 100000000, wallMs: 15000, acknowledgmentMs: 1000, idleMs: 1000, rssIntervalMs: 100, rssTimeoutMs: 100 }, + leaseMs: 1500, heartbeatMs: 200, sources: { pollMs: 50, maximumActiveRows: 10, databaseBusyTimeoutMs: 50, + frames: { maximumFrames: 1, maximumBytes: 10000, maximumEventAgeMs: 1000, maximumFutureSkewMs: 0 }, + context: { presenceStaleMs: 300000, pumpStaleMs: 1000, maximumPresenceBytes: 4096 } } } }) + const paths = { biometricsDatabase: fixture.path, configurationDatabase: configPath, presenceFile: presencePath } + const abort = new AbortController(), active = vi.fn(), fail = vi.fn() + const execution = plan.execution + if (!execution) throw new Error('Fixture execution missing') + const input = { execution, collectionEndsAtMs: plan.endMs, signal: abort.signal, active, fail } + const factory = plannedShadowSourceFactory(plan, paths) + const open = () => { + const source = factory(input) + cleanups.unshift(source.close) + return source + } + return { ...fixture, paths, plan, input, factory, open, abort, active, fail } +} +it('applies the saved context and capture policy to real connections and subscriptions', async () => { + const { open, insert, active, fail } = plannedSetup() + const source = open(), frames = source.frames[Symbol.asyncIterator]() + const next = frames.next() + publishProspectiveFrame({ type: 'piezo-dual', ts: start / 1000, freq: 500, left1: [1], right1: [2] }, false, start) + expect((await next).value.context).toEqual({ activeSides: ['left'], leftOccupied: true, rightOccupied: null, pumpContextKnown: false }) + insert() + source.flush?.() + expect(active).toHaveBeenCalledOnce() + expect(fail).not.toHaveBeenCalled() +}) +it('refuses changed source policies before opening resources', () => { + const { factory, input } = plannedSetup() + const changed = structuredClone(input.execution) + changed.sources.frames.maximumFrames++ + expect(() => factory({ ...input, execution: changed })).toThrow(/saved plan/) + expect(vi.getTimerCount()).toBe(0) +}) +it('closes both owned read-only handles on abort exactly once', () => { + const { open, abort } = plannedSetup() + const close = vi.spyOn(Database.prototype, 'close') + const source = open() + abort.abort() + expect(close).toHaveBeenCalledTimes(2) + source.close() + expect(close).toHaveBeenCalledTimes(2) + expect(vi.getTimerCount()).toBe(0) + close.mockRestore() +}) +it('unwinds the first connection if the second database cannot open', () => { + const { plan, paths, input } = plannedSetup() + const factory = plannedShadowSourceFactory(plan, { ...paths, configurationDatabase: paths.configurationDatabase + '.missing' }) + const close = vi.spyOn(Database.prototype, 'close') + expect(() => factory(input)).toThrow() + expect(close).toHaveBeenCalledOnce() + expect(vi.getTimerCount()).toBe(0) + close.mockRestore() +}) +it('enforces the saved frame count and releases owned resources on overflow', () => { + const { open, fail } = plannedSetup() + open() + const frame = { type: 'piezo-dual', ts: start / 1000, freq: 500, left1: [1], right1: [2] } + const close = vi.spyOn(Database.prototype, 'close') + publishProspectiveFrame(frame, false, start) + publishProspectiveFrame(frame, false, start) + expect(fail).toHaveBeenCalledOnce() + expect(close).toHaveBeenCalledTimes(2) + expect(vi.getTimerCount()).toBe(0) + close.mockRestore() +}) + +it('opens owned shadow state writable while keeping active capture read-only and detecting replacement', () => { + const { path, readonly } = setup() + const state = openShadowStateDatabase(path, { busyTimeoutMs: 50 }) + try { + expect(state.db.readonly).toBe(false) + state.db.exec('CREATE TABLE shadow_connection_test (value INTEGER)') + state.db.prepare('INSERT INTO shadow_connection_test VALUES (1)').run() + expect(() => readonly.prepare('INSERT INTO shadow_connection_test VALUES (2)').run()).toThrow(/readonly/) + state.assertCurrent() + renameSync(path, path + '.moved') + expect(state.assertCurrent).toThrow() + } + finally { state.close() } +}) diff --git a/src/biometrics/shadow/tests/admission.test.ts b/src/biometrics/shadow/tests/admission.test.ts new file mode 100644 index 000000000..7ffe609ca --- /dev/null +++ b/src/biometrics/shadow/tests/admission.test.ts @@ -0,0 +1,94 @@ +// @vitest-environment node +import { mkdtempSync, rmSync, writeFileSync, symlinkSync } from 'node:fs' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { afterEach, expect, it } from 'vitest' +import { dataHash } from '../../algorithms/canonical' +import { manifestHash } from '../../algorithms/registry' +import { installedShadowAdmitter, readInstalledShadowQualifications } from '../admission' +const a = 'a'.repeat(64), b = 'b'.repeat(64), now = 1700000000000 +const directories: string[] = [] +afterEach(() => directories.splice(0).forEach(path => rmSync(path, { recursive: true, force: true }))) +function setup() { + const directory = mkdtempSync(join(tmpdir(), 'shadow-admission-')) + directories.push(directory) + const workerPath = join(directory, 'worker.cjs'), file = join(directory, 'shadow-qualifications.json') + const execution = { version: 'shadow-execution-policy-v1', configHashes: { left: a, right: a }, + maximumRecords: 10, maximumOutputs: 10, maximumCpuMs: 1000, maximumInputBytes: 100000, maximumOutputBytes: 100000, + budgets: { cpuSeconds: 2, addressSpaceBytes: 100000000, maximumRssBytes: 100000000, wallMs: 10000, acknowledgmentMs: 1000, idleMs: 1000, rssIntervalMs: 100, rssTimeoutMs: 100 }, + leaseMs: 1500, heartbeatMs: 200, sources: { pollMs: 50, maximumActiveRows: 10, databaseBusyTimeoutMs: 50, + frames: { maximumFrames: 10, maximumBytes: 100000, maximumEventAgeMs: 1000, maximumFutureSkewMs: 0 }, + context: { presenceStaleMs: 300000, pumpStaleMs: 5000, maximumPresenceBytes: 4096 } } } + const entry = { qualificationId: 'reviewed-test-only', manifestHash, engineBuildHash: a, pythonBuildHash: b, hardwareId: 'fixture-hardware', sampleRateHz: 500, + activeConfigHash: a, executionPolicyHash: dataHash(execution), fixtureEvidenceHash: a, podEvidenceHash: b, heldOutEvidenceHash: a, + contextEvidence: 'host_observations_not_live_processor_cache_attestation', maximumRuntimeHeartbeatAgeMs: 1000, maximumHardwareObservationAgeMs: 1000, + healthPolicy: { qualificationId: 'reviewed-test-only', minimumAvailableMemoryBytes: 100, minimumFreeDiskBytes: 100, + maximumLoadPerCpu: 1, maximumEventLoopDelayMs: 100, maximumPiezoAgeMs: 1000, maximumTemperatureC: 70, maximumSampleAgeMs: 1000 } } + const plan = { schema: 'biometric-shadow-plan-v1', side: 'left', subjectId: 'test', execution, startMs: now, endMs: now + 10000, + candidateConfigHash: a, activeConfigHash: a, engineBuildHash: a, qualificationHash: dataHash(entry), maximumPoints: 10, maximumBytes: 10000, metrics: ['heart_rate'], + joining: { maximumPendingPoints: 10, maximumLatenessMs: 100, drainGraceMs: 500 }, + monitoring: { expectedIntervalMs: 1000, maximumGapMs: 2000, heartbeatStaleMs: 1000, minimumCandidateCoverage: 0.8, maximumCoverageLoss: 0.1, minimumQuality: 0.8, + alignment: { version: 'reference-alignment-v1', pointToleranceMs: 0, clockOffsetMs: 0, offsetSource: 'fixed' } } } + const identity = { engineBuildHash: a, pythonBuildHash: b, hardwareId: 'fixture-hardware', sampleRateHz: 500 } + const binding = { side: 'left', generation: 3, resolvedHash: a, appliedHash: a, appliedGeneration: 3, state: 'applied', heartbeatAtMs: now } + const health = { measuredAtMs: now, availableMemoryBytes: 1000, freeDiskBytes: 1000, loadPerCpu: 0.1, eventLoopDelayMs: 1, piezoReceivedAtMs: now, temperatureC: 40 } + const options = { workerPath, expectedBinding: { side: 'left' as const, generation: 3, resolvedHash: a }, + identity: () => identity, binding: () => binding, health: () => health, now: () => now } + const install = (entries: unknown[] = [entry]) => writeFileSync(file, JSON.stringify({ version: 'biometric-shadow-qualifications-v1', entries })) + install() + return { entry, plan, options, identity, binding, health, install, file, workerPath, admit: installedShadowAdmitter(options) } +} +it('admits only an exact installed qualification with applied current binding and fresh health', () => { + const { admit, plan, binding } = setup() + expect(admit(plan)).toBeUndefined() + binding.generation++ + binding.appliedGeneration++ + expect(() => admit(plan)).toThrow(/binding changed/) +}) +it.each(['engineBuildHash', 'pythonBuildHash', 'hardwareId', 'sampleRateHz'] as const)('refuses changed installed %s', (field) => { + const fixture = setup() + Object.assign(fixture.identity, { [field]: field === 'sampleRateHz' ? 250 : field === 'pythonBuildHash' ? a : b }) + expect(() => fixture.admit(fixture.plan)).toThrow() +}) +it.each(['appliedHash', 'appliedGeneration', 'state', 'heartbeatAtMs', 'side'])('refuses invalid runtime %s', (field) => { + const fixture = setup() + Object.assign(fixture.binding, { [field]: field === 'appliedGeneration' ? 2 : field === 'heartbeatAtMs' ? now - 1001 : field === 'state' ? 'pending_boundary' : field === 'side' ? 'right' : b }) + expect(() => fixture.admit(fixture.plan)).toThrow() +}) +it.each(['availableMemoryBytes', 'freeDiskBytes', 'measuredAtMs', 'piezoReceivedAtMs', 'temperatureC', 'loadPerCpu'])('refuses unhealthy or stale %s', (field) => { + const fixture = setup() + Object.assign(fixture.health, { [field]: field.endsWith('Ms') ? now - 1001 : field.endsWith('Bytes') ? 0 : 100 }) + expect(() => fixture.admit(fixture.plan)).toThrow(/admission refused/) +}) +it('binds all resource limits and refuses substitution or duplicate qualification entries', () => { + const { admit, plan, entry, install } = setup() + plan.execution.budgets.maximumRssBytes++ + expect(() => admit(plan)).toThrow(/installed execution/) + plan.execution.budgets.maximumRssBytes-- + install([entry, entry]) + expect(() => admit(plan)).toThrow(/ambiguous/) +}) +it('reads revocation on subsequent admission instead of caching authorization', () => { + const { admit, plan, install } = setup() + admit(plan) + install([]) + expect(() => admit(plan)).toThrow(/missing/) +}) +it.each(['missing', 'oversize', 'symlink', 'malformed'])('rejects a %s installed file', (mode) => { + const { file, workerPath } = setup() + if (mode === 'missing') rmSync(file) + if (mode === 'oversize') writeFileSync(file, 'x'.repeat(256 * 1024 + 1)) + if (mode === 'malformed') writeFileSync(file, '{}') + if (mode === 'symlink') { + rmSync(file) + writeFileSync(file + '.target', '{}') + symlinkSync(file + '.target', file) + } + expect(() => readInstalledShadowQualifications(workerPath)).toThrow(/unavailable/) +}) +it('rejects asynchronous host observations without leaking rejected promises', async () => { + const { options, plan } = setup() + const admit = installedShadowAdmitter({ ...options, identity: () => Promise.reject(new Error('collector failed')) }) + expect(() => admit(plan)).toThrow(/synchronous/) + await Promise.resolve() +}) diff --git a/src/biometrics/shadow/tests/attempt.test.ts b/src/biometrics/shadow/tests/attempt.test.ts new file mode 100644 index 000000000..379ac392a --- /dev/null +++ b/src/biometrics/shadow/tests/attempt.test.ts @@ -0,0 +1,453 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { fileURLToPath } from 'node:url' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { defaultConfig, resolveConfig } from '../../algorithms/registry' +import { configHash } from '../../algorithms/canonical' +import { reportShadowRun } from '../report' +import { shadowReportSchema } from '../report-contracts' +import { type ShadowPartialReceipt, validateShadowPartialReceipt, validateShadowWorkerReceipt } from '../receipt' +import { ShadowStore } from '../store' +import { runShadowAttempt, type ShadowAttemptHost } from '../attempt' + +const worker = vi.hoisted(() => ({ calls: 0, cleaned: false, fail: false, corruptReceipt: false, cleanupUnconfirmed: false })) +vi.mock('../worker-child', () => ({ runShadowChild: async (_location: unknown, init: { side: string, configHashes: Record }, frames: AsyncIterable<{ observedAtMs: number }>, output: (row: unknown) => void, options: { budgets: ShadowPartialReceipt['budgets'], onResources?: (value: ShadowPartialReceipt) => void }) => { + worker.calls++ + let result + try { + let sequence = 0 + for await (const frame of frames) { + if (worker.fail) throw new Error('processor_failed') + output({ side: init.side, sourceSide: init.side, timestampMs: frame.observedAtMs, configHash: init.configHashes[init.side], sequence: sequence++, + inputHash: 'a'.repeat(64), flags: null, limitations: [], readings: [ + { metric: 'heart_rate', unit: 'bpm', windowMs: null, value: 60, quality: 1 }, + { metric: 'breathing_rate', unit: 'breaths/min', windowMs: null, value: 12, quality: 1 }, + { metric: 'window_ibi_variability_v1', unit: 'ms', windowMs: 300000, value: null, quality: 0 }, + ] }) + } + result = { health: { state: 'completed', reason: 'window_complete', records: sequence, outputs: sequence, cpu_ms: 1, + input_hash: 'a'.repeat(64), counts: { records: sequence, accepted_frames: sequence, empty_channel_frames: 0, + pump_gated_frames: 0, preroll_outputs: 0, scored_outputs: sequence }, limitations: [] }, + inputBytes: worker.corruptReceipt ? 100001 : 100, outputBytes: 100, peakObservedRssBytes: 1000, rssSamples: 2, maximumObservationGapMs: 100, + budgets: options.budgets, memoryScope: 'sum_rss_of_shadow_process_group', exactKernelPeak: false } + } + catch (error) { + if (!worker.cleanupUnconfirmed) throw error + } + finally { + worker.cleaned = true + options.onResources?.({ kind: 'partial_supervisor_observation', health: null, inputBytes: 100, outputBytes: 100, diagnosticBytes: 0, + peakObservedRssBytes: 1000, rssSamples: 2, maximumObservationGapMs: 100, budgets: options.budgets, + cleanupConfirmed: !worker.cleanupUnconfirmed, memoryScope: 'sum_rss_of_shadow_process_group', exactKernelPeak: false }) + } + if (worker.cleanupUnconfirmed) throw Object.assign(new Error('shadow_worker_cleanup_unconfirmed'), { code: 'SHADOW_CLEANUP_UNCONFIRMED' }) + return result +} })) +const start = 1700000000000, hash = configHash(defaultConfig) +const databases: Database.Database[] = [] +beforeEach(() => { + vi.useFakeTimers() + vi.setSystemTime(start) + worker.calls = 0 + worker.cleaned = false + worker.fail = false + worker.corruptReceipt = false + worker.cleanupUnconfirmed = false +}) +afterEach(() => { + vi.useRealTimers() + databases.splice(0).forEach(db => db.close()) +}) +function setup(maximumPoints = 10, duration = 2000) { + const db = new Database(':memory:') + databases.push(db) + migrate(drizzle(db), { migrationsFolder: fileURLToPath(new URL('../../../db/biometrics-migrations/', import.meta.url)) }) + const store = new ShadowStore(db, { maximumRuns: 10, maximumPoints: 100, maximumBytes: 100000 }) + const execution = { version: 'shadow-execution-policy-v1', configHashes: { left: hash, right: hash }, + maximumRecords: 10, maximumOutputs: 10, maximumCpuMs: 1000, maximumInputBytes: 100000, maximumOutputBytes: 100000, + budgets: { cpuSeconds: 2, addressSpaceBytes: 100000000, maximumRssBytes: 100000000, wallMs: 10000, acknowledgmentMs: 1000, idleMs: 1000, rssIntervalMs: 100, rssTimeoutMs: 100 }, + leaseMs: 1500, heartbeatMs: 200, + sources: { pollMs: 50, maximumActiveRows: 10, databaseBusyTimeoutMs: 50, + frames: { maximumFrames: 10, maximumBytes: 100000, maximumEventAgeMs: 1000, maximumFutureSkewMs: 0 }, + context: { presenceStaleMs: 300000, pumpStaleMs: 5000, maximumPresenceBytes: 4096 } } } + const run = store.create({ execution, schema: 'biometric-shadow-plan-v1', subjectId: 'test', side: 'left', startMs: start, endMs: start + duration, + candidateConfigHash: hash, activeConfigHash: hash, qualificationHash: 'c'.repeat(64), engineBuildHash: 'd'.repeat(64), + maximumPoints, maximumBytes: 10000, metrics: ['heart_rate'], joining: { maximumPendingPoints: 10, maximumLatenessMs: 100, drainGraceMs: 500 }, + monitoring: { expectedIntervalMs: 1000, maximumGapMs: 2000, heartbeatStaleMs: 1000, minimumCandidateCoverage: 0.8, maximumCoverageLoss: 0.1, minimumQuality: 0.8, + alignment: { version: 'reference-alignment-v1', pointToleranceMs: 0, clockOffsetMs: 0, offsetSource: 'fixed' } } }, 'test') + let active: (reading: unknown) => void = () => { + throw new Error('Sources not open') + } + const close = vi.fn() + const host: ShadowAttemptHost = { + location: { pythonPath: '/trusted/python', modulesPath: '/trusted/modules' }, configs: { left: defaultConfig, right: defaultConfig }, + admit: vi.fn(), + openSources: (input) => { + active = input.active + let sent = false + return { close, frames: { [Symbol.asyncIterator]: () => ({ next: () => { + if (sent) return new Promise>(() => {}) + sent = true + return Promise.resolve({ done: false, value: { type: 'frame', observedAtMs: start } }) + } }) } } + }, + } + return { db, store, run, host, close, active: (timestampMs = start) => active({ side: 'left', timestampMs, configHash: hash, + readings: [{ metric: 'heart_rate', unit: 'bpm', windowMs: null, value: 61, quality: 1 }, + { metric: 'breathing_rate', unit: 'breaths/min', windowMs: null, value: 12, quality: 1 }, + { metric: 'window_ibi_variability_v1', unit: 'ms', windowMs: 300000, value: null, quality: 0 }] }) } +} +describe('leased shadow attempt lifecycle', () => { + it('retains both streams, renews the lease and completes only after final drain', async () => { + const { db, store, run, host, close, active } = setup() + for (const operation of ['INSERT', 'UPDATE', 'DELETE']) db.exec(`CREATE TRIGGER deny_${operation} BEFORE ${operation} ON vitals BEGIN SELECT RAISE(ABORT, 'canonical write forbidden'); END`) + const task = runShadowAttempt(store, run.id, host) + await vi.advanceTimersByTimeAsync(0) + active() + await vi.advanceTimersByTimeAsync(2000) + expect(store.get(run.id).state).toBe('running') + expect(worker.cleaned).toBe(true) + active(start + 1999) + await vi.advanceTimersByTimeAsync(101) + expect((await task).run).toMatchObject({ state: 'completed', point_count: 2, reason: 'observation_window_complete' }) + const sample = store.snapshot(run.id, { maximumPoints: 10, maximumBytes: 10000 }).samples[0] + expect(sample).toMatchObject({ readings: [{ active: { value: 61 }, candidate: { value: 60 } }] }) + expect(close).toHaveBeenCalledOnce() + expect(vi.mocked(host.admit).mock.calls.length).toBeGreaterThan(5) + }) + it('rejects asynchronous admission without claiming or opening sources', async () => { + const { store, run, host } = setup() + host.admit = async () => {} + await expect(runShadowAttempt(store, run.id, host)).rejects.toThrow(/synchronously/) + expect(store.get(run.id).state).toBe('pending') + expect(worker.calls).toBe(0) + }) + it('cancels, waits for worker cleanup, and ignores late source callbacks', async () => { + const { store, run, host, close, active } = setup() + const abort = new AbortController() + const task = runShadowAttempt(store, run.id, host, abort.signal) + const rejected = expect(task).rejects.toThrow('shadow_cancelled') + await vi.advanceTimersByTimeAsync(0) + abort.abort() + await rejected + expect(worker.cleaned).toBe(true) + expect(close).toHaveBeenCalledOnce() + expect(store.get(run.id)).toMatchObject({ state: 'stopped', reason: 'shadow_cancelled', point_count: 0 }) + active() + expect(store.get(run.id).point_count).toBe(0) + }) + it('rechecks binding/resource admission during observation', async () => { + const { store, run, host } = setup() + const task = runShadowAttempt(store, run.id, host) + const rejected = expect(task).rejects.toThrow('shadow_admission_or_lease_lost') + await vi.advanceTimersByTimeAsync(0) + host.admit = () => { + throw new Error('active binding changed') + } + await vi.advanceTimersByTimeAsync(200) + await rejected + expect(store.get(run.id)).toMatchObject({ state: 'failed', reason: 'shadow_admission_or_lease_lost' }) + expect(worker.cleaned).toBe(true) + }) + it('never finishes or writes over a replacement attempt', async () => { + const { store, run, host } = setup(10, 5000) + const task = runShadowAttempt(store, run.id, host) + const rejected = expect(task).rejects.toThrow(/lease_lost/) + await vi.advanceTimersByTimeAsync(0) + vi.setSystemTime(start + 1600) + const replacement = store.claim(run.id, 1500, () => {}) + await vi.advanceTimersByTimeAsync(200) + await rejected + expect(store.get(run.id)).toMatchObject({ state: 'running', attempt_id: replacement.attemptId, point_count: 0 }) + }) + it('preserves storage-quota termination instead of marking completion', async () => { + const { store, run, host, active } = setup(1) + const task = runShadowAttempt(store, run.id, host) + const rejected = expect(task).rejects.toThrow('storage_quota') + await vi.advanceTimersByTimeAsync(1) + active(start + 1) + await vi.advanceTimersByTimeAsync(200) + await rejected + expect(store.get(run.id)).toMatchObject({ state: 'stopped', reason: 'storage_quota', point_count: 1 }) + }) + it('fails when the source ends before the prospective window', async () => { + const { store, run, host, close } = setup() + host.openSources = () => ({ close, frames: { async* [Symbol.asyncIterator]() {} } }) + await expect(runShadowAttempt(store, run.id, host)).rejects.toThrow('shadow_source_ended_early') + expect(store.get(run.id).state).toBe('failed') + expect(close).toHaveBeenCalledOnce() + }) + it('retains a processor failure and cleans up both sources', async () => { + const { store, run, host, close } = setup() + worker.fail = true + await expect(runShadowAttempt(store, run.id, host)).rejects.toThrow('processor_failed') + expect(store.get(run.id)).toMatchObject({ state: 'failed', reason: 'processor_failed' }) + expect(close).toHaveBeenCalledOnce() + }) +}) +it('records an expired attempt as failed only after worker cleanup', async () => { + const { store, run, host } = setup(10, 5000) + const task = runShadowAttempt(store, run.id, host) + const rejected = expect(task).rejects.toThrow(/lease_lost/) + await vi.advanceTimersByTimeAsync(0) + vi.setSystemTime(start + 1600) + await vi.advanceTimersByTimeAsync(200) + await rejected + expect(worker.cleaned).toBe(true) + expect(store.get(run.id)).toMatchObject({ state: 'failed', reason: 'shadow_admission_or_lease_lost' }) +}) +it('preserves an operator stop and reaps the attempt', async () => { + const { store, run, host } = setup() + const task = runShadowAttempt(store, run.id, host) + const rejected = expect(task).rejects.toThrow(/lease_lost/) + await vi.advanceTimersByTimeAsync(0) + store.stop(run.id, 'operator_stop') + await vi.advanceTimersByTimeAsync(200) + await rejected + expect(worker.cleaned).toBe(true) + expect(store.get(run.id)).toMatchObject({ state: 'stopped', reason: 'operator_stop' }) +}) +it('does not mark completion if source cleanup fails', async () => { + const { store, run, host, close } = setup() + close.mockImplementation(() => { + throw new Error('unsubscribe failed') + }) + const task = runShadowAttempt(store, run.id, host) + const rejected = expect(task).rejects.toThrow('shadow_source_cleanup_failed') + await vi.advanceTimersByTimeAsync(2101) + await rejected + expect(store.get(run.id).state).toBe('failed') +}) +it('refuses to finish an unexpired lease through expired-attempt recovery', () => { + const { store, run } = setup() + const lease = store.claim(run.id, 1500, () => {}) + expect(store.failExpired(lease, 'worker_gone')).toBe(false) + expect(store.get(run.id).state).toBe('running') +}) +it('stops immediately when the active source reports a disconnect', async () => { + const { store, run, host } = setup() + const open = host.openSources + let disconnect = () => {} + host.openSources = (input) => { + disconnect = input.fail + return open(input) + } + const task = runShadowAttempt(store, run.id, host) + const rejected = expect(task).rejects.toThrow('shadow_source_failed') + await vi.advanceTimersByTimeAsync(0) + disconnect() + await rejected + expect(worker.cleaned).toBe(true) + expect(store.get(run.id)).toMatchObject({ state: 'failed', reason: 'shadow_source_failed' }) +}) +it('rechecks admission after cleanup before publishing completion', async () => { + const { store, run, host, close } = setup() + close.mockImplementation(() => { + host.admit = () => { + throw new Error('active profile changed during cleanup') + } + }) + const task = runShadowAttempt(store, run.id, host) + const rejected = expect(task).rejects.toThrow('shadow_admission_changed_before_completion') + await vi.advanceTimersByTimeAsync(2101) + await rejected + expect(store.get(run.id).state).toBe('failed') +}) +it('rejects future active timestamps before buffering them', async () => { + const { store, run, host, active } = setup() + const task = runShadowAttempt(store, run.id, host) + const rejected = expect(task).rejects.toThrow('shadow_active_stream_failed') + await vi.advanceTimersByTimeAsync(0) + active(start + 1) + await rejected + expect(store.get(run.id)).toMatchObject({ state: 'failed', point_count: 0 }) +}) +it('flushes active observations before sealing the final drain', async () => { + const { store, run, host, active } = setup() + const open = host.openSources + let emitted = false + host.openSources = input => ({ ...open(input), flush: () => { + if (!emitted && Date.now() >= start + 2100) { + active(start + 1999) + emitted = true + } + } }) + const task = runShadowAttempt(store, run.id, host) + await vi.advanceTimersByTimeAsync(2101) + await task + expect(store.get(run.id)).toMatchObject({ state: 'completed', point_count: 2 }) + expect(store.snapshot(run.id, { maximumPoints: 10, maximumBytes: 10000 }).samples[1]).toMatchObject({ timestampMs: start + 1999, pairing: { activePresent: true, candidatePresent: false } }) +}) +it('refuses asynchronous final-drain polling', async () => { + const { store, run, host } = setup() + const open = host.openSources + host.openSources = input => ({ ...open(input), flush: async () => {} }) + const task = runShadowAttempt(store, run.id, host) + const rejected = expect(task).rejects.toThrow(/admission_or_lease_lost/) + await vi.advanceTimersByTimeAsync(200) + await rejected + expect(store.get(run.id).state).toBe('failed') +}) +it('records lease expiry during final cleanup as failure rather than leaving a running attempt', async () => { + const { store, run, host, close } = setup() + close.mockImplementation(() => { + vi.setSystemTime(start + 3000) + }) + const task = runShadowAttempt(store, run.id, host) + const rejected = expect(task).rejects.toThrow(/Stale shadow lease/) + await vi.advanceTimersByTimeAsync(2101) + await rejected + expect(store.get(run.id)).toMatchObject({ state: 'failed', reason: 'shadow_completion_lease_lost' }) +}) +it('keeps legacy plans inspectable but refuses to execute without a frozen policy', async () => { + const { store, run, host } = setup() + const legacy = { ...run.plan } + delete legacy.execution + const old = store.create(legacy, 'legacy') + expect(store.get(old.id).plan.execution).toBeUndefined() + await expect(runShadowAttempt(store, old.id, host)).rejects.toThrow(/execution policies/) + expect(store.get(old.id).state).toBe('pending') + expect(worker.calls).toBe(0) +}) +it('binds the other physical channel configuration before launching a worker', async () => { + const { store, run, host } = setup() + host.configs.right = resolveConfig([{ path: 'heart_rate.trackingToleranceBpm', value: 20 }]).resolved + await expect(runShadowAttempt(store, run.id, host)).rejects.toThrow(/frozen plan/) + expect(worker.calls).toBe(0) + expect(store.get(run.id).state).toBe('pending') +}) +it('source factories receive a policy copy and cannot change worker lease budgets', async () => { + const { store, run, host } = setup() + const open = host.openSources + host.openSources = (input) => { + expect(input.execution).toEqual(run.plan.execution) + input.execution.leaseMs = 3600000 + input.execution.budgets.wallMs = 1 + return open(input) + } + const abort = new AbortController() + const task = runShadowAttempt(store, run.id, host, abort.signal) + const rejected = expect(task).rejects.toThrow('shadow_cancelled') + await vi.advanceTimersByTimeAsync(0) + expect(store.get(run.id).lease_until_ms).toBe(start + 1500) + expect(store.get(run.id).plan.execution).toEqual(run.plan.execution) + abort.abort() + await rejected +}) +it('rejects incompatible poll cadence and freezes policy changes into plan idempotency', () => { + const { store, run } = setup() + const execution = structuredClone(run.plan.execution) + if (!execution) throw new Error('Missing fixture policy') + execution.sources.pollMs = 100 + expect(() => store.create({ ...run.plan, execution }, 'invalid')).toThrow(/Invalid shadow window/) + execution.sources.pollMs = 50 + execution.maximumRecords++ + expect(() => store.create({ ...run.plan, execution }, 'test')).toThrow(/idempotency/) +}) + +it('persists the receipt with completion and rejects later corruption', async () => { + const { db, store, run, host } = setup() + const task = runShadowAttempt(store, run.id, host) + await vi.advanceTimersByTimeAsync(2101) + const result = await task + const reopened = new ShadowStore(db, { maximumRuns: 10, maximumPoints: 100, maximumBytes: 100000 }) + expect(reopened.get(run.id).workerReceipt).toMatchObject({ runId: run.id, attemptId: result.lease.attemptId, + planHash: run.plan_hash, receipt: result.receipt }) + expect(reopened.snapshot(run.id, { maximumPoints: 10, maximumBytes: 10000 }).run.workerReceipt).not.toBeNull() + const execution = result.run.plan.execution + if (!execution) throw new Error('Missing execution') + for (const bad of [ + { ...result.receipt, inputBytes: 100001 }, { ...result.receipt, outputBytes: 100001 }, + { ...result.receipt, peakObservedRssBytes: 100000001 }, { ...result.receipt, rssSamples: 0 }, + { ...result.receipt, budgets: { ...result.receipt.budgets, cpuSeconds: 3 } }, + { ...result.receipt, health: { ...result.receipt.health, cpu_ms: 1001 } }, + { ...result.receipt, health: { ...result.receipt.health, records: 11 } }, + ]) expect(() => validateShadowWorkerReceipt(bad, execution)).toThrow() + db.prepare('UPDATE biometric_shadow_runs SET worker_receipt_hash = \'corrupted\' WHERE id = ?').run(run.id) + expect(() => reopened.get(run.id)).toThrow(/receipt integrity/) +}) +it('does not complete an executable plan without a worker receipt', () => { + const { store, run } = setup() + const lease = store.claim(run.id, 1500, () => {}) + expect(() => store.finish(lease, 'completed', 'missing')).toThrow() + expect(store.get(run.id)).toMatchObject({ state: 'running', workerReceipt: null }) +}) +it('fails completion after cleanup when the worker receipt violates the saved budget', async () => { + const { store, run, host } = setup() + worker.corruptReceipt = true + const task = runShadowAttempt(store, run.id, host) + const rejected = expect(task).rejects.toThrow(/receipt does not match/) + await vi.advanceTimersByTimeAsync(2101) + await rejected + expect(worker.cleaned).toBe(true) + expect(store.get(run.id)).toMatchObject({ state: 'failed', workerReceipt: { receipt: { kind: 'partial_supervisor_observation' } }, reason: 'shadow_completion_failed' }) +}) + +it('preserves cleanup uncertainty through cancellation and durable failure evidence', async () => { + const { store, run, host } = setup() + worker.cleanupUnconfirmed = true + const abort = new AbortController() + const task = runShadowAttempt(store, run.id, host, abort.signal) + const rejected = expect(task).rejects.toMatchObject({ code: 'SHADOW_CLEANUP_UNCONFIRMED' }) + await vi.advanceTimersByTimeAsync(0) + abort.abort() + await rejected + expect(store.get(run.id)).toMatchObject({ state: 'failed', reason: 'shadow_worker_cleanup_unconfirmed', workerReceipt: { receipt: { cleanupConfirmed: false } } }) +}) + +it('attaches partial evidence after an operator stop without changing terminal authority', async () => { + const { db, store, run, host } = setup() + const abort = new AbortController() + const task = runShadowAttempt(store, run.id, host, abort.signal) + const rejected = expect(task).rejects.toThrow('shadow_cancelled') + await vi.advanceTimersByTimeAsync(0) + const stopped = store.stop(run.id, 'operator') + await vi.advanceTimersByTimeAsync(50) + abort.abort() + await rejected + const reopened = new ShadowStore(db).get(run.id) + expect(reopened).toMatchObject({ state: 'stopped', reason: 'operator', ended_at_ms: stopped.ended_at_ms, + workerReceipt: { schema: 'biometric-shadow-partial-receipt-v1', recordedAtMs: start + 50, receipt: { cleanupConfirmed: true } } }) + expect(shadowReportSchema.parse(reportShadowRun(store, run.id, { maximumPoints: 100, maximumBytes: 100000 })).workerReceipt).toEqual(reopened.workerReceipt) + const receipt = reopened.workerReceipt + if (!receipt) throw new Error('Receipt missing') + const lease = { runId: run.id, attemptId: receipt.attemptId } + expect(store.recordPartialReceipt(lease, receipt.receipt)).toBe(true) + expect(() => store.recordPartialReceipt(lease, { ...receipt.receipt, inputBytes: 200 })).toThrow('already recorded') + expect(store.recordPartialReceipt({ ...lease, attemptId: 'replacement' }, receipt.receipt)).toBe(false) + db.prepare('UPDATE biometric_shadow_runs SET worker_receipt_hash = ? WHERE id = ?').run('corrupted', run.id) + expect(() => store.get(run.id)).toThrow('receipt integrity') +}) +it('retains cleanup quarantine if storing partial evidence also fails', async () => { + const { store, run, host } = setup() + worker.cleanupUnconfirmed = true + worker.fail = true + vi.spyOn(store, 'recordPartialReceipt').mockImplementation(() => { + throw new Error('disk unavailable') + }) + const task = runShadowAttempt(store, run.id, host) + const rejected = expect(task).rejects.toMatchObject({ code: 'SHADOW_CLEANUP_UNCONFIRMED', cause: { message: 'Shadow partial resource evidence could not be stored' } }) + await vi.advanceTimersByTimeAsync(0) + await rejected +}) + +it('preserves overruns as partial evidence but rejects inconsistent availability and policy', () => { + const { store, run } = setup() + const execution = run.plan.execution + if (!execution) throw new Error('Missing execution') + const receipt: ShadowPartialReceipt = { kind: 'partial_supervisor_observation', health: null, inputBytes: 100001, outputBytes: 200001, + diagnosticBytes: 20000, peakObservedRssBytes: 200000000, rssSamples: 1, maximumObservationGapMs: 100, + budgets: execution.budgets, cleanupConfirmed: false, memoryScope: 'sum_rss_of_shadow_process_group', exactKernelPeak: false } + expect(validateShadowPartialReceipt(receipt, execution)).toEqual(receipt) + for (const bad of [{ ...receipt, rssSamples: 0 }, { ...receipt, peakObservedRssBytes: null }, + { ...receipt, budgets: { ...receipt.budgets, cpuSeconds: 3 } }, { ...receipt, outputBytes: -1 }]) { + expect(() => validateShadowPartialReceipt(bad, execution)).toThrow() + } + const lease = store.claim(run.id, 1500, () => {}) + expect(store.recordPartialReceipt(lease, receipt)).toBe(false) + store.stop(run.id, 'operator') + expect(store.recordPartialReceipt(lease, receipt)).toBe(true) + expect(reportShadowRun(store, run.id, { maximumPoints: 100, maximumBytes: 100000 }).issues).toContain('worker_cleanup_unconfirmed') +}) diff --git a/src/biometrics/shadow/tests/build.test.ts b/src/biometrics/shadow/tests/build.test.ts new file mode 100644 index 000000000..e90b3cac4 --- /dev/null +++ b/src/biometrics/shadow/tests/build.test.ts @@ -0,0 +1,65 @@ +// @vitest-environment node +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' +import { expect, it } from 'vitest' +import { configHash } from '../../algorithms/canonical' +import { probeShadowBuild } from '../build' +const location = { pythonPath: resolve('modules/piezo-processor/.venv/bin/python'), modulesPath: resolve('modules') } +it('probes the exact locked interpreter and requires shadow sources and intact identity', async () => { + const build = await probeShadowBuild(location) + expect(build).toEqual(await probeShadowBuild(location)) + expect(build.identity.version).toBe('shadow-python-build-v1') + expect(build.identity.sources).toHaveProperty('common/biometrics/shadow_process.py') + expect(build.identity.sources).toHaveProperty('common/biometrics/signal.py') + const directory = await mkdtemp(join(tmpdir(), 'shadow-build-')) + try { + const executable = join(directory, 'probe') + for (const raw of [{ ...build, hash: '0'.repeat(64) }, { ...build, identity: { ...build.identity, version: 'waveform-python-build-v1' } }]) { + await writeFile(executable, `#!${process.execPath}\nprocess.stdout.write(${JSON.stringify(JSON.stringify(raw))});`, { mode: 0o700 }) + await expect(probeShadowBuild({ ...location, pythonPath: executable })).rejects.toThrow() + } + const missing = structuredClone(build.identity) + delete missing.sources['common/biometrics/shadow_process.py'] + await writeFile(executable, `#!${process.execPath}\nprocess.stdout.write(${JSON.stringify(JSON.stringify({ hash: configHash(missing), identity: missing }))});`, { mode: 0o700 }) + await expect(probeShadowBuild({ ...location, pythonPath: executable })).rejects.toThrow(/incomplete/) + await expect(probeShadowBuild({ ...location, pythonPath: join(directory, 'missing') })).rejects.toThrow(/unavailable/) + const abort = new AbortController() + abort.abort() + await expect(probeShadowBuild(location, abort.signal)).rejects.toThrow(/unavailable/) + } + finally { await rm(directory, { recursive: true, force: true }) } +}) +it('packages the host without live service imports and binds its exact bytes', async () => { + const { execFile } = await import('node:child_process') + const { promisify } = await import('node:util') + const { loadInstalledShadowHost, inspectInstalledShadowHost } = await import('../installed-host') + const { readFile } = await import('node:fs/promises') + const { shadowEngineBuildIdentity } = await import('../build') + const directory = await mkdtemp(resolve('node_modules/shadow-package-test-')) + try { + await promisify(execFile)(process.execPath, ['scripts/build/biometrics-worker.mjs', directory], { timeout: 30000 }) + const path = join(directory, 'shadow-host.cjs') + const loaded = loadInstalledShadowHost(path), api = loaded.api + loaded.assertCurrent() + expect(loadInstalledShadowHost(path).api).toBe(api) + expect(Object.keys(api).sort()).toEqual(['ShadowStore', 'openProfiledShadowHost', 'prepareShadowPlan', 'readInstalledShadowQualifications', 'runProfiledShadowAttempt', + 'shadowEngineBuildIdentity', 'shadowHostApiVersion', 'shadowPlanSchema', 'shadowStoragePolicySchema'].sort()) + const dependencies = JSON.parse(await readFile(join(directory, 'shadow-host-dependencies.json'), 'utf8')) as string[] + expect(dependencies).toContain('src/biometrics/algorithms/registry.ts') + expect(dependencies).toContain('src/streaming/prospectiveFrameTap.ts') + expect(dependencies.some(path => /src\/(hardware|automation|scheduler|homekit)\//.test(path) || path.endsWith('/piezoStream.ts'))).toBe(false) + const original = await shadowEngineBuildIdentity(path, location) + expect((await inspectInstalledShadowHost(path, location)).build).toEqual(original) + expect(original.identity.hostHash).toBe(loaded.hostHash) + expect(original.python).toEqual(await probeShadowBuild(location)) + expect(await api.shadowEngineBuildIdentity(path, location)).toEqual(original) + await writeFile(path, '// changed build\n' + await readFile(path, 'utf8')) + expect(() => loaded.assertCurrent()).toThrow(/changed/) + expect(() => loadInstalledShadowHost(path)).toThrow(/changed/) + const changed = await shadowEngineBuildIdentity(path, location) + expect(changed.hash).not.toBe(original.hash) + expect(changed.python.hash).toBe(original.python.hash) + } + finally { await rm(directory, { recursive: true, force: true }) } +}) diff --git a/src/biometrics/shadow/tests/context.test.ts b/src/biometrics/shadow/tests/context.test.ts new file mode 100644 index 000000000..bc2c700bd --- /dev/null +++ b/src/biometrics/shadow/tests/context.test.ts @@ -0,0 +1,91 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { mkdtempSync, rmSync, writeFileSync, utimesSync, symlinkSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, expect, it, vi } from 'vitest' +import { ShadowContextReader } from '../context' +const at = 1700000000000 +const cleanup: (() => void)[] = [] +afterEach(() => cleanup.splice(0).forEach(close => close())) +function fixture() { + const directory = mkdtempSync(join(tmpdir(), 'shadow-context-')), path = join(directory, 'presence.json') + const db = new Database(':memory:') + db.exec('CREATE TABLE side_settings(side TEXT, away_mode INTEGER); INSERT INTO side_settings VALUES (\'left\',0),(\'right\',0); CREATE TABLE device_settings(id INTEGER, bed_mode TEXT); INSERT INTO device_settings VALUES(1,\'two\');') + db.pragma('query_only = ON') + cleanup.push(() => { + db.close() + rmSync(directory, { recursive: true, force: true }) + }) + const assertCurrent = vi.fn() + const reader = new ShadowContextReader(db, path, { presenceStaleMs: 300000, pumpStaleMs: 5000, maximumPresenceBytes: 4096 }, assertCurrent) + const write = (value: unknown, mtime = at) => { + writeFileSync(path, JSON.stringify(value)) + utimesSync(path, mtime / 1000, mtime / 1000) + } + const sql = (value: string) => { + db.pragma('query_only = OFF') + db.exec(value) + db.pragma('query_only = ON') + } + return { db, path, reader, write, sql, assertCurrent } +} +const state = { version: 1, left: { vitals_presence: true, vitals_evidence_ts: at / 1000 }, right: { vitals_presence: false, vitals_evidence_ts: at / 1000 } } +it('uses committed per-side evidence and active sleeper settings rather than virtual UI occupancy', () => { + const { reader, write, sql } = fixture() + write(state) + expect(reader.snapshot({ type: 'piezo-dual' }, at)).toEqual({ activeSides: ['left', 'right'], leftOccupied: true, rightOccupied: false, pumpContextKnown: false }) + sql('UPDATE side_settings SET away_mode=1 WHERE side=\'right\'') + expect(reader.snapshot({}, at).activeSides).toEqual(['left']) + sql('UPDATE device_settings SET bed_mode=\'solo-right\'') + expect(reader.snapshot({}, at).activeSides).toEqual([]) + sql('UPDATE side_settings SET away_mode=0') + expect(reader.snapshot({}, at).activeSides).toEqual(['right']) +}) +it.each(['missing', 'malformed', 'version', 'stale_file', 'future_file', 'stale_evidence', 'future_evidence', 'oversize', 'symlink'])('preserves unknown presence for %s evidence', (mode) => { + const { reader, write, path } = fixture() + if (mode !== 'missing') write(state) + if (mode === 'malformed') writeFileSync(path, '{') + if (mode === 'version') write({ ...state, version: 2 }) + if (mode === 'stale_file') write(state, at - 300001) + if (mode === 'future_file') write(state, at + 1000) + if (mode === 'stale_evidence') write({ ...state, left: { ...state.left, vitals_evidence_ts: at / 1000 - 301 }, right: null }) + if (mode === 'future_evidence') write({ ...state, left: { ...state.left, vitals_evidence_ts: at / 1000 + 1 }, right: null }) + if (mode === 'oversize') write({ ...state, extra: 'x'.repeat(4096) }) + if (mode === 'symlink') { + rmSync(path) + writeFileSync(path + '.target', JSON.stringify(state)) + symlinkSync(path + '.target', path) + } + expect(reader.snapshot({}, at)).toMatchObject({ leftOccupied: null, rightOccupied: null }) +}) +it('does not let one missing side erase valid evidence on the other side', () => { + const { reader, write } = fixture() + write({ ...state, right: { vitals_presence: false, vitals_evidence_ts: null } }) + expect(reader.snapshot({}, at)).toMatchObject({ leftOccupied: true, rightOccupied: null }) +}) +it('marks incomplete or invalid settings unknown rather than guessing two sleepers', () => { + const { reader, sql } = fixture() + sql('DELETE FROM side_settings WHERE side=\'right\'') + expect(reader.snapshot({}, at).activeSides).toBeNull() + sql('INSERT INTO side_settings VALUES(\'right\',0); UPDATE device_settings SET bed_mode=\'unexpected\'') + expect(reader.snapshot({}, at).activeSides).toBeNull() +}) +it('requires both pump sides in this attempt and expires or invalidates incomplete updates', () => { + const { reader } = fixture() + expect(reader.snapshot({ type: 'frzHealth', left: { pump: { rpm: 1200 } } }, at).pumpContextKnown).toBe(false) + expect(reader.snapshot({ type: 'frzHealth', left: { pumpRpm: 0 }, right: { pump: { duty: '10' } } }, at).pumpContextKnown).toBe(true) + expect(reader.snapshot({}, at + 5000).pumpContextKnown).toBe(true) + expect(reader.snapshot({}, at + 5001).pumpContextKnown).toBe(false) + expect(reader.snapshot({ type: 'frzHealth', left: { rpm: 0 }, right: { rpm: 0 } }, at + 5001).pumpContextKnown).toBe(true) + expect(reader.snapshot({ type: 'frzHealth', left: { rpm: 0 }, right: {} }, at + 5002).pumpContextKnown).toBe(false) +}) +it('propagates replacement and clock errors instead of reporting a healthy context', () => { + const { reader, assertCurrent } = fixture() + reader.snapshot({}, at) + expect(() => reader.snapshot({}, at - 1)).toThrow(/clock/) + assertCurrent.mockImplementation(() => { + throw new Error('database replaced') + }) + expect(() => reader.snapshot({}, at)).toThrow('database replaced') +}) diff --git a/src/biometrics/shadow/tests/device-identity.test.ts b/src/biometrics/shadow/tests/device-identity.test.ts new file mode 100644 index 000000000..66c4b40c7 --- /dev/null +++ b/src/biometrics/shadow/tests/device-identity.test.ts @@ -0,0 +1,58 @@ +// @vitest-environment node +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import type { inspectInstalledShadowHost } from '../installed-host' +import { observedShadowHardware, installedShadowIdentity } from '../device-identity' +import { getDacMonitorIfRunning } from '@/src/hardware/dacMonitor.instance' +vi.mock('@/src/hardware/dacMonitor.instance', () => ({ getDacMonitorIfRunning: vi.fn() })) +const now = 1700000000000 +function monitor(version = 'J00', sensorLabel = '20600-0003-J55-B0708DE3', at: number | null = now, state = 'running') { + vi.mocked(getDacMonitorIfRunning).mockReturnValue({ getStatus: () => state, getLastPollAt: () => at, + getLastStatus: () => ({ podVersion: version, sensorLabel }) } as ReturnType) +} +beforeEach(() => vi.resetAllMocks()) +afterEach(() => vi.restoreAllMocks()) +it.each([['H00', 'H00'], ['I00', 'I11'], ['J00', 'J55']])('identifies %s by exact revision while discarding serials', (version, revision) => { + monitor(version, `20600-${revision}-SERIAL`) + expect(observedShadowHardware(1000, now)).toEqual({ hardwareId: `pod-sensor-v1/${version}/${revision}`, sampleRateHz: 500 }) +}) +it.each(['UNKNOWN', 'SERIAL-H00-JUNK-I11', '20600-K00-SERIAL', '20600-J550-SERIAL'])('rejects unknown or inconsistent sensor label %s', (label) => { + monitor('J00', label) + expect(() => observedShadowHardware(1000, now)).toThrow(/revision/) +}) +it.each([null, now + 1, now - 1001])('rejects missing, future or stale poll %s', (at) => { + monitor('J00', '20600-J55-SERIAL', at) + expect(() => observedShadowHardware(1000, now)).toThrow() +}) +it.each(['stopped', 'degraded'])('rejects %s monitors', (state) => { + monitor('J00', '20600-J55-SERIAL', now, state) + expect(() => observedShadowHardware(1000, now)).toThrow(/unavailable/) +}) +it('rejects an absent monitor without initializing one', () => { + vi.mocked(getDacMonitorIfRunning).mockReturnValue(null) + expect(() => observedShadowHardware(1000, now)).toThrow(/unavailable/) + expect(getDacMonitorIfRunning).toHaveBeenCalledOnce() +}) + +it('combines the inspected build with fresh qualified hardware and repeats artifact checks', () => { + vi.spyOn(Date, 'now').mockReturnValue(now) + const a = 'a'.repeat(64), b = 'b'.repeat(64) + const guard = vi.fn() + const inspection = { build: { hash: a, python: { hash: b } }, assertCurrent: guard } as unknown as Awaited> + const qualification = { qualificationId: 'test', manifestHash: a, engineBuildHash: a, pythonBuildHash: b, + hardwareId: 'pod-sensor-v1/J00/J55', sampleRateHz: 500, activeConfigHash: a, executionPolicyHash: a, + fixtureEvidenceHash: a, podEvidenceHash: a, heldOutEvidenceHash: a, + contextEvidence: 'host_observations_not_live_processor_cache_attestation', maximumRuntimeHeartbeatAgeMs: 1000, maximumHardwareObservationAgeMs: 1000, + healthPolicy: { qualificationId: 'test', minimumAvailableMemoryBytes: 1, minimumFreeDiskBytes: 1, maximumLoadPerCpu: 1, + maximumEventLoopDelayMs: 100, maximumPiezoAgeMs: 1000, maximumTemperatureC: 70, maximumSampleAgeMs: 1000 } } + expect(() => installedShadowIdentity(inspection, { ...qualification, pythonBuildHash: a })).toThrow(/build differs/) + const identity = installedShadowIdentity(inspection, qualification) + monitor() + expect(identity()).toEqual({ engineBuildHash: a, pythonBuildHash: b, hardwareId: qualification.hardwareId, sampleRateHz: 500 }) + monitor('I00', '20600-I11-SERIAL') + expect(identity).toThrow(/hardware differs/) + expect(guard).toHaveBeenCalledTimes(2) + guard.mockImplementation(() => { + throw new Error('artifact changed') + }) + expect(identity).toThrow(/artifact changed/) +}) diff --git a/src/biometrics/shadow/tests/fixtures/submission.ts b/src/biometrics/shadow/tests/fixtures/submission.ts new file mode 100644 index 000000000..022a94d46 --- /dev/null +++ b/src/biometrics/shadow/tests/fixtures/submission.ts @@ -0,0 +1,22 @@ +import { dataHash } from '../../../algorithms/canonical' +import { submitShadowSchema } from '../../submission-contracts' +export const submissionTime = 1700000000000 +export const submissionEntry = { qualificationId: 'synthetic-test-only' } +export function submissionFixture() { + const hash = 'a'.repeat(64) + return submitShadowSchema.parse({ idempotencyKey: 'submit', plan: { + schema: 'biometric-shadow-plan-v1', side: 'left', subjectId: 'fixture', candidateRevisionId: null, + expectedBindings: { left: { generation: 1, resolvedHash: hash }, right: { generation: 2, resolvedHash: hash } }, + execution: { version: 'shadow-execution-policy-v1', configHashes: { left: hash, right: hash }, + maximumRecords: 10, maximumOutputs: 10, maximumCpuMs: 1000, maximumInputBytes: 100000, maximumOutputBytes: 100000, + budgets: { cpuSeconds: 2, addressSpaceBytes: 100000000, maximumRssBytes: 100000000, wallMs: 10000, acknowledgmentMs: 1000, idleMs: 1000, rssIntervalMs: 100, rssTimeoutMs: 100 }, + leaseMs: 1500, heartbeatMs: 200, sources: { pollMs: 50, maximumActiveRows: 10, databaseBusyTimeoutMs: 50, + frames: { maximumFrames: 10, maximumBytes: 100000, maximumEventAgeMs: 1000, maximumFutureSkewMs: 0 }, + context: { presenceStaleMs: 300000, pumpStaleMs: 5000, maximumPresenceBytes: 4096 } } }, + startMs: submissionTime + 1000, endMs: submissionTime + 10000, candidateConfigHash: hash, activeConfigHash: hash, + engineBuildHash: hash, qualificationHash: dataHash(submissionEntry), maximumPoints: 10, maximumBytes: 10000, metrics: ['heart_rate'], + joining: { maximumPendingPoints: 10, maximumLatenessMs: 100, drainGraceMs: 500 }, + monitoring: { expectedIntervalMs: 1000, maximumGapMs: 2000, heartbeatStaleMs: 1000, minimumCandidateCoverage: 0.8, maximumCoverageLoss: 0.1, minimumQuality: 0.8, + alignment: { version: 'reference-alignment-v1', pointToleranceMs: 0, clockOffsetMs: 0, offsetSource: 'fixed' } }, + } }) +} diff --git a/src/biometrics/shadow/tests/frame-source.test.ts b/src/biometrics/shadow/tests/frame-source.test.ts new file mode 100644 index 000000000..fa994fc3f --- /dev/null +++ b/src/biometrics/shadow/tests/frame-source.test.ts @@ -0,0 +1,129 @@ +// @vitest-environment node +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import { openShadowFrameSource } from '../frame-source' +import { publishProspectiveFrame, subscribeProspectiveFrames } from '../../../streaming/prospectiveFrameTap' +const start = 1700000000000 +const cleanups: (() => void)[] = [] +beforeEach(() => { + vi.useFakeTimers() + vi.setSystemTime(start) +}) +afterEach(() => { + cleanups.splice(0).forEach(close => close()) + vi.useRealTimers() +}) +function setup(maximumFrames = 2, maximumBytes = 100000) { + const fail = vi.fn(), context = { activeSides: ['left'], leftOccupied: true, rightOccupied: false, pumpContextKnown: false } + const controller = new AbortController() + const source = openShadowFrameSource({ startMs: start, endMs: start + 1000, signal: controller.signal, + budgets: { maximumFrames, maximumBytes, maximumEventAgeMs: 10000, maximumFutureSkewMs: 0 }, context: () => context, fail }) + cleanups.push(source.close) + return { source, iterator: source.frames[Symbol.asyncIterator](), fail, context, controller } +} +const frame = () => ({ type: 'piezo-dual', ts: start / 1000, freq: 500, left1: [-2147483648, 0, 2147483647], right1: [1, -2, 3] }) +function publish(value = frame(), backlog = false) { + publishProspectiveFrame(value, backlog, Date.now()) +} +it('snapshots signed int32 waveforms and arrival context without retaining mutable stream data', async () => { + const { iterator, context } = setup() + const value = frame() + publish(value) + value.left1[0] = 99 + context.activeSides[0] = 'right' + const row = (await iterator.next()).value + expect(row).toMatchObject({ type: 'frame', observedAtMs: start, record: { type: 'piezo-dual', freq: 500 }, context: { activeSides: ['left'] } }) + const bytes = Buffer.from(row.record.left1, 'base64') + expect([0, 4, 8].map(offset => bytes.readInt32LE(offset))).toEqual([-2147483648, 0, 2147483647]) +}) +it('excludes backlog and unrelated sensors from prospective evidence', async () => { + const { iterator, fail } = setup() + publish(frame(), true) + publishProspectiveFrame({ type: 'capSense' }, false, start) + const next = iterator.next() + publish() + expect((await next).value.record.type).toBe('piezo-dual') + expect(fail).not.toHaveBeenCalled() +}) +it('preserves only the bounded pump fields used by the shared gate', async () => { + const { iterator } = setup() + publishProspectiveFrame({ type: 'frzHealth', ts: start / 1000, left: { pumpRpm: 0, pump: { rpm: '1200', duty: 30 }, unrelated: 'discard' }, right: { duty: 0 } }, false, start) + expect((await iterator.next()).value.record).toEqual({ type: 'frzHealth', ts: start / 1000, left: { pumpRpm: 0, pump: { rpm: '1200', duty: 30 } }, right: { duty: 0 } }) +}) +it.each(['frames', 'bytes'])('fails and closes on %s queue overflow', async (kind) => { + const { iterator, fail } = setup(1, kind === 'bytes' ? 1 : 100000) + publish() + if (kind === 'frames') publish() + expect(fail).toHaveBeenCalledOnce() + expect((await iterator.next()).done).toBe(true) + publish() + expect(fail).toHaveBeenCalledOnce() +}) +it.each([Array(5001).fill(0), [2147483648], [1.5], [NaN]])('rejects malformed or oversized channel before queueing', async (left1) => { + const { iterator, fail } = setup() + publish({ ...frame(), left1 }) + expect(fail).toHaveBeenCalledOnce() + expect((await iterator.next()).done).toBe(true) +}) +it('ends pending reads and unsubscribes on abort', async () => { + const { iterator, controller, fail } = setup() + const pending = iterator.next() + controller.abort() + expect((await pending).done).toBe(true) + publish() + expect(fail).not.toHaveBeenCalled() +}) +it('reports unread input at collection end instead of silently dropping it', async () => { + const { iterator, fail } = setup() + publish() + await vi.advanceTimersByTimeAsync(1000) + expect(fail).toHaveBeenCalledOnce() + expect((await iterator.next()).done).toBe(true) +}) +it('ends an empty collection normally and rejects concurrent consumers', async () => { + const { source, iterator, fail } = setup() + expect(() => source.frames[Symbol.asyncIterator]()).toThrow(/one consumer/) + const pending = iterator.next() + await expect(iterator.next()).rejects.toThrow(/Concurrent/) + await vi.advanceTimersByTimeAsync(1000) + expect((await pending).done).toBe(true) + expect(fail).not.toHaveBeenCalled() +}) +it('isolates subscriber faults so live ingestion and other subscribers continue', () => { + const failed = vi.fn(), healthy = vi.fn() + cleanups.push(subscribeProspectiveFrames({ frame: () => { + throw new Error('consumer failed') + }, failed })) + cleanups.push(subscribeProspectiveFrames({ frame: healthy, failed: () => {} })) + expect(() => publish()).not.toThrow() + publish() + expect(failed).toHaveBeenCalledOnce() + expect(healthy).toHaveBeenCalledTimes(2) +}) +it('unwinds subscriptions and timers when subscriber admission fails', () => { + for (let i = 0; i < 4; i++) cleanups.push(subscribeProspectiveFrames({ frame: () => {}, failed: () => {} })) + expect(() => setup()).toThrow(/subscriber limit/) + expect(vi.getTimerCount()).toBe(0) +}) +it.each([-20, 1])('fails on stale/future event timestamps (%s seconds)', async (seconds) => { + const { iterator, fail } = setup() + publish({ ...frame(), ts: start / 1000 + seconds }) + expect(fail).toHaveBeenCalledOnce() + expect((await iterator.next()).done).toBe(true) +}) +it('releases queue byte capacity when a frame is consumed', async () => { + const { iterator, fail } = setup(1, 1000) + publish() + await iterator.next() + publish() + expect((await iterator.next()).done).toBe(false) + expect(fail).not.toHaveBeenCalled() +}) +it('isolates a failing consumer notification from the collection timer', async () => { + const { iterator, fail } = setup() + fail.mockImplementation(() => { + throw new Error('consumer failed') + }) + publish() + await expect(vi.advanceTimersByTimeAsync(1000)).resolves.toBeDefined() + expect((await iterator.next()).done).toBe(true) +}) diff --git a/src/biometrics/shadow/tests/health-monitor.test.ts b/src/biometrics/shadow/tests/health-monitor.test.ts new file mode 100644 index 000000000..1d2f46a17 --- /dev/null +++ b/src/biometrics/shadow/tests/health-monitor.test.ts @@ -0,0 +1,102 @@ +// @vitest-environment node +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import { startShadowHealthMonitor, openShadowHealthMonitor } from '../health-monitor' +import { collectAdmissionSample } from '../../backtest/admission' +import type * as Admission from '../../backtest/admission' +vi.mock('../../backtest/admission', async original => ({ ...await original(), collectAdmissionSample: vi.fn() })) +const now = 1700000000000 +const policy = { qualificationId: 'test', minimumAvailableMemoryBytes: 100, minimumFreeDiskBytes: 100, + maximumLoadPerCpu: 1, maximumEventLoopDelayMs: 100, maximumPiezoAgeMs: 1000, maximumTemperatureC: 70, maximumSampleAgeMs: 1000 } +const sample = () => ({ measuredAtMs: Date.now(), availableMemoryBytes: 1000, freeDiskBytes: 1000, + loadPerCpu: 0.1, eventLoopDelayMs: 1, piezoReceivedAtMs: Date.now(), temperatureC: 40 }) +beforeEach(() => { + vi.useFakeTimers() + vi.setSystemTime(now) +}) +afterEach(() => { + vi.clearAllTimers() + vi.useRealTimers() + vi.clearAllMocks() +}) +it('refreshes real collector arguments and isolates published snapshots', async () => { + vi.mocked(collectAdmissionSample).mockImplementation(async (directory, piezo) => { + expect(directory).toBe('/trusted/data') + expect(piezo()).toBe(Date.now() - 10) + return sample() + }) + const monitor = await openShadowHealthMonitor('/trusted/data', policy, () => Date.now() - 10) + const first = monitor.read() + expect(first).not.toBeNull() + if (first) first.availableMemoryBytes = 0 + expect(monitor.read()?.availableMemoryBytes).toBe(1000) + await vi.advanceTimersByTimeAsync(500) + expect(collectAdmissionSample).toHaveBeenCalledTimes(2) + expect(monitor.read()?.measuredAtMs).toBe(now + 500) + monitor.close() + expect(monitor.read()).toBeNull() + await vi.advanceTimersByTimeAsync(1000) + expect(collectAdmissionSample).toHaveBeenCalledTimes(2) +}) +it('makes a stalled refresh terminal without overlapping I/O or accepting its late result', async () => { + let resolve!: (value: ReturnType) => void + const collect = vi.fn().mockResolvedValueOnce(sample()).mockImplementationOnce(() => new Promise((done) => { + resolve = done + })) + const monitor = await startShadowHealthMonitor(collect, policy) + await vi.advanceTimersByTimeAsync(1000) + expect(monitor.read()).toBeNull() + resolve(sample()) + await vi.advanceTimersByTimeAsync(10000) + expect(monitor.read()).toBeNull() + expect(collect).toHaveBeenCalledTimes(2) +}) +it('rejects initial timeout and cancellation without waiting for blocked collection', async () => { + const pending = startShadowHealthMonitor(() => new Promise(() => {}), policy) + const rejected = expect(pending).rejects.toThrow(/deadline/) + await vi.advanceTimersByTimeAsync(500) + await rejected + const abort = new AbortController() + const task = startShadowHealthMonitor(() => new Promise(() => {}), policy, abort.signal) + const cancelled = expect(task).rejects.toThrow(/closed/) + abort.abort() + await cancelled +}) +it('denies stale or unhealthy evidence and clears previous health on failed refresh', async () => { + const collect = vi.fn().mockResolvedValueOnce(sample()).mockResolvedValueOnce(null).mockResolvedValueOnce({ ...sample(), temperatureC: 100 }) + const monitor = await startShadowHealthMonitor(collect, policy) + vi.setSystemTime(now + 1001) + expect(monitor.read()).toBeNull() + await vi.advanceTimersByTimeAsync(500) + expect(monitor.read()).toBeNull() + await vi.advanceTimersByTimeAsync(500) + expect(monitor.read()).toBeNull() + monitor.close() +}) +it('closes on collector rejection and observes late rejected promises after timeout', async () => { + const monitor = await startShadowHealthMonitor(vi.fn().mockResolvedValueOnce(sample()).mockRejectedValueOnce(new Error('I/O failed')), policy) + await vi.advanceTimersByTimeAsync(500) + expect(monitor.read()).toBeNull() + expect(vi.getTimerCount()).toBe(0) + let reject!: (error: Error) => void + const pending = startShadowHealthMonitor(() => new Promise((_resolve, fail) => { + reject = fail + }), policy) + const rejected = expect(pending).rejects.toThrow(/deadline/) + await vi.advanceTimersByTimeAsync(500) + await rejected + reject(new Error('late failure')) + await Promise.resolve() +}) +it('does not collect after pre-abort and clears scheduled refresh on later abort', async () => { + const pre = new AbortController(), collect = vi.fn().mockResolvedValue(sample()) + pre.abort() + await expect(startShadowHealthMonitor(collect, policy, pre.signal)).rejects.toThrow(/closed/) + expect(collect).not.toHaveBeenCalled() + const abort = new AbortController() + const monitor = await startShadowHealthMonitor(collect, policy, abort.signal) + abort.abort() + expect(monitor.read()).toBeNull() + expect(vi.getTimerCount()).toBe(0) + await vi.advanceTimersByTimeAsync(2000) + expect(collect).toHaveBeenCalledTimes(1) +}) diff --git a/src/biometrics/shadow/tests/host.test.ts b/src/biometrics/shadow/tests/host.test.ts new file mode 100644 index 000000000..fa89709b9 --- /dev/null +++ b/src/biometrics/shadow/tests/host.test.ts @@ -0,0 +1,178 @@ +// @vitest-environment node +import { beforeEach, expect, it, vi } from 'vitest' +import { resolveShadowProfiles } from '../profiles' +import { dataHash } from '../../algorithms/canonical' +import { openPlannedShadowHost, openProfiledShadowHost, runPlannedShadowAttempt, runProfiledShadowAttempt } from '../host' +import { openShadowActiveDatabase } from '../active-database' +import { openShadowHealthMonitor } from '../health-monitor' +import { installedShadowAdmitter, readInstalledShadowQualifications } from '../admission' +import { plannedShadowSourceFactory } from '../planned-sources' +import { shadowRuntimeBindingReader } from '../runtime-binding' +import { runShadowAttempt } from '../attempt' +import type { ShadowStore } from '../store' +vi.mock('../profiles', () => ({ resolveShadowProfiles: vi.fn() })) +vi.mock('../active-database', () => ({ openShadowActiveDatabase: vi.fn() })) +vi.mock('../health-monitor', () => ({ openShadowHealthMonitor: vi.fn() })) +vi.mock('../admission', () => ({ installedShadowAdmitter: vi.fn(), readInstalledShadowQualifications: vi.fn() })) +vi.mock('../planned-sources', () => ({ plannedShadowSourceFactory: vi.fn() })) +vi.mock('../runtime-binding', () => ({ shadowRuntimeBindingReader: vi.fn() })) +vi.mock('../attempt', () => ({ runShadowAttempt: vi.fn() })) +const a = 'a'.repeat(64), now = 1700000000000 +function setup() { + const entry = { healthPolicy: { qualificationId: 'test' } } + const execution = { version: 'shadow-execution-policy-v1', configHashes: { left: a, right: a }, + maximumRecords: 10, maximumOutputs: 10, maximumCpuMs: 1000, maximumInputBytes: 100000, maximumOutputBytes: 100000, + budgets: { cpuSeconds: 2, addressSpaceBytes: 100000000, maximumRssBytes: 100000000, wallMs: 10000, acknowledgmentMs: 1000, idleMs: 1000, rssIntervalMs: 100, rssTimeoutMs: 100 }, + leaseMs: 1500, heartbeatMs: 200, sources: { pollMs: 50, maximumActiveRows: 10, databaseBusyTimeoutMs: 50, + frames: { maximumFrames: 10, maximumBytes: 100000, maximumEventAgeMs: 1000, maximumFutureSkewMs: 0 }, + context: { presenceStaleMs: 300000, pumpStaleMs: 5000, maximumPresenceBytes: 4096 } } } + const plan = { schema: 'biometric-shadow-plan-v1', side: 'left', subjectId: 'test', execution, startMs: now, endMs: now + 10000, + candidateConfigHash: a, activeConfigHash: a, engineBuildHash: a, qualificationHash: dataHash(entry), maximumPoints: 10, maximumBytes: 10000, metrics: ['heart_rate'], + joining: { maximumPendingPoints: 10, maximumLatenessMs: 100, drainGraceMs: 500 }, + monitoring: { expectedIntervalMs: 1000, maximumGapMs: 2000, heartbeatStaleMs: 1000, minimumCandidateCoverage: 0.8, maximumCoverageLoss: 0.1, minimumQuality: 0.8, + alignment: { version: 'reference-alignment-v1', pointToleranceMs: 0, clockOffsetMs: 0, offsetSource: 'fixed' } } } + const databases = [0, 1].map(() => ({ db: {}, close: vi.fn(), assertCurrent: vi.fn() })) + vi.mocked(openShadowActiveDatabase).mockReturnValueOnce(databases[0] as unknown as ReturnType) + .mockReturnValueOnce(databases[1] as unknown as ReturnType) + vi.mocked(readInstalledShadowQualifications).mockReturnValue({ entries: [entry] } as unknown as ReturnType) + const health = { read: vi.fn(), close: vi.fn() }, admit = vi.fn(), sources = vi.fn() + vi.mocked(openShadowHealthMonitor).mockResolvedValue(health) + vi.mocked(installedShadowAdmitter).mockReturnValue(admit) + vi.mocked(plannedShadowSourceFactory).mockReturnValue(sources) + vi.mocked(shadowRuntimeBindingReader).mockReturnValue(vi.fn()) + const installation = { workerPath: '/trusted/worker.cjs', pythonPath: '/trusted/python', modulesPath: '/trusted/modules', + configurationDatabase: '/trusted/config.db', biometricsDatabase: '/trusted/bio.db', presenceFile: '/trusted/presence.json', + configs: { left: {}, right: {} }, expectedBindings: { left: { generation: 1, resolvedHash: a }, right: { generation: 2, resolvedHash: a } }, identity: vi.fn(), piezoReceivedAt: vi.fn(() => now) } + return { plan, installation, databases, health, admit, sources } +} +beforeEach(() => vi.resetAllMocks()) +it('composes frozen policy, both binding guards and sampled health before returning a host', async () => { + const { plan, installation, databases, health, admit, sources } = setup() + const owned = await openPlannedShadowHost(plan, installation) + expect(admit).toHaveBeenCalledOnce() + expect(openShadowActiveDatabase).toHaveBeenCalledWith('/trusted/config.db', { busyTimeoutMs: 50 }) + const bindingOptions = vi.mocked(shadowRuntimeBindingReader).mock.calls[0][2] + bindingOptions.assertCurrent() + databases.forEach(db => expect(db.assertCurrent).toHaveBeenCalledOnce()) + expect(bindingOptions.expected.right.generation).toBe(2) + expect(openShadowHealthMonitor).toHaveBeenCalledWith('/trusted', { qualificationId: 'test' }, installation.piezoReceivedAt, owned.signal) + expect(owned.host.openSources).toBe(sources) + installation.configs.left = { changed: true } + expect(owned.host.configs.left).toEqual({}) + expect(() => owned.host.admit({ ...plan, endMs: plan.endMs + 1 } as Parameters[0])).toThrow(/plan changed/) + owned.close() + owned.close() + expect(owned.signal.aborted).toBe(true) + databases.forEach(db => expect(db.close).toHaveBeenCalledOnce()) + expect(health.close).toHaveBeenCalledOnce() + expect(() => owned.host.admit(plan as Parameters[0])).toThrow(/closed/) +}) +it('unwinds the first connection when the second database fails', async () => { + const { plan, installation, databases, health } = setup() + vi.mocked(openShadowActiveDatabase).mockReset().mockReturnValueOnce(databases[0] as unknown as ReturnType) + .mockImplementationOnce(() => { throw new Error('open failed') }) + await expect(openPlannedShadowHost(plan, installation)).rejects.toThrow(/open failed/) + expect(databases[0].close).toHaveBeenCalledOnce() + expect(openShadowHealthMonitor).not.toHaveBeenCalled() + expect(health.close).not.toHaveBeenCalled() +}) +it('unwinds both connections and health when initial admission fails', async () => { + const { plan, installation, databases, health, admit } = setup() + admit.mockImplementation(() => { + throw new Error('unqualified') + }) + await expect(openPlannedShadowHost(plan, installation)).rejects.toThrow(/unqualified/) + databases.forEach(db => expect(db.close).toHaveBeenCalledOnce()) + expect(health.close).toHaveBeenCalledOnce() +}) +it('cleans up a late health handle after cancellation during startup', async () => { + const { plan, installation, databases, health } = setup() + let resolve!: (value: typeof health) => void + vi.mocked(openShadowHealthMonitor).mockImplementation(() => new Promise((done) => { + resolve = done + })) + const abort = new AbortController() + const pending = openPlannedShadowHost(plan, installation, abort.signal) + abort.abort() + resolve(health) + await expect(pending).rejects.toThrow(/cancelled/) + databases.forEach(db => expect(db.close).toHaveBeenCalledOnce()) + expect(health.close).toHaveBeenCalledOnce() +}) +it('rejects pre-cancellation and missing qualification before opening resources', async () => { + const { plan, installation } = setup() + const abort = new AbortController() + abort.abort() + await expect(openPlannedShadowHost(plan, installation, abort.signal)).rejects.toThrow(/cancelled/) + vi.mocked(readInstalledShadowQualifications).mockReturnValue({ version: 'biometric-shadow-qualifications-v1', entries: [] }) + await expect(openPlannedShadowHost(plan, installation)).rejects.toThrow(/qualification/) + expect(openShadowActiveDatabase).not.toHaveBeenCalled() +}) +it('closes host resources when the attempt rejects after startup', async () => { + const { plan, installation, databases, health } = setup() + vi.mocked(runShadowAttempt).mockRejectedValue(new Error('claim failed')) + const store = { get: () => ({ plan }) } as unknown as ShadowStore + await expect(runPlannedShadowAttempt(store, 'test', installation)).rejects.toThrow(/claim failed/) + databases.forEach(db => expect(db.close).toHaveBeenCalledOnce()) + expect(health.close).toHaveBeenCalledOnce() +}) +it('closes admission connections when health collection rejects', async () => { + const { plan, installation, databases, health } = setup() + vi.mocked(openShadowHealthMonitor).mockRejectedValue(new Error('health unavailable')) + await expect(openPlannedShadowHost(plan, installation)).rejects.toThrow(/health unavailable/) + databases.forEach(db => expect(db.close).toHaveBeenCalledOnce()) + expect(health.close).not.toHaveBeenCalled() + expect(plannedShadowSourceFactory).not.toHaveBeenCalled() +}) + +it('resolves profiles using temporary handles then reopens admission with captured generations', async () => { + const { plan, installation, databases } = setup() + const capture = [0, 1].map(() => ({ db: {}, close: vi.fn(), assertCurrent: vi.fn() })) + vi.mocked(openShadowActiveDatabase).mockReset() + for (const db of [...capture, ...databases]) vi.mocked(openShadowActiveDatabase).mockReturnValueOnce(db as unknown as ReturnType) + vi.mocked(resolveShadowProfiles).mockReturnValue({ revisionId: null, configs: installation.configs, + expectedBindings: installation.expectedBindings, heartbeatAtMs: now } as ReturnType) + vi.mocked(runShadowAttempt).mockImplementation(async () => { + capture.forEach(db => expect(db.close).toHaveBeenCalledOnce()) + databases.forEach(db => expect(db.close).not.toHaveBeenCalled()) + throw new Error('test attempt finished') + }) + const store = { get: () => ({ plan }) } as unknown as ShadowStore + await expect(runProfiledShadowAttempt(store, 'test', installation, null)).rejects.toThrow(/test attempt finished/) + expect(resolveShadowProfiles).toHaveBeenCalledWith(capture[0].db, capture[1].db, plan, null, expect.any(Function)) + databases.forEach(db => expect(db.close).toHaveBeenCalledOnce()) +}) + +it('checks both captured database identities during profile resolution and closes them on rejection', async () => { + const { plan, installation, databases } = setup() + vi.mocked(resolveShadowProfiles).mockImplementation((_config, _bio, _plan, _revision, assertCurrent) => { + assertCurrent() + throw new Error('profile incompatible') + }) + await expect(openProfiledShadowHost(plan, installation, null)).rejects.toThrow('profile incompatible') + databases.forEach((db) => { + expect(db.assertCurrent).toHaveBeenCalledOnce() + expect(db.close).toHaveBeenCalledOnce() + }) + expect(openShadowHealthMonitor).not.toHaveBeenCalled() + expect(runShadowAttempt).not.toHaveBeenCalled() +}) + +it('unwinds profile capture if the biometrics database cannot be opened', async () => { + const { plan, installation, databases } = setup() + vi.mocked(openShadowActiveDatabase).mockReset().mockReturnValueOnce(databases[0] as unknown as ReturnType) + .mockImplementationOnce(() => { throw new Error('open failed') }) + await expect(openProfiledShadowHost(plan, installation, null)).rejects.toThrow('open failed') + expect(databases[0].close).toHaveBeenCalledOnce() + expect(resolveShadowProfiles).not.toHaveBeenCalled() +}) + +it('requires execution policy before opening either kind of host and honors pre-cancellation', async () => { + const { plan, installation } = setup() + const withoutExecution: Partial = { ...plan } + delete withoutExecution.execution + await expect(openPlannedShadowHost(withoutExecution, installation)).rejects.toThrow(/execution policy/) + await expect(openProfiledShadowHost(withoutExecution, installation, null)).rejects.toThrow(/execution policy/) + await expect(openProfiledShadowHost(plan, installation, null, AbortSignal.abort())).rejects.toThrow(/cancelled/) + expect(openShadowActiveDatabase).not.toHaveBeenCalled() +}) diff --git a/src/biometrics/shadow/tests/installed-attempt.test.ts b/src/biometrics/shadow/tests/installed-attempt.test.ts new file mode 100644 index 000000000..5f5b93204 --- /dev/null +++ b/src/biometrics/shadow/tests/installed-attempt.test.ts @@ -0,0 +1,120 @@ +import { withBiometricBackgroundSlot } from '../../background-admission' +// @vitest-environment node +import { beforeEach, expect, it, vi } from 'vitest' +import { dataHash } from '../../algorithms/canonical' +import { inspectInstalledShadowHost, loadInstalledShadowHost } from '../installed-host' +import { installedShadowIdentity } from '../device-identity' +import { openShadowStateDatabase } from '../active-database' +import { runInstalledShadowAttempt, reconcileInstalledShadowRuns, pendingInstalledShadowRun } from '../installed-attempt' +vi.mock('../installed-host', () => ({ inspectInstalledShadowHost: vi.fn(), loadInstalledShadowHost: vi.fn() })) +vi.mock('../device-identity', () => ({ installedShadowIdentity: vi.fn() })) +vi.mock('../active-database', () => ({ openShadowStateDatabase: vi.fn() })) +vi.mock('@/src/streaming/piezoStream', () => ({ getSensorFrameTimes: () => ({ 'piezo-dual': 1234 }) })) +const runId = '11111111-1111-4111-8111-111111111111' +const paths = { workerPath: '/trusted/shadow-host.cjs', pythonPath: '/trusted/python', modulesPath: '/trusted/modules', + configurationDatabase: '/trusted/config.db', biometricsDatabase: '/trusted/bio.db', presenceFile: '/trusted/presence' } +function setup() { + const entry = { qualificationId: 'test' } + const plan = { expectedBindings: { left: { generation: 1, resolvedHash: 'a'.repeat(64) }, right: { generation: 1, resolvedHash: 'a'.repeat(64) } }, candidateRevisionId: null as string | null | undefined, qualificationHash: dataHash(entry), execution: { sources: { databaseBusyTimeoutMs: 50 } } } + const get = vi.fn(() => ({ plan })), execute = vi.fn().mockResolvedValue('completed') + const registry = vi.fn(() => ({ entries: [entry] })), guard = vi.fn() + const recover = vi.fn(() => ({ scanned: 1, recovered: [], nextCursor: null })) + const pending = vi.fn(() => ({ run: { runId, revisionId: null }, nextCursor: runId })) + class Store { + pending = pending + get = get + reconcileExpired = recover + } + const inspected = { api: { ShadowStore: Store, runProfiledShadowAttempt: execute, readInstalledShadowQualifications: registry }, assertCurrent: guard } + vi.mocked(inspectInstalledShadowHost).mockResolvedValue(inspected as unknown as Awaited>) + vi.mocked(loadInstalledShadowHost).mockReturnValue(inspected as unknown as ReturnType) + const database = { db: {}, close: vi.fn(), assertCurrent: vi.fn() } + vi.mocked(openShadowStateDatabase).mockReturnValue(database as unknown as ReturnType) + const identity = vi.fn(() => ({ hardwareId: 'test', sampleRateHz: 500 as const, engineBuildHash: 'a'.repeat(64), pythonBuildHash: 'b'.repeat(64) })) + vi.mocked(installedShadowIdentity).mockReturnValue(identity) + return { plan, execute, registry, guard, database, identity, recover, pending } +} +beforeEach(() => vi.resetAllMocks()) +it('executes the loaded package with real frame-clock wiring and guarded state ownership', async () => { + const { execute, guard, database, identity } = setup() + execute.mockImplementation(async (_store, id, installation, revision) => { + expect(id).toBe(runId) + expect(revision).toBeNull() + expect(installation.piezoReceivedAt()).toBe(1234) + expect(installation.identity()).toMatchObject({ hardwareId: 'test', sampleRateHz: 500 }) + expect(database.close).not.toHaveBeenCalled() + return 'completed' + }) + expect(await runInstalledShadowAttempt(runId, null, paths, {}, 50)).toBe('completed') + expect(guard).toHaveBeenCalledOnce() + expect(database.assertCurrent).toHaveBeenCalledTimes(2) + expect(identity).toHaveBeenCalledTimes(2) + expect(database.close).toHaveBeenCalledOnce() +}) +it.each(['qualification', 'timeout', 'identity', 'execution'])('closes state and refuses invalid %s', async (mode) => { + const { registry, plan, identity, execute, database } = setup() + if (mode === 'qualification') registry.mockReturnValue({ entries: [] }) + if (mode === 'timeout') plan.execution.sources.databaseBusyTimeoutMs = 49 + if (mode === 'identity') identity.mockImplementation(() => { + throw new Error('identity unavailable') + }) + if (mode === 'execution') execute.mockRejectedValue(new Error('worker failed')) + await expect(runInstalledShadowAttempt(runId, null, paths, {}, 50)).rejects.toThrow() + expect(database.close).toHaveBeenCalledOnce() + if (mode !== 'execution') expect(execute).not.toHaveBeenCalled() +}) +it('does not open state after cancelled inspection', async () => { + setup() + const abort = new AbortController() + const pending = runInstalledShadowAttempt(runId, null, paths, {}, 50, abort.signal) + abort.abort() + await expect(pending).rejects.toThrow(/cancelled/) + expect(openShadowStateDatabase).not.toHaveBeenCalled() +}) + +it('reconciles through the installed store without starting probes, hardware checks or workers', () => { + const { recover, execute, database, guard } = setup() + expect(reconcileInstalledShadowRuns(paths, {}, 50, { maximumRuns: 10 })).toEqual({ scanned: 1, recovered: [], nextCursor: null }) + expect(recover).toHaveBeenCalledWith({ maximumRuns: 10 }) + expect(database.assertCurrent).toHaveBeenCalledTimes(2) + expect(guard).toHaveBeenCalledTimes(2) + expect(database.close).toHaveBeenCalledOnce() + expect(inspectInstalledShadowHost).not.toHaveBeenCalled() + expect(installedShadowIdentity).not.toHaveBeenCalled() + expect(execute).not.toHaveBeenCalled() +}) + +it.each([undefined, '22222222-2222-4222-8222-222222222222'])('refuses missing or substituted saved revision %s', async (revision) => { + const { plan, execute, database } = setup() + plan.candidateRevisionId = revision + await expect(runInstalledShadowAttempt(runId, null, paths, {}, 50)).rejects.toThrow('revision differs') + expect(execute).not.toHaveBeenCalled() + expect(database.close).toHaveBeenCalledOnce() +}) +it('reads pending identity from the installed store without worker inspection', () => { + const { pending, database } = setup() + expect(pendingInstalledShadowRun(paths, {}, 50, { maximumRuns: 10 })).toEqual({ run: { runId, revisionId: null }, nextCursor: runId }) + expect(pending).toHaveBeenCalledWith({ maximumRuns: 10 }) + expect(inspectInstalledShadowHost).not.toHaveBeenCalled() + expect(database.close).toHaveBeenCalledOnce() +}) + +it('defers before inspection or claim while coordinator work owns application capacity', async () => { + const { execute, database } = setup() + let finish!: () => void + const busy = withBiometricBackgroundSlot('work', () => new Promise((resolve) => { + finish = resolve + })) + try { + await expect(runInstalledShadowAttempt(runId, null, paths, {}, 50)).rejects.toMatchObject({ code: 'RESOURCE_LIMIT' }) + expect(inspectInstalledShadowHost).not.toHaveBeenCalled() + expect(openShadowStateDatabase).not.toHaveBeenCalled() + expect(execute).not.toHaveBeenCalled() + } + finally { + finish() + await busy + } + expect(await runInstalledShadowAttempt(runId, null, paths, {}, 50)).toBe('completed') + expect(database.close).toHaveBeenCalledOnce() +}) diff --git a/src/biometrics/shadow/tests/installed-host.test.ts b/src/biometrics/shadow/tests/installed-host.test.ts new file mode 100644 index 000000000..a1b2657f1 --- /dev/null +++ b/src/biometrics/shadow/tests/installed-host.test.ts @@ -0,0 +1,48 @@ +// @vitest-environment node +import { mkdtempSync, rmSync, writeFileSync, symlinkSync, renameSync } from 'node:fs' +import { createRequire } from 'node:module' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, expect, it } from 'vitest' +import { loadInstalledShadowHost } from '../installed-host' +const directories: string[] = [] +const fixture = `module.exports = { shadowHostApiVersion: 'shadow-host-api-v1', runProfiledShadowAttempt() {}, openProfiledShadowHost() {}, prepareShadowPlan() {}, ShadowStore() {}, +shadowEngineBuildIdentity() {}, readInstalledShadowQualifications() {}, shadowPlanSchema: {}, shadowStoragePolicySchema: {} };` +function file(contents = fixture) { + const directory = mkdtempSync(join(tmpdir(), 'installed-shadow-')) + directories.push(directory) + const path = join(directory, 'shadow-host.cjs') + writeFileSync(path, contents) + return path +} +afterEach(() => { + delete (globalThis as typeof globalThis & { __sleepypodInstalledShadowHostV1?: unknown }).__sleepypodInstalledShadowHostV1 + directories.splice(0).forEach(path => rmSync(path, { recursive: true, force: true })) +}) +it.each(['symlink', 'oversize', 'directory', 'missing'])('rejects a %s artifact before executing it', (mode) => { + const path = file() + if (mode === 'oversize') writeFileSync(path, 'x'.repeat(16 * 1024 * 1024 + 1)) + if (mode === 'symlink') { + renameSync(path, path + '.target') + symlinkSync(path + '.target', path) + } + if (mode === 'missing') rmSync(path) + expect(() => loadInstalledShadowHost(mode === 'directory' ? directories[0] : path)).toThrow() +}) +it('refuses preloaded Node cache entries with no verified loading history', () => { + const path = file() + createRequire(import.meta.url)(path) + expect(() => loadInstalledShadowHost(path)).toThrow(/without installed identity/) +}) +it('refuses incomplete or unsupported API exports', () => { + const path = file('module.exports = { shadowHostApiVersion: \'other\' };') + expect(() => loadInstalledShadowHost(path)).toThrow(/API/) +}) +it('rejects replacement with identical bytes and a second installation path', () => { + const path = file(), loaded = loadInstalledShadowHost(path) + loaded.assertCurrent() + expect(() => loadInstalledShadowHost(file())).toThrow(/restart/) + renameSync(path, path + '.old') + writeFileSync(path, fixture) + expect(() => loaded.assertCurrent()).toThrow(/restart/) +}) diff --git a/src/biometrics/shadow/tests/preparation.test.ts b/src/biometrics/shadow/tests/preparation.test.ts new file mode 100644 index 000000000..e7806636e --- /dev/null +++ b/src/biometrics/shadow/tests/preparation.test.ts @@ -0,0 +1,91 @@ +// @vitest-environment node +import type Database from 'better-sqlite3' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import { configHash, dataHash } from '../../algorithms/canonical' +import { defaultConfig, manifestHash } from '../../algorithms/registry' +import { captureShadowProfiles } from '../profiles' +import { readInstalledShadowQualifications } from '../admission' +import { prepareShadowPlan } from '../preparation' +import { configuredShadowTemplates, prepareInstalledShadow } from '../installed-preparation' +import { openShadowActiveDatabase } from '../active-database' +import { loadInstalledShadowHost } from '../installed-host' +import { submissionFixture, submissionTime } from './fixtures/submission' +vi.mock('../profiles', () => ({ captureShadowProfiles: vi.fn() })) +vi.mock('../admission', () => ({ readInstalledShadowQualifications: vi.fn() })) +vi.mock('../active-database', () => ({ openShadowActiveDatabase: vi.fn() })) +vi.mock('../installed-host', () => ({ loadInstalledShadowHost: vi.fn() })) + +const paths = { workerPath: '/installed/shadow-host.cjs', pythonPath: '/installed/python', modulesPath: '/installed/modules', + configurationDatabase: '/installed/config.db', biometricsDatabase: '/installed/bio.db', presenceFile: '/run/presence' } +const db = {} as Database.Database +function setup() { + const { plan } = submissionFixture(), hash = configHash(defaultConfig) + plan.execution.configHashes = { left: hash, right: hash } + const entry = { manifestHash, executionPolicyHash: dataHash(plan.execution), activeConfigHash: hash, engineBuildHash: 'b'.repeat(64) } + const template = { id: 'test', label: 'Synthetic fixture', maximumWindowMs: 9000, qualificationHash: dataHash(entry), + execution: plan.execution, joining: plan.joining, monitoring: plan.monitoring, metrics: plan.metrics, + maximumPoints: plan.maximumPoints, maximumBytes: plan.maximumBytes } + const request = { templateId: 'test', side: 'left' as const, subjectId: 'sleeper', candidateRevisionId: null, startMs: submissionTime + 1000, endMs: submissionTime + 9000 } + const captured = { revisionId: null, configs: { left: defaultConfig, right: defaultConfig }, + expectedBindings: { left: { generation: 4, resolvedHash: hash }, right: { generation: 5, resolvedHash: hash } }, heartbeatAtMs: submissionTime, + candidateConfigHash: hash, activeConfigHash: hash } + vi.mocked(captureShadowProfiles).mockReturnValue(captured) + vi.mocked(readInstalledShadowQualifications).mockReturnValue({ entries: [entry] } as unknown as ReturnType) + const config = { version: 'shadow-service-configuration-v1', pythonPath: paths.pythonPath, modulesPath: paths.modulesPath, presenceFile: paths.presenceFile, + storagePolicy: { maximumRuns: 10, maximumPoints: 100, maximumBytes: 100000 }, busyTimeoutMs: 50, recoveryIntervalMs: 1000, recoveryMaximumRuns: 10, + planTemplates: [template] } + vi.stubEnv('SLEEPYPOD_BIOMETRICS_SHADOW_SERVICE', JSON.stringify(config)) + return { template, request, entry, captured, config } +} +beforeEach(() => vi.resetAllMocks()) +afterEach(() => vi.unstubAllEnvs()) + +it('derives candidate/baseline identities and generations from server capture without admitting execution', () => { + const { request, template, captured, entry } = setup(), guard = vi.fn() + const prepared = prepareShadowPlan(db, db, request, template, paths.workerPath, guard, submissionTime) + expect(prepared).toMatchObject({ status: 'prepared_not_admitted', plan: { candidateRevisionId: null, + expectedBindings: captured.expectedBindings, candidateConfigHash: captured.candidateConfigHash, activeConfigHash: captured.activeConfigHash, engineBuildHash: entry.engineBuildHash } }) + expect(prepared.planHash).toBe(dataHash(prepared.plan)) + expect(captureShadowProfiles).toHaveBeenCalledWith(db, db, 'left', null, guard, submissionTime) +}) + +it.each(['past', 'duration', 'wall', 'qualification', 'candidate', 'baseline'])('refuses incompatible preparation: %s', (failure) => { + const { request, template, captured } = setup() + if (failure === 'past') request.startMs = submissionTime - 1 + if (failure === 'duration') request.endMs += 100000 + if (failure === 'wall') template.execution.budgets.wallMs = 1000 + if (failure === 'qualification') template.qualificationHash = 'f'.repeat(64) + if (failure === 'candidate') captured.configs.left = { ...defaultConfig, version: 'invalid' } as unknown as typeof defaultConfig + if (failure === 'baseline') captured.activeConfigHash = 'f'.repeat(64) + expect(() => prepareShadowPlan(db, db, request, template, paths.workerPath, () => {}, submissionTime)).toThrow() +}) + +it('lists only template metadata and owns readonly preparation handles on success and failure', () => { + const { request, template } = setup() + expect(configuredShadowTemplates()).toEqual({ items: [{ id: 'test', label: 'Synthetic fixture', maximumWindowMs: 9000 }] }) + const handles = [0, 1].map(() => ({ db, close: vi.fn(), assertCurrent: vi.fn() })) + const prepare = vi.fn(() => ({ status: 'prepared_not_admitted' })), guard = vi.fn() + vi.mocked(loadInstalledShadowHost).mockReturnValue({ api: { prepareShadowPlan: prepare }, assertCurrent: guard } as unknown as ReturnType) + for (const failed of [false, true]) { + vi.mocked(openShadowActiveDatabase).mockReturnValueOnce(handles[0]).mockReturnValueOnce(handles[1]) + if (failed) prepare.mockImplementationOnce(() => { + throw new Error('changed') + }) + if (failed) expect(() => prepareInstalledShadow(request, paths, 50)).toThrow('changed') + else expect(prepareInstalledShadow(request, paths, 50)).toEqual({ status: 'prepared_not_admitted' }) + } + expect(prepare).toHaveBeenCalledWith(db, db, request, template, paths.workerPath, expect.any(Function)) + handles.forEach(handle => expect(handle.close).toHaveBeenCalledTimes(2)) +}) + +it('refuses templates that exceed deployment quotas or duplicate an ID', () => { + const { config } = setup() + config.storagePolicy.maximumBytes = 1 + vi.stubEnv('SLEEPYPOD_BIOMETRICS_SHADOW_SERVICE', JSON.stringify(config)) + expect(configuredShadowTemplates).toThrow('storage policy') + config.storagePolicy.maximumBytes = 100000 + config.planTemplates.push(config.planTemplates[0]) + vi.stubEnv('SLEEPYPOD_BIOMETRICS_SHADOW_SERVICE', JSON.stringify(config)) + expect(configuredShadowTemplates).toThrow('Duplicate') + expect(openShadowActiveDatabase).not.toHaveBeenCalled() +}) diff --git a/src/biometrics/shadow/tests/process-cleanup.test.ts b/src/biometrics/shadow/tests/process-cleanup.test.ts new file mode 100644 index 000000000..b87ea6a31 --- /dev/null +++ b/src/biometrics/shadow/tests/process-cleanup.test.ts @@ -0,0 +1,78 @@ +// @vitest-environment node +import { afterEach, expect, it, vi } from 'vitest' +import { confirmShadowProcessCleanup } from '../process-cleanup' + +afterEach(() => { + vi.restoreAllMocks() + vi.useRealTimers() +}) +const missing = () => { + throw Object.assign(new Error('missing'), { code: 'ESRCH' }) +} +it('requires disappearance of descendants even after the direct child closes', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'performance'] }) + const kill = vi.spyOn(process, 'kill').mockReturnValue(true) + let settled = false + const result = confirmShadowProcessCleanup(123, () => true).then((value) => { + settled = true + return value + }) + await vi.advanceTimersByTimeAsync(100) + expect(settled).toBe(false) + kill.mockImplementation(missing) + await vi.advanceTimersByTimeAsync(25) + expect(await result).toBe(true) + expect(kill.mock.calls.every(args => args[0] === -123 && args[1] === 0)).toBe(true) +}) +it('requires the close event even after the group disappears', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'performance'] }) + vi.spyOn(process, 'kill').mockImplementation(missing) + let closed = false, settled = false + const result = confirmShadowProcessCleanup(123, () => closed).then((value) => { + settled = true + return value + }) + await vi.advanceTimersByTimeAsync(100) + expect(settled).toBe(false) + closed = true + await vi.advanceTimersByTimeAsync(25) + expect(await result).toBe(true) +}) +it('bounds confirmation when a group still exists and when close is missing', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'performance'] }) + const kill = vi.spyOn(process, 'kill').mockReturnValue(true) + const alive = confirmShadowProcessCleanup(123, () => true) + await vi.advanceTimersByTimeAsync(5000) + expect(await alive).toBe(false) + kill.mockImplementation(missing) + const unclosed = confirmShadowProcessCleanup(123, () => false) + await vi.advanceTimersByTimeAsync(5000) + expect(await unclosed).toBe(false) + expect(vi.getTimerCount()).toBe(0) +}) +it('treats probe denial as unknown and never signals invalid or missing pids', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'performance'] }) + const kill = vi.spyOn(process, 'kill').mockImplementation(() => { + throw Object.assign(new Error('denied'), { code: 'EPERM' }) + }) + const denied = confirmShadowProcessCleanup(123, () => true) + await vi.advanceTimersByTimeAsync(5000) + expect(await denied).toBe(false) + kill.mockClear() + for (const pid of [0, -1, NaN, Infinity, 1.5]) expect(await confirmShadowProcessCleanup(pid, () => true)).toBe(false) + expect(await confirmShadowProcessCleanup(undefined, () => true)).toBe(true) + expect(kill).not.toHaveBeenCalled() +}) + +it('retries transient unknown observations but still requires explicit absence', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'performance'] }) + const kill = vi.spyOn(process, 'kill').mockImplementationOnce(() => { + throw Object.assign(new Error('reaping'), { code: 'EPERM' }) + }).mockImplementation(missing) + const result = confirmShadowProcessCleanup(123, () => true) + expect(kill).toHaveBeenCalledOnce() + await vi.advanceTimersByTimeAsync(25) + expect(await result).toBe(true) + expect(kill).toHaveBeenCalledTimes(2) + expect(vi.getTimerCount()).toBe(0) +}) diff --git a/src/biometrics/shadow/tests/python-installation.test.ts b/src/biometrics/shadow/tests/python-installation.test.ts new file mode 100644 index 000000000..85c9b0200 --- /dev/null +++ b/src/biometrics/shadow/tests/python-installation.test.ts @@ -0,0 +1,61 @@ +// @vitest-environment node +import { copyFileSync, mkdirSync, mkdtempSync, rmSync, writeFileSync, renameSync, symlinkSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { dirname, join, resolve } from 'node:path' +import { afterEach, expect, it } from 'vitest' +import { probeShadowBuild } from '../build' +import { guardShadowPythonInstallation } from '../python-installation' +const directories: string[] = [] +afterEach(() => directories.splice(0).forEach(path => rmSync(path, { recursive: true, force: true }))) +async function setup() { + const original = { pythonPath: resolve('modules/piezo-processor/.venv/bin/python'), modulesPath: resolve('modules') } + const build = await probeShadowBuild(original) + const root = mkdtempSync(join(tmpdir(), 'python-installation-')) + directories.push(root) + for (const relative of Object.keys(build.identity.sources)) { + const path = join(root, relative) + mkdirSync(dirname(path), { recursive: true }) + copyFileSync(join(original.modulesPath, relative), path) + } + for (const name of ['numpy', 'scipy'] as const) { + for (const kind of ['record', 'wheel'] as const) { + const path = join(root, `${name}-${kind}`) + copyFileSync(build.dependencyFiles[name][kind], path) + build.dependencyFiles[name][kind] = path + } + } + const pythonPath = join(root, 'python') + symlinkSync(original.pythonPath, pythonPath) + const location = { pythonPath, modulesPath: root } + return { build, location, source: join(root, 'common/biometrics/shadow.py') } +} +it('validates real source and wheel metadata against the probe, then detects source replacement', async () => { + const { build, location, source } = await setup() + const guard = guardShadowPythonInstallation(location, build) + guard() + renameSync(source, source + '.old') + copyFileSync(source + '.old', source) + expect(guard).toThrow(/changed/) +}) +it('refuses source content changed after probing', async () => { + const { build, location, source } = await setup() + writeFileSync(source, '# changed') + expect(() => guardShadowPythonInstallation(location, build)).toThrow(/differs from probe/) +}) +it('detects metadata mutation and executable link replacement', async () => { + const { build, location } = await setup() + const guard = guardShadowPythonInstallation(location, build) + writeFileSync(build.dependencyFiles.numpy.wheel, 'changed') + expect(guard).toThrow(/changed/) + const other = await setup() + const executableGuard = guardShadowPythonInstallation(other.location, other.build) + rmSync(other.location.pythonPath) + symlinkSync(process.execPath, other.location.pythonPath) + expect(executableGuard).toThrow(/changed/) +}) +it('refuses malformed build hashes and non-absolute dependency paths', async () => { + const { build, location } = await setup() + expect(() => guardShadowPythonInstallation(location, { ...build, hash: 'a'.repeat(64) })).toThrow(/integrity/) + build.dependencyFiles.numpy.record = 'relative/RECORD' + expect(() => guardShadowPythonInstallation(location, build)).toThrow(/absolute/) +}) diff --git a/src/biometrics/shadow/tests/runtime-binding.test.ts b/src/biometrics/shadow/tests/runtime-binding.test.ts new file mode 100644 index 000000000..46fbb191f --- /dev/null +++ b/src/biometrics/shadow/tests/runtime-binding.test.ts @@ -0,0 +1,161 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, expect, it } from 'vitest' +import * as configSchema from '../../../db/schema' +import * as bioSchema from '../../../db/biometrics-schema' +import { configHash } from '../../algorithms/canonical' +import { defaultConfig } from '../../algorithms/registry' +import { requestManualActivation } from '../../profiles/activation' +import { deliverRuntimeOutbox } from '../../profiles/runtime' +import { resolveShadowProfiles } from '../profiles' +import { ProfileStore } from '../../profiles/store' +import { shadowRuntimeBindingReader } from '../runtime-binding' +const at = 1700000000000, hash = configHash(defaultConfig) +const cleanups: (() => void)[] = [] +afterEach(() => cleanups.splice(0).forEach(fn => fn())) +function setup() { + const root = mkdtempSync(join(tmpdir(), 'shadow-binding-')) + const config = new Database(join(root, 'config.db')), bio = new Database(join(root, 'bio.db')) + const configDb = drizzle(config, { schema: configSchema }), bioDb = drizzle(bio, { schema: bioSchema }) + migrate(configDb, { migrationsFolder: 'src/db/migrations' }) + migrate(bioDb, { migrationsFolder: 'src/db/biometrics-migrations' }) + for (const side of ['left', 'right'] as const) { + requestManualActivation(configDb, { side, action: 'reset', revisionId: null, expectedGeneration: 0, + expectedResolvedHash: hash, boundary: 'immediate', requester: 'fixture', idempotencyKey: side, allowUnvalidated: true }, at - 2000) + } + deliverRuntimeOutbox(configDb, bioDb, at - 1000) + for (const side of ['left', 'right'] as const) { + for (const component of ['heart_rate', 'breathing_rate', 'hrv_index']) { + bioDb.insert(bioSchema.biometricRuntimeReceipts).values({ side, component, generation: 1, appliedHash: hash, + effectiveSampleTimeMs: at - 1000, heartbeatAtMs: at, state: 'applied' }).run() + } + } + const configReader = new Database(join(root, 'config.db'), { readonly: true }), bioReader = new Database(join(root, 'bio.db'), { readonly: true }) + cleanups.push(() => { + configReader.close() + bioReader.close() + config.close() + bio.close() + rmSync(root, { recursive: true, force: true }) + }) + const options = { side: 'left' as const, expected: { left: { generation: 1, resolvedHash: hash }, right: { generation: 1, resolvedHash: hash } }, + assertCurrent: () => {}, now: () => at } + return { configDb, config, bio, configReader, bioReader, options, read: shadowRuntimeBindingReader(configReader, bioReader, options) } +} +it('aggregates both channels without requiring an on-demand stage receipt', () => { + const { read, bio, options } = setup() + bio.prepare('UPDATE biometric_runtime_receipts SET heartbeat_at_ms = ? WHERE side = \'right\' AND component = \'hrv_index\'').run(at - 500) + options.expected.right.generation = 99 // caller mutation cannot change captured identity + expect(read()).toEqual({ side: 'left', generation: 1, resolvedHash: hash, appliedGeneration: 1, appliedHash: hash, state: 'applied', heartbeatAtMs: at - 500 }) +}) +it.each(['left', 'right'])('rejects same-hash desired resets on %s', (side) => { + const { config, read } = setup() + config.prepare('UPDATE biometric_profile_bindings SET generation = 2 WHERE side = ?').run(side) + expect(read).toThrow(/generation changed/) +}) +it.each([ + 'DELETE FROM biometric_runtime_receipts WHERE side = \'right\' AND component = \'breathing_rate\'', + 'UPDATE biometric_runtime_receipts SET state = \'pending_boundary\' WHERE component = \'hrv_index\'', + 'UPDATE biometric_runtime_receipts SET state = \'failed\' WHERE component = \'heart_rate\'', + 'UPDATE biometric_runtime_receipts SET effective_sample_time_ms = NULL WHERE component = \'hrv_index\'', + 'UPDATE biometric_runtime_receipts SET generation = 2 WHERE side = \'right\'', + 'UPDATE biometric_runtime_receipts SET applied_hash = \'wrong\' WHERE side = \'right\'', + `UPDATE biometric_runtime_receipts SET heartbeat_at_ms = ${at + 1} WHERE side = 'right'`, + `UPDATE biometric_runtime_receipts SET effective_sample_time_ms = ${at + 1} WHERE side = 'right'`, +])('rejects missing, unapplied, changed or future component evidence: %s', (sql) => { + const { bio, read } = setup() + bio.exec(sql) + expect(read).toThrow() +}) +it.each(['DELETE FROM biometric_runtime_configs WHERE side = \'right\'', + 'UPDATE biometric_runtime_configs SET resolved_hash = \'wrong\' WHERE side = \'right\'', + 'UPDATE biometric_runtime_configs SET payload = \'{}\' WHERE side = \'right\'', + 'UPDATE biometric_runtime_configs SET payload = printf(\'%070000d\', 0) WHERE side = \'right\''])('rejects unavailable or corrupt installed projection: %s', (sql) => { + const { bio, read } = setup() + bio.exec(sql) + expect(read).toThrow() +}) +it('requires read-only handles and propagates post-snapshot identity failure', () => { + const { config, bio, configReader, bioReader, options } = setup() + expect(() => shadowRuntimeBindingReader(config, bio, options)).toThrow(/read-only/) + let checks = 0 + const read = shadowRuntimeBindingReader(configReader, bioReader, { ...options, assertCurrent: () => { + if (++checks === 3) throw new Error('database replaced') + } }) + expect(read).toThrow(/database replaced/) +}) +it('refuses desired hash changes even when all old receipts remain applied', () => { + const { config, read } = setup() + config.prepare('UPDATE biometric_activation_events SET next_hash = \'changed\' WHERE side = \'right\'').run() + expect(read).toThrow(/configuration changed/) +}) +it('does not fall back to an older applied generation behind a pending receipt', () => { + const { bio, read } = setup() + bio.prepare(`INSERT INTO biometric_runtime_receipts + (side, component, generation, applied_hash, effective_sample_time_ms, heartbeat_at_ms, state) + VALUES ('right', 'heart_rate', 2, ?, NULL, ?, 'pending_boundary')`).run(hash, at) + expect(read).toThrow() +}) + +function savedPlan() { + const execution = { version: 'shadow-execution-policy-v1', configHashes: { left: hash, right: hash }, + maximumRecords: 10, maximumOutputs: 10, maximumCpuMs: 1000, maximumInputBytes: 100000, maximumOutputBytes: 100000, + budgets: { cpuSeconds: 2, addressSpaceBytes: 100000000, maximumRssBytes: 100000000, wallMs: 10000, acknowledgmentMs: 1000, idleMs: 1000, rssIntervalMs: 100, rssTimeoutMs: 100 }, + leaseMs: 1500, heartbeatMs: 200, sources: { pollMs: 50, maximumActiveRows: 10, databaseBusyTimeoutMs: 50, + frames: { maximumFrames: 10, maximumBytes: 100000, maximumEventAgeMs: 1000, maximumFutureSkewMs: 0 }, + context: { presenceStaleMs: 300000, pumpStaleMs: 5000, maximumPresenceBytes: 4096 } } } + const plan = { schema: 'biometric-shadow-plan-v1', side: 'left', subjectId: 'test', execution, startMs: at, endMs: at + 10000, + candidateConfigHash: hash, activeConfigHash: hash, engineBuildHash: hash, qualificationHash: 'a'.repeat(64), maximumPoints: 10, maximumBytes: 10000, metrics: ['heart_rate'], + joining: { maximumPendingPoints: 10, maximumLatenessMs: 100, drainGraceMs: 500 }, + monitoring: { expectedIntervalMs: 1000, maximumGapMs: 2000, heartbeatStaleMs: 1000, minimumCandidateCoverage: 0.8, maximumCoverageLoss: 0.1, minimumQuality: 0.8, + alignment: { version: 'reference-alignment-v1', pointToleranceMs: 0, clockOffsetMs: 0, offsetSource: 'fixed' } } } + return plan +} + +it('resolves the saved candidate while preserving acknowledged other-channel config', () => { + const { configDb, configReader, bioReader } = setup() + const revision = new ProfileStore(configDb).createRevision({ side: 'left', overrides: [{ path: 'heart_rate.trackingToleranceBpm', value: 20 }], + origin: 'manual', requester: 'test', idempotencyKey: 'candidate' }) + const plan = savedPlan() + plan.candidateConfigHash = revision.resolvedHash + plan.execution.configHashes.left = revision.resolvedHash + const result = resolveShadowProfiles(configReader, bioReader, plan, revision.id, () => {}, at) + expect(result.configs.left).toEqual(revision.resolved) + expect(result.configs.right).toEqual(defaultConfig) + expect(result.expectedBindings).toEqual({ left: { generation: 1, resolvedHash: hash }, right: { generation: 1, resolvedHash: hash } }) + expect(result.revisionId).toBe(revision.id) +}) +it('allows explicit default candidates but refuses unstored active defaults', () => { + const { config, configReader, bioReader } = setup() + expect(resolveShadowProfiles(configReader, bioReader, savedPlan(), null, () => {}, at).configs.left).toEqual(defaultConfig) + config.prepare('DELETE FROM biometric_profile_bindings WHERE side = \'right\'').run() + expect(() => resolveShadowProfiles(configReader, bioReader, savedPlan(), null, () => {}, at)).toThrow() +}) +it('refuses a same-hash active generation change from a queued plan', () => { + const { configReader, bioReader } = setup() + const expectedBindings = { left: { generation: 1, resolvedHash: hash }, right: { generation: 1, resolvedHash: hash } } + expect(resolveShadowProfiles(configReader, bioReader, { ...savedPlan(), expectedBindings }, null, () => {}, at).expectedBindings).toEqual(expectedBindings) + expectedBindings.right.generation = 2 + expect(() => resolveShadowProfiles(configReader, bioReader, { ...savedPlan(), expectedBindings }, null, () => {}, at)).toThrow('generations differ') +}) +it('rejects wrong-side or corrupted revisions and frozen physical-channel substitution', () => { + const { configDb, config, configReader, bioReader } = setup() + const revision = new ProfileStore(configDb).createRevision({ side: 'right', overrides: [], origin: 'manual', requester: 'test', idempotencyKey: 'candidate' }) + const plan = savedPlan() + expect(() => resolveShadowProfiles(configReader, bioReader, plan, revision.id, () => {}, at)).toThrow(/other side/) + config.prepare('UPDATE biometric_profile_revisions SET side = \'left\', resolved_hash = \'corrupt\' WHERE id = ?').run(revision.id) + expect(() => resolveShadowProfiles(configReader, bioReader, plan, revision.id, () => {}, at)).toThrow(/integrity/) + plan.execution.configHashes.right = 'b'.repeat(64) + expect(() => resolveShadowProfiles(configReader, bioReader, plan, null, () => {}, at)).toThrow(/physical-channel/) +}) +it('refuses unapplied baselines and writable profile resolution handles', () => { + const { config, bio, configReader, bioReader } = setup() + expect(() => resolveShadowProfiles(config, bio, savedPlan(), null, () => {}, at)).toThrow(/read-only/) + bio.prepare('UPDATE biometric_runtime_receipts SET state = \'pending_boundary\' WHERE side = \'right\'').run() + expect(() => resolveShadowProfiles(configReader, bioReader, savedPlan(), null, () => {}, at)).toThrow() +}) diff --git a/src/biometrics/shadow/tests/service-host.test.ts b/src/biometrics/shadow/tests/service-host.test.ts new file mode 100644 index 000000000..b83ea3df7 --- /dev/null +++ b/src/biometrics/shadow/tests/service-host.test.ts @@ -0,0 +1,55 @@ +// @vitest-environment node +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import { runInstalledShadowAttempt, reconcileInstalledShadowRuns, pendingInstalledShadowRun } from '../installed-attempt' +import { startShadowService, stopShadowService } from '../service-host' + +vi.mock('../installed-attempt', () => ({ runInstalledShadowAttempt: vi.fn(), reconcileInstalledShadowRuns: vi.fn(), pendingInstalledShadowRun: vi.fn() })) +const runId = '11111111-1111-4111-8111-111111111111' +const options = () => ({ installation: { workerPath: '/trusted/shadow-host.cjs', pythonPath: '/trusted/python', modulesPath: '/trusted/modules', + configurationDatabase: '/trusted/config.db', biometricsDatabase: '/trusted/bio.db', presenceFile: '/trusted/presence' }, +storagePolicy: { maximumRuns: 10, maximumPoints: 1000, maximumBytes: 1000000 }, busyTimeoutMs: 50, recoveryIntervalMs: 1000, recoveryMaximumRuns: 10 }) +beforeEach(() => { + vi.mocked(pendingInstalledShadowRun).mockReturnValue({ run: null, nextCursor: null }) +}) +afterEach(async () => { + await stopShadowService() + vi.resetAllMocks() +}) + +it('owns one global service, freezes deployment inputs and rejects configuration drift', async () => { + vi.mocked(reconcileInstalledShadowRuns).mockReturnValue({ scanned: 0, recovered: [], nextCursor: null }) + const raw = options(), original = structuredClone(raw) + const service = startShadowService(raw) + expect(startShadowService(options())).toBe(service) + expect(reconcileInstalledShadowRuns).toHaveBeenCalledOnce() + raw.installation.workerPath = '/different' + raw.storagePolicy.maximumRuns = 999 + expect(() => startShadowService(raw)).toThrow('restart required') + await service.run(runId, null) + expect(runInstalledShadowAttempt).toHaveBeenCalledWith(runId, null, original.installation, + original.storagePolicy, 50, expect.any(AbortSignal)) + await stopShadowService() + expect(() => service.run(runId, null)).toThrow('stopped') + expect(startShadowService(options())).not.toBe(service) +}) + +it('keeps the singleton until shutdown finishes worker cleanup', async () => { + vi.mocked(reconcileInstalledShadowRuns).mockReturnValue({ scanned: 0, recovered: [], nextCursor: null }) + let release!: () => void + vi.mocked(runInstalledShadowAttempt).mockImplementation(async (_id, _revision, _paths, _policy, _timeout, signal) => { + await new Promise((resolve) => { + release = resolve + }) + expect(signal?.aborted).toBe(true) + throw new Error('interrupted') + }) + const service = startShadowService(options()) + const failed = expect(service.run(runId, null)).rejects.toThrow('interrupted') + await Promise.resolve() + const stopping = stopShadowService() + expect(startShadowService(options())).toBe(service) + expect(() => service.run(runId, null)).toThrow('stopped') + release() + await Promise.all([stopping, failed]) + expect(startShadowService(options())).not.toBe(service) +}) diff --git a/src/biometrics/shadow/tests/service.test.ts b/src/biometrics/shadow/tests/service.test.ts new file mode 100644 index 000000000..c62f04d41 --- /dev/null +++ b/src/biometrics/shadow/tests/service.test.ts @@ -0,0 +1,176 @@ +// @vitest-environment node +import { afterEach, expect, it, vi } from 'vitest' +import { ShadowService } from '../service' + +const runId = '11111111-1111-4111-8111-111111111111' +const otherId = '22222222-2222-4222-8222-222222222222' +function setup() { + const execute = vi.fn[0]['execute']>().mockResolvedValue('complete') + const recover = vi.fn(() => ({ scanned: 0, recovered: [], nextCursor: null as string | null })) + const onRecoveryError = vi.fn() + return { execute, recover, onRecoveryError, service: new ShadowService({ execute, recover, onRecoveryError, + recoveryIntervalMs: 100, recoveryMaximumRuns: 7 }) } +} +afterEach(() => vi.useRealTimers()) + +it('schedules saved pending identity while maintenance continues during worker cleanup', async () => { + vi.useFakeTimers() + const { execute, recover } = setup() + const pending = vi.fn().mockReturnValueOnce({ run: { runId, revisionId: otherId }, nextCursor: runId }) + .mockReturnValue({ run: null, nextCursor: null }) + let release!: () => void + execute.mockImplementationOnce(async () => { + await new Promise((resolve) => { + release = resolve + }) + }) + const service = new ShadowService({ execute, recover, pending, recoveryIntervalMs: 100, recoveryMaximumRuns: 7 }) + service.start() + await Promise.resolve() + expect(execute).toHaveBeenCalledWith(runId, otherId, expect.any(AbortSignal)) + await vi.advanceTimersByTimeAsync(300) + expect(recover).toHaveBeenCalledTimes(4) + expect(pending).toHaveBeenCalledOnce() + release() + await vi.advanceTimersByTimeAsync(100) + expect(pending).toHaveBeenLastCalledWith({ maximumRuns: 7, afterId: runId }) + await service.stop() +}) + +it('retries failed queue scans and observes scheduled execution errors', async () => { + vi.useFakeTimers() + const { execute, recover } = setup() + const pending = vi.fn().mockImplementationOnce(() => { + throw new Error('database busy') + }).mockReturnValueOnce({ run: { runId, revisionId: null }, nextCursor: runId }) + .mockReturnValue({ run: null, nextCursor: null }) + const onExecutionError = vi.fn(() => { + throw new Error('logging failed') + }) + execute.mockRejectedValueOnce(new Error('admission refused')) + const service = new ShadowService({ execute, recover, pending, onExecutionError, recoveryIntervalMs: 100, recoveryMaximumRuns: 7 }) + service.start() + await vi.advanceTimersByTimeAsync(100) + expect(pending.mock.calls.slice(0, 2)).toEqual([[{ maximumRuns: 7 }], [{ maximumRuns: 7 }]]) + expect(onExecutionError).toHaveBeenCalledTimes(2) + await vi.advanceTimersByTimeAsync(100) + expect(pending).toHaveBeenLastCalledWith({ maximumRuns: 7, afterId: runId }) + await service.stop() +}) + +it('advances bounded sweeps, retains the cursor on failure and stops maintenance', async () => { + vi.useFakeTimers() + const { service, recover, onRecoveryError } = setup() + recover.mockReturnValueOnce({ scanned: 7, recovered: [], nextCursor: runId }) + service.start() + service.start() + expect(recover).toHaveBeenCalledTimes(1) + recover.mockImplementationOnce(() => { + throw new Error('busy') + }) + onRecoveryError.mockImplementationOnce(() => { + throw new Error('logger failed') + }) + await vi.advanceTimersByTimeAsync(100) + expect(onRecoveryError).toHaveBeenCalledOnce() + await vi.advanceTimersByTimeAsync(100) + expect(recover.mock.calls.slice(1)).toEqual([[{ maximumRuns: 7, afterId: runId }], [{ maximumRuns: 7, afterId: runId }]]) + await vi.advanceTimersByTimeAsync(100) + expect(recover).toHaveBeenLastCalledWith({ maximumRuns: 7 }) + await service.stop() + service.start() + await vi.advanceTimersByTimeAsync(1000) + expect(recover).toHaveBeenCalledTimes(4) +}) + +it('retains the execution slot through cancellation and asynchronous cleanup', async () => { + const { service, execute } = setup() + let release!: () => void + let signal!: AbortSignal + execute.mockImplementation(async (_id, _revision, received) => { + signal = received + await new Promise((resolve) => { + release = resolve + }) + throw new Error('cancelled after cleanup') + }) + const run = service.run(runId, otherId) + const failure = expect(run).rejects.toThrow('cancelled after cleanup') + await Promise.resolve() + expect(execute).toHaveBeenCalledWith(runId, otherId, signal) + expect(await service.cancel(otherId)).toBe(false) + expect(signal.aborted).toBe(false) + let cancelled = false + const cancellation = service.cancel(runId).then((result) => { + cancelled = result + }) + expect(signal.aborted).toBe(true) + expect(() => service.run(otherId, null)).toThrow('busy') + await Promise.resolve() + expect(cancelled).toBe(false) + release() + await Promise.all([failure, cancellation]) + expect(cancelled).toBe(true) + execute.mockResolvedValueOnce('next') + expect(await service.run(otherId, null)).toBe('next') + await service.stop() +}) + +it('shutdown aborts, waits for cleanup, rejects new work and is repeatable', async () => { + const { service, execute } = setup() + let release!: () => void + execute.mockImplementation(async (_id, _revision, signal) => { + await new Promise((resolve) => { + release = resolve + }) + expect(signal.aborted).toBe(true) + }) + const run = service.run(runId, null) + await Promise.resolve() + let stopped = false + const stop = service.stop().then(() => { + stopped = true + }) + await Promise.resolve() + expect(stopped).toBe(false) + expect(() => service.run(otherId, null)).toThrow('stopped') + release() + await Promise.all([stop, run, service.stop()]) + expect(stopped).toBe(true) +}) + +it('cancels before starting a deferred worker and releases synchronous failures', async () => { + const { service, execute } = setup() + const run = service.run(runId, null) + const failed = expect(run).rejects.toThrow('cancelled') + await service.cancel(runId) + await failed + expect(execute).not.toHaveBeenCalled() + execute.mockImplementationOnce(() => { + throw new Error('startup failed') + }) + await expect(service.run(runId, null)).rejects.toThrow('startup failed') + expect(await service.run(otherId, null)).toBe('complete') + await service.stop() +}) + +it('quarantines scheduling after unconfirmed cleanup while maintenance continues', async () => { + vi.useFakeTimers() + const { execute, recover } = setup() + const pending = vi.fn(() => ({ run: { runId, revisionId: null }, nextCursor: runId })) + let reject!: (error: Error) => void + execute.mockImplementation(() => new Promise((_resolve, fail) => { + reject = fail + })) + const service = new ShadowService({ execute, recover, pending, recoveryIntervalMs: 100, recoveryMaximumRuns: 7 }) + service.start() + await Promise.resolve() + const cancelled = service.cancel(runId) + reject(Object.assign(new Error('shadow_worker_cleanup_unconfirmed'), { code: 'SHADOW_CLEANUP_UNCONFIRMED' })) + expect(await cancelled).toBe(false) + await vi.advanceTimersByTimeAsync(300) + expect(pending).toHaveBeenCalledOnce() + expect(recover).toHaveBeenCalledTimes(4) + expect(() => service.run(otherId, null)).toThrow('cleanup unconfirmed') + await service.stop() +}) diff --git a/src/biometrics/shadow/tests/startup.test.ts b/src/biometrics/shadow/tests/startup.test.ts new file mode 100644 index 000000000..8f5d7bd85 --- /dev/null +++ b/src/biometrics/shadow/tests/startup.test.ts @@ -0,0 +1,88 @@ +// @vitest-environment node +import { mkdtempSync, rmSync, realpathSync } from 'node:fs' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import * as configSchema from '@/src/db/schema' +import * as biometricsSchema from '@/src/db/biometrics-schema' +import { packagedShadowHostPath } from '../../backtest/location' +import { startShadowService } from '../service-host' +import { configuredShadowService, startConfiguredShadowService } from '../startup' + +vi.mock('../../backtest/location', () => ({ packagedShadowHostPath: vi.fn(() => '/installed/shadow-host.cjs') })) +vi.mock('../service-host', () => ({ startShadowService: vi.fn() })) +const cleanup: (() => void)[] = [] +const policy = () => ({ version: 'shadow-service-configuration-v1', pythonPath: '/installed/python', modulesPath: '/installed/modules', presenceFile: '/run/presence.json', + storagePolicy: { maximumRuns: 10, maximumPoints: 1000, maximumBytes: 1000000 }, busyTimeoutMs: 50, recoveryIntervalMs: 1000, recoveryMaximumRuns: 10 }) +function databases(memory = false) { + const directory = realpathSync(mkdtempSync(join(tmpdir(), 'shadow-startup-'))) + const configPath = join(directory, 'config.db'), bioPath = join(directory, 'bio.db') + const config = new Database(memory ? ':memory:' : configPath), bio = new Database(bioPath) + cleanup.push(() => { + config.close() + bio.close() + rmSync(directory, { recursive: true, force: true }) + }) + return { config: drizzle(config, { schema: configSchema }), bio: drizzle(bio, { schema: biometricsSchema }), configPath, bioPath } +} +beforeEach(() => { + vi.stubEnv('SLEEPYPOD_BIOMETRICS_SHADOW_SERVICE', undefined) + vi.spyOn(console, 'warn').mockImplementation(() => {}) +}) +afterEach(() => { + cleanup.splice(0).forEach(close => close()) + vi.unstubAllEnvs() + vi.restoreAllMocks() + vi.clearAllMocks() +}) + +it('leaves scheduling disabled without explicit configuration', () => { + const { config, bio } = databases() + expect(configuredShadowService()).toBeNull() + expect(startConfiguredShadowService(config, bio)).toBeNull() + expect(packagedShadowHostPath).not.toHaveBeenCalled() + expect(startShadowService).not.toHaveBeenCalled() + expect(console.warn).not.toHaveBeenCalled() +}) + +it('wires deployment policy to the actual application databases and fixed package', () => { + const { config, bio, configPath, bioPath } = databases() + const raw = policy() + vi.stubEnv('SLEEPYPOD_BIOMETRICS_SHADOW_SERVICE', JSON.stringify(raw)) + startConfiguredShadowService(config, bio) + expect(startShadowService).toHaveBeenCalledWith({ installation: { workerPath: '/installed/shadow-host.cjs', + pythonPath: raw.pythonPath, modulesPath: raw.modulesPath, presenceFile: raw.presenceFile, + configurationDatabase: configPath, biometricsDatabase: bioPath }, storagePolicy: raw.storagePolicy, + busyTimeoutMs: 50, recoveryIntervalMs: 1000, recoveryMaximumRuns: 10 }) +}) + +it.each([ + 'null', '{broken', 'x'.repeat(16385), + JSON.stringify({ ...policy(), pythonPath: 'relative/python' }), + JSON.stringify({ ...policy(), recoveryIntervalMs: 0 }), + JSON.stringify({ ...policy(), busyTimeoutMs: 1001 }), + JSON.stringify({ ...policy(), configurationDatabase: '/other.db' }), + JSON.stringify({ ...policy(), workerPath: '/other.cjs' }), +])('refuses invalid startup configuration without echoing its contents (%#)', (raw) => { + const { config, bio } = databases() + vi.stubEnv('SLEEPYPOD_BIOMETRICS_SHADOW_SERVICE', raw) + expect(startConfiguredShadowService(config, bio)).toBeNull() + expect(startShadowService).not.toHaveBeenCalled() + expect(packagedShadowHostPath).not.toHaveBeenCalled() + expect(console.warn).toHaveBeenCalledWith('[biometrics] Shadow service unavailable; check deployment configuration and installed package') +}) + +it('requires persistent databases and contains missing-package startup failure', () => { + vi.stubEnv('SLEEPYPOD_BIOMETRICS_SHADOW_SERVICE', JSON.stringify(policy())) + const memory = databases(true) + expect(startConfiguredShadowService(memory.config, memory.bio)).toBeNull() + expect(startShadowService).not.toHaveBeenCalled() + const persistent = databases() + vi.mocked(packagedShadowHostPath).mockImplementationOnce(() => { + throw new Error('missing') + }) + expect(startConfiguredShadowService(persistent.config, persistent.bio)).toBeNull() + expect(startShadowService).not.toHaveBeenCalled() +}) diff --git a/src/biometrics/shadow/tests/store.test.ts b/src/biometrics/shadow/tests/store.test.ts new file mode 100644 index 000000000..0563f9477 --- /dev/null +++ b/src/biometrics/shadow/tests/store.test.ts @@ -0,0 +1,431 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { fileURLToPath } from 'node:url' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { ShadowStore } from '../store' +import { reportShadowRun } from '../report' +import { ShadowWorkerSink } from '../worker-sink' +import { ShadowStreamJoin } from '../stream-join' + +const databases: Database.Database[] = [] +afterEach(() => databases.splice(0).forEach(db => db.close())) +const hash = 'a'.repeat(64), start = 1700000000000 +const plan = { subjectId: 'fixture', monitoring: { expectedIntervalMs: 1000, maximumGapMs: 2000, heartbeatStaleMs: 2000, minimumCandidateCoverage: 0.8, maximumCoverageLoss: 0.1, minimumQuality: 0.8, + alignment: { version: 'reference-alignment-v1', pointToleranceMs: 0, clockOffsetMs: 0, offsetSource: 'fixed' } }, schema: 'biometric-shadow-plan-v1', side: 'left', candidateConfigHash: hash, activeConfigHash: 'b'.repeat(64), +qualificationHash: 'c'.repeat(64), engineBuildHash: 'd'.repeat(64), startMs: start, endMs: start + 60000, +maximumPoints: 3, maximumBytes: 10000, metrics: ['heart_rate'] } +function setup(policy = { maximumRuns: 10, maximumPoints: 10, maximumBytes: 100000 }) { + const db = new Database(':memory:') + databases.push(db) + migrate(drizzle(db), { migrationsFolder: fileURLToPath(new URL('../../../db/biometrics-migrations/', import.meta.url)) }) + let now = start + return { db, store: new ShadowStore(db, policy, () => now), advance: (ms: number) => { + now += ms + } } +} +function sample(sequence = 0) { + return { sequence, side: 'left', timestampMs: start + sequence, inputHash: hash, candidateConfigHash: hash, activeConfigHash: plan.activeConfigHash, + readings: [{ metric: 'heart_rate', candidate: { value: null as number | null, quality: 0 }, active: { value: 60, quality: 1 } }] } +} +const joining = { maximumPendingPoints: 10, maximumLatenessMs: 1000, drainGraceMs: 2000 } +const executable = { ...plan, joining, candidateRevisionId: null, + expectedBindings: { left: { generation: 1, resolvedHash: hash }, right: { generation: 1, resolvedHash: hash } }, + execution: { version: 'shadow-execution-policy-v1', configHashes: { left: hash, right: hash }, + maximumRecords: 10, maximumOutputs: 10, maximumCpuMs: 1000, maximumInputBytes: 100000, maximumOutputBytes: 100000, + budgets: { cpuSeconds: 2, addressSpaceBytes: 100000000, maximumRssBytes: 100000000, wallMs: 10000, acknowledgmentMs: 1000, idleMs: 1000, rssIntervalMs: 100, rssTimeoutMs: 100 }, + leaseMs: 1500, heartbeatMs: 200, sources: { pollMs: 50, maximumActiveRows: 10, databaseBusyTimeoutMs: 50, + frames: { maximumFrames: 10, maximumBytes: 100000, maximumEventAgeMs: 1000, maximumFutureSkewMs: 0 }, + context: { presenceStaleMs: 300000, pumpStaleMs: 5000, maximumPresenceBytes: 4096 } } } } + +it('persists candidate revision identity and rejects identity changes on retry', () => { + const { store } = setup() + const revision = '11111111-1111-4111-8111-111111111111' + const run = store.create({ ...executable, candidateRevisionId: revision }, 'identity') + expect(store.get(run.id).plan.candidateRevisionId).toBe(revision) + expect(() => store.create(executable, 'identity')).toThrow('idempotency') + expect(() => store.create({ ...executable, candidateRevisionId: 'latest' }, 'invalid')).toThrow() +}) + +it('scans bounded pages past legacy, future and running plans using saved identity', () => { + const { store, advance } = setup() + store.create(plan, 'legacy') + const { candidateRevisionId, ...legacyExecution } = executable + expect(candidateRevisionId).toBeNull() + store.create(legacyExecution, 'no-identity') + const future = store.create({ ...executable, startMs: start + 1000 }, 'future') + const running = store.create(executable, 'running') + store.claim(running.id, 10000, () => {}) + const ready = store.create(executable, 'ready') + const scan = () => { + const results: string[] = [] + let cursor: string | undefined + let pages = 0 + do { + const page = store.pending({ maximumRuns: 1, ...(cursor ? { afterId: cursor } : {}) }) + if (page.run) { + expect(page.run.revisionId).toBeNull() + results.push(page.run.runId) + } + cursor = page.nextCursor ?? undefined + expect(++pages).toBeLessThan(8) + } while (cursor) + return results + } + expect(scan()).toEqual([ready.id]) + advance(1000) + expect(scan().sort()).toEqual([ready.id, future.id].sort()) + advance(60000) + expect(scan()).toEqual([]) + expect(store.get(running.id).state).toBe('running') +}) + +it('fails queue inspection on corrupted pending evidence', () => { + const { store, db } = setup() + const run = store.create(executable, 'ready') + db.prepare('UPDATE biometric_shadow_runs SET plan_hash = \'corrupt\' WHERE id = ?').run(run.id) + expect(() => store.pending({ maximumRuns: 1 })).toThrow('integrity') +}) +function workerOutput(timestampMs: number, sequence = 0) { + return { sequence, inputHash: hash, side: 'left', sourceSide: 'left', timestampMs, configHash: hash, + readings: [{ metric: 'heart_rate', unit: 'bpm', windowMs: null, value: 60, quality: 1 }, + { metric: 'breathing_rate', unit: 'breaths/min', windowMs: null, value: 15, quality: 1 }, + { metric: 'window_ibi_variability_v1', unit: 'ms', windowMs: 300000, value: null, quality: 0 }], flags: null, limitations: [] } +} +function activeOutput(timestampMs: number) { + return { side: 'left', timestampMs, configHash: plan.activeConfigHash, readings: workerOutput(timestampMs).readings } +} +describe('separate prospective shadow storage', () => { + it('uses the generated additive migration and never writes canonical observations', () => { + const { db, store } = setup() + for (const operation of ['INSERT', 'UPDATE', 'DELETE']) db.exec(`CREATE TRIGGER deny_vitals_${operation} BEFORE ${operation} ON vitals BEGIN SELECT RAISE(ABORT, 'canonical write forbidden'); END`) + const before = db.prepare('SELECT * FROM vitals').all() + const run = store.create(plan, 'create') + const lease = store.claim(run.id, 10000, (admitted) => { + expect(admitted.qualificationHash).toBe(plan.qualificationHash) + }) + expect(store.append(lease, sample())).toEqual({ state: 'recorded' }) + expect(store.append(lease, sample())).toEqual({ state: 'already_recorded' }) + expect(store.get(run.id)).toMatchObject({ point_count: 1, state: 'running' }) + expect(db.prepare('SELECT * FROM vitals').all()).toEqual(before) + const payload = JSON.parse((db.prepare('SELECT payload FROM biometric_shadow_samples').get() as { payload: string }).payload) + expect(payload).toMatchObject({ attemptId: lease.attemptId, readings: [{ candidate: { value: null }, active: { value: 60 } }] }) + }) + it('rejects retroactive windows and changed idempotency requests, while exact retries survive time passing', () => { + const { store, advance } = setup() + const run = store.create(plan, 'first') + advance(100) + expect(store.create(plan, 'first').id).toBe(run.id) + expect(() => store.create(plan, 'second')).toThrow('prospective') + expect(() => store.create({ ...plan, side: 'right' }, 'first')).toThrow('idempotency') + }) + it('requires admission and fences replaced workers without losing retained samples', () => { + const { store, advance } = setup() + const run = store.create(plan, 'first') + expect(() => store.claim(run.id, 1000, () => { + throw new Error('Unqualified build') + })).toThrow('Unqualified build') + expect(store.get(run.id).state).toBe('pending') + const old = store.claim(run.id, 1000, () => undefined) + store.append(old, sample()) + advance(1000) + expect(() => store.append(old, sample(1))).toThrow('Stale') + const current = store.claim(run.id, 1000, () => undefined) + expect(() => store.renew(old, 1000)).toThrow('Stale') + expect(() => store.finish(old, 'completed', 'stale')).toThrow('Stale') + expect(store.append(current, sample(1))).toEqual({ state: 'recorded' }) + expect(store.get(run.id).point_count).toBe(2) + }) + it('limits concurrent workers per side but permits the other side', () => { + const { store } = setup() + const first = store.create(plan, 'first'), second = store.create(plan, 'second'), right = store.create({ ...plan, side: 'right' }, 'right') + store.claim(first.id, 1000, () => undefined) + expect(() => store.claim(second.id, 1000, () => undefined)).toThrow('Side already') + expect(store.claim(right.id, 1000, () => undefined).runId).toBe(right.id) + }) + it('rejects configuration changes, duplicate metrics, future points, reordered points and altered retries', () => { + const { store, advance } = setup() + const lease = store.claim(store.create(plan, 'first').id, 10000, () => undefined) + for (const invalid of [{ ...sample(), side: 'right' }, { ...sample(), candidateConfigHash: 'f'.repeat(64) }, { ...sample(), activeConfigHash: 'f'.repeat(64) }, + { ...sample(), timestampMs: start - 1 }, { ...sample(), timestampMs: start + 1 }, { ...sample(), readings: [...sample().readings, ...sample().readings] }]) { + expect(() => store.append(lease, invalid)).toThrow('frozen plan') + } + store.append(lease, sample()) + expect(() => store.append(lease, { ...sample(), inputHash: 'f'.repeat(64) })).toThrow('idempotency') + advance(100) + expect(() => store.append(lease, sample(2))).toThrow('Out-of-order') + expect(() => store.append(lease, { ...sample(1), timestampMs: start })).toThrow('Out-of-order') + }) + it('atomically stops at per-run and global quotas and keeps stored evidence', () => { + const { store, advance } = setup({ maximumRuns: 2, maximumPoints: 3, maximumBytes: 100000 }) + const first = store.create({ ...plan, maximumPoints: 1 }, 'first'), second = store.create({ ...plan, side: 'right' }, 'second') + expect(() => store.create(plan, 'third')).toThrow('run quota') + const lease = store.claim(first.id, 10000, () => undefined) + const other = store.claim(second.id, 10000, () => undefined) + store.append(lease, sample()) + advance(1) + expect(store.append(lease, sample(1))).toEqual({ state: 'quota_reached' }) + expect(store.get(first.id)).toMatchObject({ state: 'stopped', reason: 'storage_quota', point_count: 1 }) + store.append(other, { ...sample(), side: 'right' }) + store.append(other, { ...sample(1), side: 'right' }) + advance(1) + expect(store.append(other, { ...sample(2), side: 'right' })).toEqual({ state: 'quota_reached' }) + expect(store.get(second.id).point_count).toBe(2) + }) + it('can cancel after deadline and cannot accept asynchronous admission', () => { + const { store, advance } = setup() + const run = store.create(plan, 'first') + expect(() => store.claim(run.id, 1000, async () => undefined)).toThrow('synchronously') + expect(store.get(run.id).state).toBe('pending') + const lease = store.claim(run.id, 1000, () => undefined) + store.append(lease, sample()) + advance(60000) + expect(() => store.renew(lease, 1000)).toThrow('Stale') + expect(store.stop(run.id, 'window_elapsed')).toMatchObject({ state: 'stopped', reason: 'window_elapsed', point_count: 1 }) + expect(store.stop(run.id, 'retry')).toMatchObject({ reason: 'window_elapsed' }) + expect(() => store.claim(run.id, 1000, () => undefined)).toThrow('unavailable') + }) + it('reports dropped candidate values, common reference support and current worker health', () => { + const { store, advance } = setup() + const lease = store.claim(store.create(plan, 'first').id, 10000, () => undefined) + const first = sample() + first.readings[0].candidate = { value: 61, quality: 1 } + store.append(lease, first) + advance(1000) + store.append(lease, { ...sample(1), timestampMs: start + 1000 }) + const reference = { subjectId: 'fixture', side: 'left', source: 'test', rows: [0, 1000].map((offset, index) => ({ + recordKey: String(index), metric: 'heart_rate', value: 60, unit: 'bpm', timestampMs: start + offset })) } + const report = reportShadowRun(store, lease.runId, { maximumPoints: 10, maximumBytes: 10000 }, reference) + expect(report).toMatchObject({ expectedSlots: 2, observedSamples: 2, health: 'issues_detected', issues: ['coverage_degraded'], + metrics: [{ candidate: { coverage: 0.5, usableCount: 1 }, current: { coverage: 1 }, coverageDifference: -0.5, + comparison: { commonObservationCount: 1, lostObservationTimestamps: 1 } }] }) + expect(report.reportHash).toMatch(/^[a-f0-9]{64}$/) + advance(10000) + expect(report.contextEvidence).toBe('host_observations_not_live_processor_cache_attestation') + const stale = reportShadowRun(store, lease.runId, { maximumPoints: 10, maximumBytes: 10000 }) + expect(stale.issues).toEqual(expect.arrayContaining(['worker_stale', 'observation_gap'])) + }) + it('does not inflate coverage from bursts within one cadence slot', () => { + const { store, advance } = setup() + const lease = store.claim(store.create(plan, 'first').id, 10000, () => undefined) + for (let index = 0; index < 3; index++) { + const point = sample(index) + point.readings[0].candidate = { value: 60, quality: 1 } + store.append(lease, point) + advance(1) + } + advance(1997) + const report = reportShadowRun(store, lease.runId, { maximumPoints: 10, maximumBytes: 10000 }) + expect(report).toMatchObject({ expectedSlots: 3, metrics: [{ candidate: { usableCount: 3, usableSlots: 1, coverage: 1 / 3 } }] }) + }) + it('requires matching reference identity and never compares RMSSD to the variability index', () => { + const { store } = setup() + const lease = store.claim(store.create({ ...plan, metrics: ['window_ibi_variability_v1'], windowIbiWindowMs: 300000 }, 'first').id, 10000, () => undefined) + store.append(lease, { ...sample(), readings: [{ metric: 'window_ibi_variability_v1', active: { value: 20, quality: 1 }, candidate: { value: 20, quality: 1 } }] }) + const reference = { subjectId: 'fixture', side: 'left', source: 'test', rows: [{ recordKey: '0', metric: 'hrv_rmssd', value: 20, unit: 'ms', windowMs: 300000, timestampMs: start }] } + const limits = { maximumPoints: 10, maximumBytes: 10000 } + expect(reportShadowRun(store, lease.runId, limits, reference).metrics[0]).toMatchObject({ + referenceExclusions: { incompatible_metric_definition: 1 }, comparison: { commonSupport: { candidate: { status: 'incompatible', metrics: null } } }, + }) + expect(() => reportShadowRun(store, lease.runId, limits, { ...reference, subjectId: 'other' })).toThrow('subject or side') + expect(() => reportShadowRun(store, lease.runId, limits, { ...reference, side: 'right' })).toThrow('subject or side') + expect(() => store.create({ ...plan, metrics: ['window_ibi_variability_v1'] }, 'invalid')).toThrow() + }) + it('refuses oversized or corrupted snapshots instead of scoring partial history', () => { + const { db, store } = setup() + const lease = store.claim(store.create(plan, 'first').id, 10000, () => undefined) + store.append(lease, sample()) + expect(() => store.snapshot(lease.runId, { maximumPoints: 10, maximumBytes: 1 })).toThrow('input limit') + db.prepare('UPDATE biometric_shadow_samples SET content_hash = ? WHERE run_id = ?').run('f'.repeat(64), lease.runId) + expect(() => store.snapshot(lease.runId, { maximumPoints: 10, maximumBytes: 10000 })).toThrow('integrity') + }) + it('reports clock skew and missing observations without converting unknown values to zero', () => { + const { store, advance } = setup() + const run = store.create(plan, 'first') + expect(reportShadowRun(store, run.id, { maximumPoints: 10, maximumBytes: 10000 }).issues).toContain('no_observations') + const lease = store.claim(run.id, 10000, () => undefined) + store.append(lease, sample()) + advance(-1) + expect(reportShadowRun(store, run.id, { maximumPoints: 10, maximumBytes: 10000 })).toMatchObject({ + health: 'issues_detected', issues: expect.arrayContaining(['clock_skew']), metrics: [{ candidate: { coverage: null } }], + }) + }) + it('bridges typed worker readings with exact active identity and preserves missing data and limitations', () => { + const { store, advance } = setup() + const lease = store.claim(store.create(plan, 'first').id, 1000, () => undefined) + const sink = new ShadowWorkerSink(store, lease) + const output = { sequence: 0, inputHash: hash, side: 'left', sourceSide: 'right', timestampMs: start, configHash: hash, + readings: [{ metric: 'heart_rate', unit: 'bpm', windowMs: null, value: 60, quality: 1 }, + { metric: 'breathing_rate', unit: 'breaths/min', windowMs: null, value: 15, quality: 1 }, + { metric: 'window_ibi_variability_v1', unit: 'ms', windowMs: 300000, value: null, quality: 0 }], + flags: null, limitations: ['cold_start_without_serialized_state'] } + const active = { side: 'left', timestampMs: start, configHash: plan.activeConfigHash, readings: output.readings } + for (const changed of [{ ...active, side: 'right' }, { ...active, timestampMs: start + 1 }, { ...active, configHash: hash }]) { + expect(() => sink.append(output, changed)).toThrow('not contemporaneous') + } + expect(store.get(lease.runId).point_count).toBe(0) + expect(sink.append(output, null)).toEqual({ state: 'recorded' }) + expect(sink.append(output, null)).toEqual({ state: 'already_recorded' }) + const snapshot = store.snapshot(lease.runId, { maximumPoints: 10, maximumBytes: 10000 }) + expect(snapshot.samples[0]).toMatchObject({ readings: [{ active: { value: null, quality: 0 }, candidate: { value: 60 } }], + candidateProvenance: { sourceSide: 'right', limitations: expect.arrayContaining(['missing_contemporaneous_active_reading']) } }) + expect(reportShadowRun(store, lease.runId, { maximumPoints: 10, maximumBytes: 10000 }).limitations).toContain('cold_start_without_serialized_state') + advance(1000) + expect(() => sink.append({ ...output, sequence: 1, timestampMs: start + 1000 }, null)).toThrow('Stale') + }) + it('retains active-only times so dropped candidate readings reduce reported coverage', () => { + const { store, advance } = setup() + const lease = store.claim(store.create({ ...plan, joining }, 'joined').id, 10000, () => undefined) + const join = new ShadowStreamJoin(store, lease) + for (const offset of [2000, 0, 1000]) join.active(activeOutput(start + offset)) + join.candidate(workerOutput(start + 1000)) + advance(2999) + expect(join.advance(start + 2999)).toEqual({ emitted: 2, stopped: false }) + advance(1) + expect(join.advance(start + 3000)).toEqual({ emitted: 1, stopped: false }) + const report = reportShadowRun(store, lease.runId, { maximumPoints: 10, maximumBytes: 10000 }) + expect(report).toMatchObject({ pairing: { activeOnly: 2, candidateOnly: 0, unknownInputIdentity: 2 }, + metrics: [{ current: { usableCount: 3 }, candidate: { usableCount: 1 }, issues: expect.arrayContaining(['coverage_regression']) }] }) + const points = store.snapshot(lease.runId, { maximumPoints: 10, maximumBytes: 10000 }).samples + expect(points.map(row => row.timestampMs)).toEqual([start, start + 1000, start + 2000]) + expect(points[0]).toMatchObject({ inputHash: null, inputIdentityKind: 'unavailable', readings: [{ candidate: { value: null } }] }) + }) + it('does not match nearby timestamps and fails explicitly on late or conflicting arrivals', () => { + const { store, advance } = setup() + const lease = store.claim(store.create({ ...plan, joining }, 'joined').id, 10000, () => undefined) + const join = new ShadowStreamJoin(store, lease) + join.active(activeOutput(start)) + join.active(activeOutput(start)) + join.candidate(workerOutput(start + 1)) + advance(1001) + expect(join.advance(start + 1001).emitted).toBe(2) + expect(reportShadowRun(store, lease.runId, { maximumPoints: 10, maximumBytes: 10000 }).pairing).toMatchObject({ activeOnly: 1, candidateOnly: 1 }) + expect(() => join.candidate(workerOutput(start))).toThrow('join_late_arrival') + expect(store.get(lease.runId)).toMatchObject({ state: 'failed', reason: 'join_late_arrival' }) + const nextPlan = { ...plan, startMs: start + 1001, joining } + const secondLease = store.claim(store.create(nextPlan, 'second').id, 10000, () => undefined) + const second = new ShadowStreamJoin(store, secondLease) + second.active(activeOutput(start + 1001)) + const changed = activeOutput(start + 1001) + changed.readings[0].value = 80 + expect(() => second.active(changed)).toThrow('join_conflicting_duplicate') + }) + it('bounds pending points and preserves uncertain-write retry identity', () => { + const { store, advance } = setup() + const lease = store.claim(store.create({ ...plan, joining: { ...joining, maximumPendingPoints: 1 } }, 'joined').id, 10000, () => undefined) + const join = new ShadowStreamJoin(store, lease) + join.active(activeOutput(start)) + join.candidate(workerOutput(start)) + const append = store.append.bind(store) + vi.spyOn(store, 'append').mockImplementationOnce((lease, point) => { + append(lease, point) + throw new Error('lost acknowledgment') + }) + advance(1000) + expect(() => join.advance(start + 1000)).toThrow('lost acknowledgment') + expect(store.get(lease.runId).point_count).toBe(1) + expect(join.advance(start + 1000).emitted).toBe(1) + expect(store.get(lease.runId).point_count).toBe(1) + join.active(activeOutput(start + 2000)) + expect(() => join.active(activeOutput(start + 3000))).toThrow('join_queue_quota') + }) + it('drains final buffered samples during explicit grace without extending the sampling window', () => { + const { store, advance } = setup() + const selected = { ...plan, endMs: start + 5000, joining } + const run = store.create(selected, 'joined') + const lease = store.claim(run.id, 10000, () => undefined) + const join = new ShadowStreamJoin(store, lease) + join.active(activeOutput(start + 4999)) + advance(6000) + expect(join.advance(start + 6000)).toEqual({ emitted: 1, stopped: false }) + expect(store.get(run.id).point_count).toBe(1) + store.renew(lease, 10000) + expect(store.get(run.id).lease_until_ms).toBe(start + 7000) + advance(1000) + expect(() => store.renew(lease, 10000)).toThrow('Stale') + }) + it('enforces byte quotas and terminal states', () => { + const { store } = setup() + const lease = store.claim(store.create({ ...plan, maximumBytes: 1 }, 'first').id, 1000, () => undefined) + expect(store.append(lease, sample()).state).toBe('quota_reached') + expect(() => store.append(lease, sample())).toThrow('Stale') + const next = store.claim(store.create(plan, 'second').id, 1000, () => undefined) + store.finish(next, 'failed', 'processor_failure') + expect(store.get(next.runId)).toMatchObject({ state: 'failed', reason: 'processor_failure' }) + expect(() => store.renew(next, 1000)).toThrow('Stale') + }) +}) + +describe('bounded expired shadow recovery', () => { + it('revokes expired leases while retaining observations and exposing unconfirmed cleanup', () => { + const { store, advance } = setup() + const run = store.create(plan, 'recover') + const lease = store.claim(run.id, 1000, () => {}) + store.append(lease, sample()) + advance(1000) + expect(store.reconcileExpired({ maximumRuns: 10 }).recovered).toEqual([{ runId: run.id, attemptId: lease.attemptId, reason: 'shadow_lease_expired_cleanup_unconfirmed' }]) + expect(store.get(run.id)).toMatchObject({ state: 'failed', point_count: 1, attempt_id: lease.attemptId, workerReceipt: null }) + expect(() => store.append(lease, sample(1))).toThrow(/Stale/) + expect(() => store.renew(lease, 1000)).toThrow(/Stale/) + expect(store.reconcileExpired({ maximumRuns: 10 }).recovered).toEqual([]) + const report = reportShadowRun(store, run.id, { maximumPoints: 10, maximumBytes: 10000 }) + expect(report.issues).toContain('worker_cleanup_unconfirmed') + expect(report.observedSamples).toBe(1) + }) + it('marks missed pending windows without claiming processor execution', () => { + const { store, advance } = setup() + const run = store.create(plan, 'missed') + advance(60000) + expect(store.reconcileExpired({ maximumRuns: 10 }).recovered).toEqual([{ runId: run.id, attemptId: null, reason: 'shadow_observation_window_missed' }]) + expect(store.get(run.id)).toMatchObject({ state: 'failed', point_count: 0, attempt_id: null }) + const report = reportShadowRun(store, run.id, { maximumPoints: 10, maximumBytes: 10000 }) + expect(report.issues).toContain('observation_window_missed') + expect(report.issues).not.toContain('processor_failed') + }) + it('preserves renewed replacement attempts and terminal evidence', () => { + const { store, advance } = setup() + const run = store.create(plan, 'replacement'), stopped = store.create({ ...plan, side: 'right' }, 'stopped') + store.claim(run.id, 1000, () => {}) + store.stop(stopped.id, 'operator') + advance(1000) + const replacement = store.claim(run.id, 1000, () => {}) + expect(store.reconcileExpired({ maximumRuns: 10 }).recovered).toEqual([]) + expect(store.get(run.id)).toMatchObject({ state: 'running', attempt_id: replacement.attemptId }) + expect(store.get(stopped.id)).toMatchObject({ state: 'stopped', reason: 'operator' }) + }) + it('advances a bounded keyset scan past future runs without starving later rows', () => { + const { store, advance } = setup() + const runs = Array.from({ length: 5 }, (_, index) => store.create({ ...plan, endMs: start + (index % 2 ? 120000 : 60000) }, String(index))) + advance(60000) + const recovered: string[] = [], visited: number[] = [] + let cursor: string | undefined + do { + const batch = store.reconcileExpired({ maximumRuns: 2, ...(cursor ? { afterId: cursor } : {}) }) + recovered.push(...batch.recovered.map(row => row.runId)) + visited.push(batch.scanned) + cursor = batch.nextCursor ?? undefined + } while (cursor) + expect(visited).toEqual([2, 2, 1]) + expect(recovered.sort()).toEqual([runs[0].id, runs[2].id, runs[4].id].sort()) + expect(() => store.reconcileExpired({ maximumRuns: 101 })).toThrow() + }) +}) + +it('recovery preserves final drain authority until the actual lease deadline', () => { + const { store, advance } = setup() + const run = store.create({ ...plan, joining }, 'draining') + store.claim(run.id, 100000, () => {}) + advance(61000) + expect(store.reconcileExpired({ maximumRuns: 10 }).recovered).toEqual([]) + advance(1000) + expect(store.reconcileExpired({ maximumRuns: 10 }).recovered).toHaveLength(1) +}) +it('rolls back the entire recovery batch if a plan fails integrity validation', () => { + const { db, store, advance } = setup() + const ids = [store.create(plan, 'one').id, store.create(plan, 'two').id].sort() + db.prepare('UPDATE biometric_shadow_runs SET plan_hash = \'corrupt\' WHERE id = ?').run(ids[1]) + advance(60000) + expect(() => store.reconcileExpired({ maximumRuns: 10 })).toThrow(/integrity/) + expect(store.get(ids[0]).state).toBe('pending') +}) diff --git a/src/biometrics/shadow/tests/submission.test.ts b/src/biometrics/shadow/tests/submission.test.ts new file mode 100644 index 000000000..cf3287fa0 --- /dev/null +++ b/src/biometrics/shadow/tests/submission.test.ts @@ -0,0 +1,81 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import { ShadowStore } from '../store' +import { submitInstalledShadow } from '../submission' +import { inspectInstalledShadowHost, loadInstalledShadowHost } from '../installed-host' +import { openShadowStateDatabase } from '../active-database' +import { installedShadowIdentity } from '../device-identity' +import { submissionFixture, submissionEntry, submissionTime } from './fixtures/submission' +vi.mock('../installed-host', () => ({ inspectInstalledShadowHost: vi.fn(), loadInstalledShadowHost: vi.fn() })) +vi.mock('../active-database', () => ({ openShadowStateDatabase: vi.fn() })) +vi.mock('../device-identity', () => ({ installedShadowIdentity: vi.fn() })) +vi.mock('@/src/streaming/piezoStream', () => ({ getSensorFrameTimes: () => ({ 'piezo-dual': submissionTime }) })) +const paths = { workerPath: '/installed/shadow-host.cjs', pythonPath: '/installed/python', modulesPath: '/installed/modules', + configurationDatabase: '/installed/config.db', biometricsDatabase: '/installed/bio.db', presenceFile: '/run/presence' } +const policy = { maximumRuns: 10, maximumPoints: 100, maximumBytes: 100000 } +let db: Database.Database, now: number +beforeEach(() => { + vi.resetAllMocks() + db = new Database(':memory:') + migrate(drizzle(db), { migrationsFolder: 'src/db/biometrics-migrations' }) + now = submissionTime +}) +afterEach(() => db.close()) +function setup() { + const owned = { host: { admit: vi.fn() }, close: vi.fn() } + const open = vi.fn().mockResolvedValue(owned), guard = vi.fn() + class Store extends ShadowStore { + constructor(raw: Database.Database, limits: unknown) { super(raw, limits, () => now) } + } + const inspected = { api: { ShadowStore: Store, openProfiledShadowHost: open, + readInstalledShadowQualifications: () => ({ entries: [submissionEntry] }) }, assertCurrent: guard } + vi.mocked(loadInstalledShadowHost).mockReturnValue(inspected as unknown as ReturnType) + vi.mocked(inspectInstalledShadowHost).mockResolvedValue(inspected as unknown as Awaited>) + const handle = { db, assertCurrent: vi.fn(), close: vi.fn() } + vi.mocked(openShadowStateDatabase).mockReturnValue(handle) + vi.mocked(installedShadowIdentity).mockReturnValue(vi.fn()) + return { owned, open, guard, handle } +} + +it('checks admission inside the creation transaction and closes owned admission resources', async () => { + const { owned, open, handle } = setup(), input = submissionFixture() + owned.host.admit.mockImplementation(() => { + expect(db.inTransaction).toBe(true) + expect(db.prepare('SELECT count(*) AS n FROM biometric_shadow_runs').get()).toEqual({ n: 0 }) + }) + const run = await submitInstalledShadow(input, paths, policy, 50) + expect(run).toMatchObject({ state: 'pending', plan: input.plan }) + expect(open).toHaveBeenCalledWith(input.plan, expect.objectContaining(paths), null, undefined) + expect(owned.host.admit).toHaveBeenCalledOnce() + expect(owned.close).toHaveBeenCalledOnce() + expect(handle.close).toHaveBeenCalledOnce() +}) + +it('returns exact retries before probing changed health and rejects plan substitution', async () => { + setup() + const input = submissionFixture(), first = await submitInstalledShadow(input, paths, policy, 50) + now += 100000 + vi.mocked(inspectInstalledShadowHost).mockClear().mockRejectedValue(new Error('unavailable')) + expect((await submitInstalledShadow(input, paths, policy, 50)).id).toBe(first.id) + await expect(submitInstalledShadow({ ...input, plan: { ...input.plan, subjectId: 'other' } }, paths, policy, 50)).rejects.toThrow('idempotency') + expect(inspectInstalledShadowHost).not.toHaveBeenCalled() +}) + +it.each(['admission', 'window', 'cancel'])('does not queue after %s changes while opening admission', async (failure) => { + const { owned, open, handle } = setup(), abort = new AbortController() + open.mockImplementation(async () => { + if (failure === 'window') now += 2000 + if (failure === 'cancel') abort.abort() + if (failure === 'admission') owned.host.admit.mockImplementation(() => { + throw new Error('binding changed') + }) + return owned + }) + await expect(submitInstalledShadow(submissionFixture(), paths, policy, 50, abort.signal)).rejects.toThrow() + expect(db.prepare('SELECT count(*) AS n FROM biometric_shadow_runs').get()).toEqual({ n: 0 }) + expect(owned.close).toHaveBeenCalledOnce() + expect(handle.close).toHaveBeenCalledOnce() +}) diff --git a/src/biometrics/shadow/tests/worker-child.test.ts b/src/biometrics/shadow/tests/worker-child.test.ts new file mode 100644 index 000000000..f85a976fa --- /dev/null +++ b/src/biometrics/shadow/tests/worker-child.test.ts @@ -0,0 +1,322 @@ +// @vitest-environment node +import type * as ChildProcess from 'node:child_process' +import { randomUUID } from 'node:crypto' +import { existsSync } from 'node:fs' +import { resolve } from 'node:path' +import { setTimeout as delay } from 'node:timers/promises' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { defaultConfig } from '../../algorithms/registry' +import { configHash } from '../../algorithms/canonical' +import { shadowPartialReceiptSchema } from '../receipt' +import { runShadowChild } from '../worker-child' + +const state = vi.hoisted(() => ({ mode: 'normal', pid: 0, descendantPid: 0, rss: 1000 as number | null, hangRss: false, delayClose: false, missingAfterFinish: false, launches: [] as unknown[][] })) +vi.mock('../../backtest/process-group-memory', () => ({ sampleProcessGroupRss: () => state.hangRss ? new Promise(() => {}) : Promise.resolve(state.rss) })) +vi.mock('node:child_process', async (original) => { + const actual = await original() + return { ...actual, spawn: (command: string, args: string[], options: ChildProcess.SpawnOptions) => { + state.launches.push([command, args, options]) + const script = ` +const readline = require('node:readline'); +const mode = ${JSON.stringify(state.mode)}; +let init, records = 0, outputs = 0; +const health = state => ({state, reason:null, records, outputs, cpu_ms:1, input_hash:'a'.repeat(64), limitations:[], + counts:{records,accepted_frames:records,empty_channel_frames:0,pump_gated_frames:0,preroll_outputs:0,scored_outputs:outputs}}); +function emit(row) { process.stdout.write(JSON.stringify({runId:init.runId,attemptId:init.attemptId,...row})+'\\n'); } +if(mode==='ignore') { setInterval(()=>{},1000); } +else readline.createInterface({input:process.stdin}).on('line', line => { + const message = JSON.parse(line); + if(!init) { + init = message; + if(mode==='descendant') { + const grandchild = require('node:child_process').spawn(process.execPath, ['-e', 'setInterval(()=>{},1000)'], {stdio:'inherit'}); + process.stderr.write('descendant:'+grandchild.pid+'\\n'); grandchild.unref(); + } + if(mode==='bad') return process.stdout.write('{}\\n'); + if(mode==='flood') return process.stdout.write('x'.repeat(140000)); + if(mode==='stderr') return process.stderr.write('x'.repeat(17000)); + if(mode==='duplicate') return process.stdout.write((JSON.stringify({runId:init.runId,attemptId:init.attemptId,type:'ready',configHashes:init.configHashes})+'\\n').repeat(2)); + return emit({type:'ready', configHashes:init.configHashes}); + } + if(message.type==='frame') { + records++; + const batch = mode==='output' ? [{sequence:outputs++,inputHash:'a'.repeat(64),side:init.side,sourceSide:init.side, + timestampMs:init.startMs+records*1000,configHash:init.configHashes[init.side],flags:null,limitations:[],readings:[ + {metric:'heart_rate',unit:'bpm',windowMs:null,value:60,quality:1}, + {metric:'breathing_rate',unit:'breaths/min',windowMs:null,value:12,quality:1}, + {metric:'window_ibi_variability_v1',unit:'ms',windowMs:300000,value:null,quality:0}]}] : []; + return emit({type:'frame_result',sequence:message.sequence,outputs:batch,health:health('running')}); + } + if(mode==='noexit') { emit({type:'finished',outputs:[],health:health('completed')}); setInterval(()=>{},1000); return; } + if(mode==='truncated') { process.stdout.write('{'); return process.exit(0); } + emit({type:'finished',outputs:[],health:health('completed')}); + process.exitCode = mode==='badexit' ? 1 : 0; +});` + const child = state.mode === 'python' + ? actual.spawn(resolve('modules/piezo-processor/.venv/bin/python'), ['-B', '-m', 'common.biometrics.shadow_worker'], options) + : state.mode === 'missing' ? actual.spawn('/nonexistent/shadow-test-python', [], options) : actual.spawn(process.execPath, ['-e', script], options) + child.stderr?.on('data', (chunk: Buffer) => { + const match = /descendant:(\d+)/.exec(chunk.toString()) + if (match) state.descendantPid = Number(match[1]) + }) + child.stdout?.on('data', (chunk: Buffer) => { + if (state.missingAfterFinish && chunk.toString().includes('"type":"finished"')) state.rss = null + }) + if (state.delayClose) { + // procfs can lose the address space before Node drains/closes its pipes. + child.once('exit', () => { + state.rss = null + }) + const emit = child.emit.bind(child) + child.emit = ((event: string, ...args: unknown[]) => { + if (event === 'close') { + setTimeout(() => emit(event, ...args), 150) + return true + } + return emit(event, ...args) + }) as typeof child.emit + } + state.pid = child.pid ?? 0 + return child + } } +}) +const hash = configHash(defaultConfig) +function init() { + return { schema: 'biometric-shadow-worker-v1', runId: randomUUID(), attemptId: randomUUID(), side: 'left', + startMs: 1700000000000, endMs: 1700000150000, configs: { left: defaultConfig, right: defaultConfig }, + configHashes: { left: hash, right: hash }, maximumRecords: 10, maximumOutputs: 10, maximumCpuMs: 1000, + maximumInputBytes: 1024 * 1024, maximumOutputBytes: 1024 * 1024 } +} +const budgets = { cpuSeconds: 2, addressSpaceBytes: 100000000, maximumRssBytes: 2000, wallMs: 2000, + acknowledgmentMs: 500, idleMs: 300, rssIntervalMs: 50, rssTimeoutMs: 100 } +async function* source() { + yield { type: 'frame', sequence: 0 } +} +const run = (frames: AsyncIterable = source(), options: Partial[4]> = {}) => + runShadowChild({ pythonPath: '/trusted/python', modulesPath: '/trusted/modules' }, init(), frames, () => {}, { budgets, ...options }) +function reaped() { + expect(() => process.kill(state.pid, 0)).toThrow() +} +beforeEach(() => { + state.mode = 'normal' + state.rss = 1000 + state.hangRss = false + state.delayClose = false + state.missingAfterFinish = false + state.pid = 0 + state.descendantPid = 0 + state.launches = [] +}) +describe('shadow child supervision', () => { + it('uses fixed shipped entrypoint, sanitized environment and bounded sequential replies', async () => { + const result = await run() + expect(result).toMatchObject({ health: { state: 'completed', records: 1 }, peakObservedRssBytes: 1000, exactKernelPeak: false }) + expect(state.launches[0]).toMatchObject(['/trusted/python', ['-B', '-m', 'common.biometrics.shadow_process', '--parent-pid', String(process.pid), '--wall-ms', '2000', '--cpu-seconds', '2', '--address-space-bytes', '100000000'], { detached: true }]) + const env = (state.launches[0][2] as { env: Record }).env + expect(Object.keys(env).sort()).toEqual(['LANG', 'MKL_NUM_THREADS', 'NODE_ENV', 'NUMEXPR_NUM_THREADS', 'OMP_NUM_THREADS', 'OPENBLAS_NUM_THREADS', 'PYTHONPATH', 'TZ']) + reaped() + }) + it.each(['bad', 'duplicate', 'flood', 'stderr', 'truncated', 'badexit'])('rejects %s worker output and reaps it', async (mode) => { + state.mode = mode + const observed = vi.fn() + await expect(run(source(), { onResources: observed })).rejects.toThrow(/Shadow/) + expect(observed).toHaveBeenCalledOnce() + expect(shadowPartialReceiptSchema.parse(observed.mock.calls[0][0]).cleanupConfirmed).toBe(true) + reaped() + }) + it.each(['ignore', 'noexit'])('enforces acknowledgment/exit deadlines for %s', async (mode) => { + state.mode = mode + await expect(run()).rejects.toThrow(/deadline/) + reaped() + }) + it('kills an idle source and invokes source cleanup without waiting for it', async () => { + const cleanup = vi.fn(() => new Promise>(() => {})) + const frames = { [Symbol.asyncIterator]: () => ({ next: () => new Promise>(() => {}), return: cleanup }) } + await expect(run(frames)).rejects.toThrow(/idle deadline/) + expect(cleanup).toHaveBeenCalledOnce() + reaped() + }) + it('honors cancellation while awaiting a source', async () => { + const controller = new AbortController() + const frames = { [Symbol.asyncIterator]: () => ({ next: () => new Promise>(() => {}) }) } + const task = run(frames, { signal: controller.signal }) + await delay(100) + controller.abort() + await expect(task).rejects.toThrow(/cancelled/) + reaped() + }) + it.each([null, 3000])('fails closed for unavailable/over-budget RSS %s', async (rss) => { + state.rss = rss + const observed = vi.fn() + await expect(run(source(), { onResources: observed })).rejects.toThrow(/RSS/) + expect(observed).toHaveBeenCalledOnce() + const receipt = shadowPartialReceiptSchema.parse(observed.mock.calls[0][0]) + expect(receipt.peakObservedRssBytes).toBe(rss) + expect(receipt.cleanupConfirmed).toBe(true) + if (rss === null) expect(receipt.rssSamples).toBe(0) + reaped() + }) + it('accepts disappearing RSS after completion only when clean pipe closure follows', async () => { + state.delayClose = true + const result = await run() + expect(result.health.state).toBe('completed') + expect(result.rssSamples).toBeGreaterThan(0) + reaped() + }) + it('still rejects a failed exit after completion when RSS disappears', async () => { + state.delayClose = true + state.mode = 'badexit' + await expect(run()).rejects.toThrow(/failed or returned an incomplete result/) + reaped() + }) + it('bounds missing RSS after a terminal response when the worker never exits', async () => { + state.mode = 'noexit' + state.missingAfterFinish = true + await expect(run()).rejects.toThrow(/exit deadline/) + reaped() + }) + it('bounds a stalled RSS observation', async () => { + state.hangRss = true + state.mode = 'ignore' + await expect(run()).rejects.toThrow(/RSS observation timed out/) + reaped() + }) + it('enforces wall time independently of a longer per-command deadline', async () => { + state.mode = 'ignore' + await expect(run(source(), { budgets: { ...budgets, wallMs: 100 } })).rejects.toThrow(/wall deadline/) + reaped() + }) + it('preserves source errors and still reaps the worker', async () => { + async function* broken() { + yield { type: 'frame', sequence: 0 } + throw new Error('source disconnected') + } + const observed = vi.fn() + await expect(run(broken(), { onResources: observed })).rejects.toThrow('source disconnected') + expect(observed).toHaveBeenCalledOnce() + expect(observed.mock.calls[0][0]).toMatchObject({ health: { state: 'running', records: 1 }, cleanupConfirmed: true }) + reaped() + }) + it('rejects input quota and out-of-order frames before sending them', async () => { + const task = runShadowChild({ pythonPath: '/trusted/python', modulesPath: '/trusted/modules' }, { ...init(), maximumInputBytes: 1 }, source(), () => {}, { budgets }) + await expect(task).rejects.toThrow(/input exceeds/) + reaped() + async function* wrong() { + yield { type: 'frame', sequence: 7 } + } + await expect(run(wrong())).rejects.toThrow() + reaped() + }) + it('does not launch for invalid config or cancelled admission', async () => { + await expect(runShadowChild({ pythonPath: '/trusted/python', modulesPath: '/trusted/modules' }, { ...init(), configHashes: { left: 'b'.repeat(64), right: hash } }, source(), () => {}, { budgets })).rejects.toThrow(/identity/) + await expect(run(source(), { signal: AbortSignal.abort() })).rejects.toThrow(/cancelled/) + expect(state.launches).toHaveLength(0) + }) +}) + +it('waits for the fenced output sink before requesting another frame', async () => { + state.mode = 'output' + let acknowledged = false + async function* frames() { + yield { type: 'frame', sequence: 0 } + expect(acknowledged).toBe(true) + } + const sink = vi.fn(async () => { + await delay(30) + acknowledged = true + }) + await runShadowChild({ pythonPath: '/trusted/python', modulesPath: '/trusted/modules' }, init(), frames(), sink, { budgets }) + expect(sink).toHaveBeenCalledOnce() + expect(sink.mock.calls[0]).toMatchObject([{ side: 'left', timestampMs: 1700000001000 }]) + reaped() +}) +it('bounds a hung sink by the independent wall deadline', async () => { + state.mode = 'output' + await expect(runShadowChild({ pythonPath: '/trusted/python', modulesPath: '/trusted/modules' }, init(), source(), + () => new Promise(() => {}), { budgets: { ...budgets, wallMs: 200 } })).rejects.toThrow(/wall deadline/) + reaped() +}) +it('preserves storage errors while terminating the worker', async () => { + state.mode = 'output' + await expect(runShadowChild({ pythonPath: '/trusted/python', modulesPath: '/trusted/modules' }, init(), source(), + () => { throw new Error('lease replaced') }, { budgets })).rejects.toThrow('lease replaced') + reaped() +}) +it('kills descendants that retain output pipes after the worker exits', async () => { + state.mode = 'descendant' + await expect(run()).rejects.toThrow(/exit deadline/) + reaped() + expect(state.descendantPid).toBeGreaterThan(0) + let exists = true + for (let attempt = 0; exists && attempt < 50; attempt++) { + try { + process.kill(state.descendantPid, 0) + } + catch { exists = false } + if (exists) await delay(20) + } + expect(exists).toBe(false) +}) +it('handles failed process creation without an unhandled event or cleanup hang', async () => { + state.mode = 'missing' + const observed = vi.fn() + await expect(run(source(), { onResources: observed })).rejects.toThrow(/could not start/) + expect(observed.mock.calls[0][0]).toMatchObject({ health: null, rssSamples: 0, peakObservedRssBytes: null, cleanupConfirmed: true }) + expect(state.pid).toBe(0) +}) +it.skipIf(!existsSync(resolve('modules/piezo-processor/.venv/bin/python')))('supervises the actual locked Python transport through clean completion', async () => { + state.mode = 'python' + async function* frames() { + yield { type: 'frame', sequence: 0, observedAtMs: 1700000000000, record: { type: 'frzHealth' }, + context: { activeSides: ['left'], leftOccupied: true, rightOccupied: false, pumpContextKnown: true } } + } + // Kernel bootstrap is verified separately on Linux; this host test exercises + // real Python pipe behavior and protocol validation through the supervisor. + const result = await runShadowChild({ pythonPath: '/trusted/python', modulesPath: resolve('modules') }, init(), frames(), () => {}, + { budgets: { ...budgets, acknowledgmentMs: 5000, wallMs: 10000 } }) + expect(result.health).toMatchObject({ state: 'completed', records: 1 }) + reaped() +}) +it('enforces cumulative output and frame input limits', async () => { + await expect(runShadowChild({ pythonPath: '/trusted/python', modulesPath: '/trusted/modules' }, { ...init(), maximumOutputBytes: 10 }, source(), + () => {}, { budgets })).rejects.toThrow(/output exceeds byte quota/) + reaped() + async function* oversized() { + yield { type: 'frame', sequence: 0, payload: 'x'.repeat(128 * 1024) } + } + await expect(run(oversized())).rejects.toThrow(/input exceeds byte quota/) + reaped() +}) +it('does not send frames beyond the frozen record quota', async () => { + async function* two() { + yield { type: 'frame', sequence: 0 } + yield { type: 'frame', sequence: 1 } + } + await expect(runShadowChild({ pythonPath: '/trusted/python', modulesPath: '/trusted/modules' }, { ...init(), maximumRecords: 1 }, two(), + () => {}, { budgets })).rejects.toThrow(/record quota/) + reaped() +}) + +it('does not hide failed process-group cleanup behind the original protocol error', async () => { + state.mode = 'bad' + const kill = process.kill.bind(process) + const spy = vi.spyOn(process, 'kill').mockImplementation((pid, signal) => { + if (pid < 0 && signal === 'SIGKILL') throw Object.assign(new Error('denied'), { code: 'EPERM' }) + return kill(pid, signal) + }) + try { + const observed = vi.fn() + await expect(run(source(), { onResources: observed })).rejects.toMatchObject({ code: 'SHADOW_CLEANUP_UNCONFIRMED', + message: 'shadow_worker_cleanup_unconfirmed', cause: expect.objectContaining({ message: 'Shadow worker emitted an invalid response' }) }) + expect(observed.mock.calls[0][0]).toMatchObject({ cleanupConfirmed: false }) + } + finally { + spy.mockRestore() + if (state.pid) { + try { + kill(-state.pid, 'SIGKILL') + } + catch { /* The direct child may have exited when its pipes closed. */ } + } + } +}) diff --git a/src/biometrics/shadow/tests/worker-protocol.test.ts b/src/biometrics/shadow/tests/worker-protocol.test.ts new file mode 100644 index 000000000..4284e7a36 --- /dev/null +++ b/src/biometrics/shadow/tests/worker-protocol.test.ts @@ -0,0 +1,84 @@ +// @vitest-environment node +import { randomUUID } from 'node:crypto' +import { existsSync } from 'node:fs' +import { spawnSync } from 'node:child_process' +import { resolve } from 'node:path' +import { describe, expect, it } from 'vitest' +import { defaultConfig } from '../../algorithms/registry' +import { configHash } from '../../algorithms/canonical' +import { ShadowWorkerProtocol } from '../worker-protocol' + +const hash = 'a'.repeat(64), start = 1700000000000 +const expected = { runId: randomUUID(), attemptId: randomUUID(), side: 'left', startMs: start, endMs: start + 150000, + configHashes: { left: hash, right: hash }, maximumRecords: 200, maximumOutputs: 10, maximumCpuMs: 30000 } +function fixture() { + const protocol = new ShadowWorkerProtocol(expected) + const ready = { type: 'ready', runId: expected.runId, attemptId: expected.attemptId, configHashes: expected.configHashes } + const output = { sequence: 0, inputHash: hash, side: 'left', sourceSide: 'right', timestampMs: start, configHash: hash, + readings: [{ metric: 'heart_rate', unit: 'bpm', windowMs: null, value: 60, quality: 1 }, + { metric: 'breathing_rate', unit: 'breaths/min', windowMs: null, value: null, quality: 0 }, + { metric: 'window_ibi_variability_v1', unit: 'ms', windowMs: 300000, value: null, quality: 0 }], flags: null, limitations: ['cold_start_without_serialized_state'] } + const frame = { type: 'frame_result', runId: expected.runId, attemptId: expected.attemptId, sequence: 0, outputs: [output], + health: { state: 'running', reason: null, records: 1, outputs: 1, cpu_ms: 1, input_hash: hash, limitations: [], + counts: { records: 1, accepted_frames: 1, empty_channel_frames: 0, pump_gated_frames: 0, preroll_outputs: 0, scored_outputs: 1 } } } + return { protocol, ready, frame } +} +describe('shadow worker response validation', () => { + it('requires matching initialization and validates a complete typed batch', () => { + const { protocol, ready, frame } = fixture() + expect(() => protocol.accept(frame, 'frame_result')).toThrow('initialization') + protocol.accept(ready, 'ready') + expect(protocol.accept(frame, 'frame_result')).toMatchObject({ outputs: [{ side: 'left', sourceSide: 'right' }] }) + expect(() => protocol.accept(frame, 'frame_result')).toThrow('sequence') + }) + it('rejects stale identities, side/config/window changes, metric substitutions and forged health without advancing', () => { + const { protocol, ready, frame } = fixture() + protocol.accept(ready, 'ready') + for (const changed of [{ ...frame, attemptId: randomUUID() }, { ...frame, sequence: 1 }, + { ...frame, health: { ...frame.health, outputs: 2 } }, { ...frame, health: { ...frame.health, cpu_ms: 30001 } }]) { + expect(() => protocol.accept(changed, 'frame_result')).toThrow() + } + for (const changes of [{ side: 'right' }, { configHash: 'b'.repeat(64) }, { timestampMs: start - 1 }, { timestampMs: expected.endMs }, + { readings: frame.outputs[0].readings.map(row => row.metric === 'window_ibi_variability_v1' ? { ...row, metric: 'hrv_rmssd' } : row) }]) { + expect(() => protocol.accept({ ...frame, outputs: [{ ...frame.outputs[0], ...changes }] }, 'frame_result')).toThrow() + } + expect(protocol.accept(frame, 'frame_result').type).toBe('frame_result') + }) + it('requires the requested terminal command and closes after acknowledgment', () => { + const { protocol, ready, frame } = fixture() + protocol.accept(ready, 'ready') + protocol.accept(frame, 'frame_result') + const finish = { type: 'finished', runId: expected.runId, attemptId: expected.attemptId, outputs: [], health: { ...frame.health, state: 'completed' } } + expect(() => protocol.accept(finish, 'frame_result')).toThrow('command') + protocol.accept(finish, 'finished') + expect(() => protocol.accept(frame, 'frame_result')).toThrow('completion') + }) +}) +const python = resolve('modules/piezo-processor/.venv/bin/python') +it.skipIf(!existsSync(python))('validates actual locked-Python worker output across the process boundary', () => { + const configs = { left: defaultConfig, right: defaultConfig } + const configHashes = { left: configHash(defaultConfig), right: configHash(defaultConfig) } + const init = { ...expected, configHashes, schema: 'biometric-shadow-worker-v1', configs, + maximumInputBytes: 2 * 1024 * 1024, maximumOutputBytes: 512 * 1024 } + const messages: unknown[] = [init] + for (let second = 0; second < 150; second++) { + const left = Buffer.alloc(2000) + for (let index = 0; index < 500; index++) left.writeInt32LE(Math.trunc(Math.sin((second * 500 + index) / 500 * 2 * Math.PI * 1.2) * 1000000), index * 4) + messages.push({ type: 'frame', sequence: second, observedAtMs: start + second * 1000, + record: { type: 'piezo-dual', left1: left.toString('base64'), right1: Buffer.alloc(2000).toString('base64') }, + context: { activeSides: ['left'], leftOccupied: true, rightOccupied: false, pumpContextKnown: true } }) + } + messages.push({ type: 'finish' }) + const result = spawnSync(python, ['-m', 'common.biometrics.shadow_worker'], { env: { ...process.env, PYTHONPATH: resolve('modules') }, + input: messages.map(message => JSON.stringify(message)).join('\n') + '\n', encoding: 'utf8', timeout: 30000, maxBuffer: 2 * 1024 * 1024 }) + expect(result.status, result.stderr).toBe(0) + const replies = result.stdout.trim().split('\n').map(line => JSON.parse(line)) + const protocol = new ShadowWorkerProtocol({ ...expected, configHashes }) + let outputs = 0 + replies.forEach((reply, index) => { + const result = protocol.accept(reply, index === 0 ? 'ready' : index === replies.length - 1 ? 'finished' : 'frame_result') + if ('outputs' in result) outputs += result.outputs.length + }) + expect(replies).toHaveLength(152) + expect(outputs).toBeGreaterThan(0) +}, 30000) diff --git a/src/biometrics/shadow/worker-child.ts b/src/biometrics/shadow/worker-child.ts new file mode 100644 index 000000000..e86bd0ad1 --- /dev/null +++ b/src/biometrics/shadow/worker-child.ts @@ -0,0 +1,223 @@ +import type { ShadowPartialReceipt } from './receipt' +import { shadowChildBudgetsSchema as budgetsSchema } from './execution-policy' +import { confirmShadowProcessCleanup } from './process-cleanup' +import { ShadowCleanupError } from './cleanup-error' +import { spawn } from 'node:child_process' +import { tmpdir } from 'node:os' +import { z } from 'zod' +import { configHash } from '../algorithms/canonical' +import { validateConfig } from '../algorithms/registry' +import { sampleProcessGroupRss } from '../backtest/process-group-memory' +import { ShadowWorkerProtocol, type shadowWorkerOutputSchema } from './worker-protocol' + +const integer = z.number().int().positive().max(Number.MAX_SAFE_INTEGER) +const hash = z.string().regex(/^[a-f0-9]{64}$/) +const initSchema = z.object({ schema: z.literal('biometric-shadow-worker-v1'), + runId: z.string().uuid(), attemptId: z.string().uuid(), side: z.enum(['left', 'right']), + startMs: integer, endMs: integer, configs: z.object({ left: z.unknown(), right: z.unknown() }).strict(), + configHashes: z.object({ left: hash, right: hash }).strict(), maximumRecords: integer, + maximumOutputs: integer, maximumCpuMs: integer, maximumInputBytes: integer, maximumOutputBytes: integer, +}).strict() +export type ShadowChildBudgets = z.infer +type Output = z.infer +type Health = Extract, { type: 'finished' }>['health'] +const LINE_BYTES = 128 * 1024 + +/** Trusted host paths and shipped code only. One frame in flight; the source + * must itself bound buffering. Callers fence every output through the store. + * This supervisor never opens a database, source, network or hardware handle. */ +export async function runShadowChild(location: { pythonPath: string, modulesPath: string }, rawInit: unknown, + frames: AsyncIterable, onOutput: (output: Output) => void | Promise, + options: { budgets: ShadowChildBudgets, signal?: AbortSignal, onResources?: (receipt: ShadowPartialReceipt) => void }) { + const init = initSchema.parse(structuredClone(rawInit)) + const budgets = budgetsSchema.parse(options.budgets) + for (const side of ['left', 'right'] as const) { + validateConfig(init.configs[side]) + if (configHash(init.configs[side]) !== init.configHashes[side]) throw new Error('Shadow configuration identity mismatch') + } + const protocol = new ShadowWorkerProtocol({ runId: init.runId, attemptId: init.attemptId, side: init.side, + startMs: init.startMs, endMs: init.endMs, configHashes: init.configHashes, + maximumRecords: init.maximumRecords, maximumOutputs: init.maximumOutputs, maximumCpuMs: init.maximumCpuMs }) + if (options.signal?.aborted) throw new Error('Shadow worker cancelled') + const iterator = frames[Symbol.asyncIterator]() + const child = spawn(location.pythonPath, ['-B', '-m', 'common.biometrics.shadow_process', + '--parent-pid', String(process.pid), '--wall-ms', String(budgets.wallMs), '--cpu-seconds', String(budgets.cpuSeconds), '--address-space-bytes', String(budgets.addressSpaceBytes)], { + detached: true, cwd: tmpdir(), stdio: ['pipe', 'pipe', 'pipe'], + env: { NODE_ENV: 'production', PYTHONPATH: location.modulesPath, LANG: 'C.UTF-8', TZ: 'UTC', + OPENBLAS_NUM_THREADS: '1', OMP_NUM_THREADS: '1', MKL_NUM_THREADS: '1', NUMEXPR_NUM_THREADS: '1' }, + }) + let lastHealth: Health | null = null + let cleanupUnconfirmed = false + let failure: Error | undefined, closed = false, terminal = false, stopping = false + let inputBytes = 0, outputBytes = 0, diagnosticBytes = 0, peakRssBytes = 0, rssSamples = 0 + let lastRssAt = performance.now(), maximumObservationGapMs = 0 + let pending = Buffer.alloc(0) + let reply: { type: Parameters[1], resolve: (value: ReturnType) => void } | undefined + let rejectFailure!: (error: Error) => void + const failed = new Promise((_resolve, reject) => { + rejectFailure = reject + }) + // A source/sink can be awaiting its own I/O when the process fails. + void failed.catch(() => {}) + const fail = (message: string) => { + if (!failure) { + failure = new Error(message) + rejectFailure(failure) + } + } + const abort = () => fail('Shadow worker cancelled') + options.signal?.addEventListener('abort', abort, { once: true }) + if (options.signal?.aborted) abort() + const wall = setTimeout(() => fail('Shadow worker wall deadline exceeded'), budgets.wallMs) + const exited = new Promise((resolve) => { + child.once('close', (code) => { + closed = true + if (!stopping && (code !== 0 || !terminal || pending.length)) fail('Shadow worker failed or returned an incomplete result') + resolve() + }) + }) + child.on('error', () => fail('Shadow worker could not start')) + child.stdin.on('error', () => fail('Shadow worker input pipe closed')) + child.stdout.on('error', () => fail('Shadow worker output pipe failed')) + child.stderr.on('error', () => fail('Shadow worker diagnostic pipe failed')) + child.stderr.on('data', (chunk: Buffer) => { + diagnosticBytes += chunk.length + if (diagnosticBytes > 16384) fail('Shadow diagnostics exceed byte quota') + }) + child.stdout.on('data', (chunk: Buffer) => { + if (failure) return + outputBytes += chunk.length + if (outputBytes > init.maximumOutputBytes) return fail('Shadow output exceeds byte quota') + // Only one reply is permitted per command; never queue unsolicited output. + const newline = chunk.indexOf(10) + if (pending.length + (newline < 0 ? chunk.length : newline + 1) > LINE_BYTES) return fail('Shadow response line exceeds byte quota') + pending = Buffer.concat([pending, chunk]) + const end = pending.indexOf(10) + if (end < 0) return + if (!reply || end !== pending.length - 1) return fail('Shadow worker emitted unsolicited output') + try { + const line = new TextDecoder('utf-8', { fatal: true }).decode(pending.subarray(0, end)) + const response = protocol.accept(JSON.parse(line), reply.type) + if ('health' in response) lastHealth = structuredClone(response.health) + const acknowledge = reply.resolve + reply = undefined + pending = Buffer.alloc(0) + if (response.type === 'finished') terminal = true + acknowledge(response) + } + catch { fail('Shadow worker emitted an invalid response') } + }) + async function bounded(operation: Promise, ms: number, reason: string): Promise { + const timer = setTimeout(() => fail(reason), ms) + try { + return await Promise.race([operation, failed]) + } + finally { clearTimeout(timer) } + } + let monitoring = false + const sample = async () => { + if (monitoring || closed || failure) return + monitoring = true + try { + const rss = await bounded(Promise.race([sampleProcessGroupRss(child.pid ?? 0), exited.then(() => null)]), budgets.rssTimeoutMs, 'Shadow RSS observation timed out') + if (closed || stopping || failure) return + // procfs can lose a finished worker before Node delivers pipe closure. + // Accept that disappearance only after a validated terminal response and + // a bounded, successful close; the close handler still rejects bad exits. + if (rss === null && terminal) { + await bounded(exited, budgets.acknowledgmentMs, 'Shadow worker exit deadline exceeded') + return + } + if (rss === null || !Number.isSafeInteger(rss) || rss <= 0) return fail('Shadow RSS observation unavailable') + const observedAt = performance.now() + maximumObservationGapMs = Math.max(maximumObservationGapMs, observedAt - lastRssAt) + lastRssAt = observedAt + rssSamples++ + peakRssBytes = Math.max(peakRssBytes, rss) + if (rss > budgets.maximumRssBytes) fail('Shadow RSS quota exceeded') + } + catch { + if (!closed && !stopping) fail('Shadow RSS observation unavailable') + } + finally { monitoring = false } + } + const memoryTimer = setInterval(() => { + void sample() + }, budgets.rssIntervalMs) + child.once('spawn', () => { + void sample() + }) + async function request(message: unknown, type: Parameters[1]) { + if (failure) throw failure + if (closed || terminal || reply) throw new Error('Shadow worker is not accepting commands') + const line = JSON.stringify(message) + '\n' + const bytes = Buffer.byteLength(line) + if (bytes > LINE_BYTES || inputBytes + bytes > init.maximumInputBytes) throw new Error('Shadow input exceeds byte quota') + inputBytes += bytes + const response = new Promise>((resolve) => { + reply = { type, resolve } + }) + child.stdin.write(line) + return bounded(response, budgets.acknowledgmentMs, 'Shadow acknowledgment deadline exceeded') + } + let result: { health: Health, inputBytes: number, outputBytes: number, peakObservedRssBytes: number, rssSamples: number, maximumObservationGapMs: number, budgets: ShadowChildBudgets, memoryScope: 'sum_rss_of_shadow_process_group', exactKernelPeak: false } | undefined + try { + await request(init, 'ready') + let records = 0 + while (true) { + const next = await bounded(Promise.resolve().then(() => iterator.next()), budgets.idleMs, 'Shadow source idle deadline exceeded') + if (next.done) break + if (records >= init.maximumRecords) throw new Error('Shadow record quota exceeded') + const frame = z.object({ type: z.literal('frame'), sequence: z.literal(records) }).passthrough().parse(next.value) + const response = await request(frame, 'frame_result') + if (response.type !== 'frame_result') throw new Error('Shadow frame acknowledgment missing') + for (const output of response.outputs) await Promise.race([Promise.resolve().then(() => onOutput(output)), failed]) + records++ + } + const response = await request({ type: 'finish' }, 'finished') + if (response.type !== 'finished') throw new Error('Shadow completion missing') + for (const output of response.outputs) await Promise.race([Promise.resolve().then(() => onOutput(output)), failed]) + child.stdin.end() + await bounded(exited, budgets.acknowledgmentMs, 'Shadow worker exit deadline exceeded') + if (!rssSamples) throw new Error('Shadow worker has no RSS observations') + if (failure) throw failure + result = { health: response.health, inputBytes, outputBytes, peakObservedRssBytes: peakRssBytes, rssSamples, maximumObservationGapMs: Math.max(maximumObservationGapMs, performance.now() - lastRssAt), + budgets, memoryScope: 'sum_rss_of_shadow_process_group', exactKernelPeak: false } + } + catch (error) { + fail(error instanceof Error ? error.message : 'Shadow source or sink failed') + } + finally { + stopping = true + clearTimeout(wall) + clearInterval(memoryTimer) + options.signal?.removeEventListener('abort', abort) + // Terminate the entire detached group, including a child retaining pipes. + // SIGKILL is deliberate: no canonical state exists in this worker to flush. + if (child.pid) { + try { + process.kill(-child.pid, 'SIGKILL') + } + catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ESRCH') cleanupUnconfirmed = true + } + } + child.stdin.destroy() + child.stdout.destroy() + child.stderr.destroy() + // Iterator cleanup may itself be stalled; never let it keep a worker alive. + if (iterator.return) void Promise.resolve().then(() => iterator.return?.()).catch(() => {}) + if (!await confirmShadowProcessCleanup(child.pid, () => closed)) cleanupUnconfirmed = true + } + try { + options.onResources?.({ kind: 'partial_supervisor_observation', health: lastHealth, + inputBytes, outputBytes, diagnosticBytes, peakObservedRssBytes: rssSamples ? peakRssBytes : null, rssSamples, + maximumObservationGapMs: Math.max(maximumObservationGapMs, performance.now() - lastRssAt), budgets, + cleanupConfirmed: !cleanupUnconfirmed, memoryScope: 'sum_rss_of_shadow_process_group', exactKernelPeak: false }) + } + catch (error) { fail(error instanceof Error ? error.message : 'Shadow resource observation failed') } + if (cleanupUnconfirmed) throw new ShadowCleanupError(failure) + if (failure) throw failure + if (!result) throw new Error('Shadow completion missing') + return result +} diff --git a/src/biometrics/shadow/worker-protocol.ts b/src/biometrics/shadow/worker-protocol.ts new file mode 100644 index 000000000..fb8fc6887 --- /dev/null +++ b/src/biometrics/shadow/worker-protocol.ts @@ -0,0 +1,87 @@ +import { z } from 'zod' +import { configHash } from '../algorithms/canonical' + +const integer = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) +const hash = z.string().regex(/^[a-f0-9]{64}$/) +const side = z.enum(['left', 'right']) +const reading = { value: z.number().finite().nullable(), quality: z.number().min(0).max(1) } +const metricReading = z.discriminatedUnion('metric', [ + z.object({ metric: z.literal('heart_rate'), unit: z.literal('bpm'), windowMs: z.null(), ...reading }).strict(), + z.object({ metric: z.literal('breathing_rate'), unit: z.literal('breaths/min'), windowMs: z.null(), ...reading }).strict(), + z.object({ metric: z.literal('window_ibi_variability_v1'), unit: z.literal('ms'), windowMs: z.literal(300000), ...reading }).strict(), +]) +export const shadowWorkerOutputSchema = z.object({ sequence: integer, inputHash: hash, side, sourceSide: side, + timestampMs: integer, configHash: hash, readings: z.array(metricReading).length(3), + flags: z.array(z.string().max(200)).max(32).nullable(), limitations: z.array(z.string().max(200)).max(32), +}).strict().refine(output => new Set(output.readings.map(row => row.metric)).size === 3, 'All typed metrics required') +export const shadowWorkerHealthSchema = z.object({ state: z.enum(['running', 'completed', 'stopped']), reason: z.string().max(500).nullable(), + records: integer, outputs: integer, cpu_ms: z.number().finite().nonnegative(), input_hash: hash, + counts: z.object({ records: integer, accepted_frames: integer, empty_channel_frames: integer, + pump_gated_frames: integer, preroll_outputs: integer, scored_outputs: integer }).strict(), + limitations: z.array(z.string().max(200)).max(32), +}).strict() +const identity = { runId: z.string().uuid(), attemptId: z.string().uuid() } +const hashes = z.object({ left: hash, right: hash }).strict() +const responseSchema = z.discriminatedUnion('type', [ + z.object({ ...identity, type: z.literal('ready'), configHashes: hashes }).strict(), + z.object({ ...identity, type: z.literal('frame_result'), sequence: integer, outputs: z.array(shadowWorkerOutputSchema).max(4), health: shadowWorkerHealthSchema }).strict(), + z.object({ ...identity, type: z.literal('finished'), outputs: z.array(shadowWorkerOutputSchema).max(4), health: shadowWorkerHealthSchema }).strict(), + z.object({ ...identity, type: z.literal('cancelled'), health: shadowWorkerHealthSchema }).strict(), +]) +const expectedSchema = z.object({ ...identity, side, startMs: integer, endMs: integer, configHashes: hashes, + maximumRecords: integer.positive(), maximumOutputs: integer.positive(), maximumCpuMs: integer.positive(), +}).strict().refine(value => value.startMs < value.endMs, 'Invalid shadow response window') + +/** Stateful parent-side validation. A rejected batch changes no counters, so + * callers cannot accidentally accept partial unvalidated output from a reply. */ +export class ShadowWorkerProtocol { + private readonly expected: z.infer + private ready = false + private closed = false + private records = 0 + private outputs = 0 + private lastTimestamp = -1 + private cpuMs = 0 + + constructor(expected: unknown) { + this.expected = expectedSchema.parse(structuredClone(expected)) + } + + accept(raw: unknown, expectedType: 'ready' | 'frame_result' | 'finished' | 'cancelled') { + if (this.closed) throw new Error('Shadow response after completion') + const response = responseSchema.parse(raw) + const expected = this.expected + if (response.runId !== expected.runId || response.attemptId !== expected.attemptId || response.type !== expectedType) throw new Error('Shadow response identity or command mismatch') + if (response.type === 'ready') { + if (this.ready || configHash(response.configHashes) !== configHash(expected.configHashes)) throw new Error('Shadow ready identity mismatch') + this.ready = true + return response + } + if (!this.ready) throw new Error('Shadow worker has not acknowledged initialization') + const records = this.records + (response.type === 'frame_result' ? 1 : 0) + if (response.type === 'frame_result' && response.sequence !== this.records) throw new Error('Shadow acknowledgment sequence mismatch') + const batch = response.type === 'cancelled' ? [] : response.outputs + const outputCount = this.outputs + batch.length + if (records > expected.maximumRecords || outputCount > expected.maximumOutputs) throw new Error('Shadow response exceeds run quota') + let timestamp = this.lastTimestamp + for (const [index, output] of batch.entries()) { + if (output.sequence !== this.outputs + index || output.side !== expected.side || output.configHash !== expected.configHashes[expected.side] + || output.timestampMs < expected.startMs || output.timestampMs >= expected.endMs || output.timestampMs <= timestamp) throw new Error('Shadow output identity, order or window mismatch') + timestamp = output.timestampMs + } + const health = response.health + const state = response.type === 'frame_result' ? 'running' : response.type === 'finished' ? 'completed' : 'stopped' + if (health.state !== state || health.records !== records || health.counts.records !== records || health.outputs !== outputCount + || health.cpu_ms < this.cpuMs || health.cpu_ms > expected.maximumCpuMs + || health.counts.accepted_frames + health.counts.empty_channel_frames + health.counts.pump_gated_frames > records + || health.counts.scored_outputs < outputCount + || health.counts.scored_outputs + health.counts.preroll_outputs > 2 * health.counts.accepted_frames + || batch.some(output => output.inputHash !== health.input_hash)) throw new Error('Shadow health receipt inconsistent') + this.records = records + this.outputs = outputCount + this.lastTimestamp = timestamp + this.cpuMs = health.cpu_ms + this.closed = response.type !== 'frame_result' + return response + } +} diff --git a/src/biometrics/shadow/worker-sink.ts b/src/biometrics/shadow/worker-sink.ts new file mode 100644 index 000000000..25a36a2ac --- /dev/null +++ b/src/biometrics/shadow/worker-sink.ts @@ -0,0 +1,48 @@ +import { z } from 'zod' +import type { ShadowStore } from './store' +import { type ShadowLease, shadowActiveProvenanceSchema } from './store' +import { shadowWorkerOutputSchema } from './worker-protocol' + +export const shadowActiveReadingSchema = z.object({ side: z.enum(['left', 'right']), timestampMs: z.number().int().nonnegative(), + provenance: shadowActiveProvenanceSchema.optional(), + configHash: z.string().regex(/^[a-f0-9]{64}$/), inputHash: z.string().regex(/^[a-f0-9]{64}$/).nullable().optional(), readings: shadowWorkerOutputSchema.shape.readings, +}).strict().refine(value => new Set(value.readings.map(row => row.metric)).size === 3, 'All active metric identities required') + +/** Fenced candidate-output sink. The supervisor validates response batches with + * ShadowWorkerProtocol first and supplies actual contemporaneous active data. + * Missing active data stays missing; no nearest-time substitution is allowed. */ +export class ShadowWorkerSink { + private readonly baseSequence: number + private nextOutput = 0 + private readonly plan: ReturnType['plan'] + + constructor(private readonly store: ShadowStore, private readonly lease: ShadowLease) { + const run = store.get(lease.runId) + if (run.state !== 'running' || run.attempt_id !== lease.attemptId) throw new Error('Shadow sink attempt mismatch') + this.baseSequence = run.point_count + this.plan = structuredClone(run.plan) + } + + append(rawOutput: unknown, rawActive: unknown | null) { + const output = shadowWorkerOutputSchema.parse(rawOutput) + const active = rawActive === null ? null : shadowActiveReadingSchema.parse(rawActive) + if (output.sequence > this.nextOutput || output.side !== this.plan.side || output.configHash !== this.plan.candidateConfigHash) throw new Error('Shadow sink output identity mismatch') + if (active && (active.side !== this.plan.side || active.configHash !== this.plan.activeConfigHash || active.timestampMs !== output.timestampMs)) throw new Error('Active reading is not contemporaneous with shadow output') + const readings = this.plan.metrics.map((metric) => { + const candidate = output.readings.find(row => row.metric === metric) + const baseline = active?.readings.find(row => row.metric === metric) + if (!candidate || (active && !baseline)) throw new Error('Missing sink metric') + const expectedWindow = metric === 'window_ibi_variability_v1' ? this.plan.windowIbiWindowMs : null + if (candidate.windowMs !== expectedWindow || (baseline && baseline.windowMs !== expectedWindow)) throw new Error('Shadow sink metric window mismatch') + return { metric, candidate: { value: candidate.value, quality: candidate.quality }, + active: baseline ? { value: baseline.value, quality: baseline.quality } : { value: null, quality: 0 } } + }) + const result = this.store.append(this.lease, { sequence: this.baseSequence + output.sequence, side: output.side, timestampMs: output.timestampMs, + inputHash: output.inputHash, candidateConfigHash: output.configHash, activeConfigHash: this.plan.activeConfigHash, readings, + ...(active?.provenance ? { activeProvenance: active.provenance } : {}), + candidateProvenance: { sourceSide: output.sourceSide, outputSequence: output.sequence, flags: output.flags, + limitations: active ? output.limitations : [...output.limitations, 'missing_contemporaneous_active_reading'] } }) + if (result.state !== 'quota_reached') this.nextOutput = Math.max(this.nextOutput, output.sequence + 1) + return result + } +} diff --git a/src/biometrics/tests/background-admission.test.ts b/src/biometrics/tests/background-admission.test.ts new file mode 100644 index 000000000..d29c87980 --- /dev/null +++ b/src/biometrics/tests/background-admission.test.ts @@ -0,0 +1,45 @@ +// @vitest-environment node +import { afterEach, expect, it, vi } from 'vitest' +import { withBiometricBackgroundSlot } from '../background-admission' +const shared = globalThis as typeof globalThis & { __biometricBackgroundSlotsV1?: unknown } +afterEach(() => { + delete shared.__biometricBackgroundSlotsV1 +}) +it('reserves work synchronously while leaving one report read available', async () => { + let finish!: () => void + const work = withBiometricBackgroundSlot('work', () => new Promise((resolve) => { + finish = resolve + })) + const other = vi.fn(async () => null) + await expect(withBiometricBackgroundSlot('work', other)).rejects.toMatchObject({ code: 'RESOURCE_LIMIT' }) + expect(other).not.toHaveBeenCalled() + expect(await withBiometricBackgroundSlot('read', async () => 'report')).toBe('report') + finish() + await work + expect(await withBiometricBackgroundSlot('work', other)).toBeNull() +}) +it('shares ownership across module reloads and releases ordinary failures', async () => { + let finish!: () => void + const work = withBiometricBackgroundSlot('work', () => new Promise((resolve) => { + finish = resolve + })) + vi.resetModules() + const copy = await import('../background-admission') + await expect(copy.withBiometricBackgroundSlot('work', async () => null)).rejects.toMatchObject({ code: 'RESOURCE_LIMIT' }) + finish() + await work + await expect(copy.withBiometricBackgroundSlot('work', async () => { + throw new Error('failed') + })).rejects.toThrow('failed') + expect(() => withBiometricBackgroundSlot('work', () => { + throw new Error('synchronous') + })).toThrow('synchronous') + expect(await withBiometricBackgroundSlot('work', async () => 'available')).toBe('available') +}) +it.each(['SHADOW_CLEANUP_UNCONFIRMED', 'BIOMETRIC_COORDINATOR_CLEANUP_UNCONFIRMED'])('quarantines uncertain cleanup: %s', async (code) => { + await expect(withBiometricBackgroundSlot('work', async () => { + throw Object.assign(new Error('cleanup'), { code }) + })).rejects.toMatchObject({ code }) + await expect(withBiometricBackgroundSlot('work', async () => null)).rejects.toThrow('cleanup is unconfirmed') + expect(await withBiometricBackgroundSlot('read', async () => 'diagnostics')).toBe('diagnostics') +}) diff --git a/src/components/Settings/SettingsScreen.tsx b/src/components/Settings/SettingsScreen.tsx index 920d46c99..8fce69d2e 100644 --- a/src/components/Settings/SettingsScreen.tsx +++ b/src/components/Settings/SettingsScreen.tsx @@ -22,6 +22,7 @@ import { TapGestureConfig } from './TapGestureConfig' import { MqttSettingsForm } from './MqttSettingsForm' import { HomeKitConfig } from './HomeKitConfig' import { ArchivePushSettingsForm } from './ArchivePushSettingsForm' +import { BiometricCalibration } from '@/src/components/biometrics/Calibration/BiometricCalibration' import { SleepStageRules } from './SleepStageRules' import { AppearanceSettings, TempControlPicker, TempDisplayControl, ThemeControl, TimeFormatControl, UnitsControl } from './AppearanceSettings' import { SectionColumns } from './SettingsLayout' @@ -125,6 +126,8 @@ function SectionBody({ section, side, onSideChange }: { return {data => } case 'gestures': return + case 'biometric-calibration': + return case 'sleep-stages': return case 'appearance': diff --git a/src/components/Settings/SleepStageRules.tsx b/src/components/Settings/SleepStageRules.tsx index 9343a2900..9998401f7 100644 --- a/src/components/Settings/SleepStageRules.tsx +++ b/src/components/Settings/SleepStageRules.tsx @@ -40,12 +40,14 @@ const sameRuleset = (a: StageRuleset, b: StageRuleset) => JSON.stringify(a) === export function SleepStageRules({ side, onSideChange }: { side: Side, onSideChange: (side: Side) => void }) { const utils = trpc.useUtils() const { leftName, rightName } = useSideNames() - const query = trpc.sleepStages.getClassifier.useQuery({ side }) + const query = trpc.sleepStages.getClassifier.useQuery({ side }, { + refetchInterval: query => query.state.data?.pending ? 5000 : false, + }) const save = trpc.sleepStages.setClassifier.useMutation({ onSuccess: () => utils.sleepStages.getClassifier.invalidate({ side }), }) - const saved = query.data?.classifier.ruleset + const saved = query.data?.pending?.classifier.ruleset ?? query.data?.classifier.ruleset const [draft, setDraft] = useState(null) const [editing, setEditing] = useState(null) const [nights, setNights] = useState(1) @@ -60,7 +62,7 @@ export function SleepStageRules({ side, onSideChange }: { side: Side, onSideChan } const validation = draft ? stageRulesetSchema.safeParse(draft) : null - const preview = useStagePreview(side, validation?.success ? validation.data : null, saved ?? null, nights) + const preview = useStagePreview(side, validation?.success ? validation.data : null, query.data?.classifier.ruleset ?? null, nights) if (query.error) { return ( @@ -90,6 +92,14 @@ export function SleepStageRules({ side, onSideChange }: { side: Side, onSideChan return (
+ {query.data.pending && ( +

+ {`Saved change ${query.data.pending.generation}; waiting for processor adoption. Displayed nights still use the acknowledged rules.`} +

+ )} + {(query.data.pending?.warning ?? query.data.provenance?.warning) && ( + {query.data.pending?.warning ?? query.data.provenance?.warning} + )} {/* Toolbar: profile, state, preset, reset */}
save.mutate({ side, classifier: null })} > Reset to default diff --git a/src/components/Settings/sections.ts b/src/components/Settings/sections.ts index 69ec4b032..756bb53b4 100644 --- a/src/components/Settings/sections.ts +++ b/src/components/Settings/sections.ts @@ -1,8 +1,9 @@ -import { Cog, Download, Globe, Hand, HardDriveUpload, House, Moon, Palette, RadioTower, User, type LucideIcon } from 'lucide-react' +import { Activity, Cog, Download, Globe, Hand, HardDriveUpload, House, Moon, Palette, RadioTower, User, type LucideIcon } from 'lucide-react' export const SECTIONS = [ { id: 'appearance', label: 'Appearance', icon: Palette, description: 'How this device shows the app' }, { id: 'backup', label: 'Backup', icon: HardDriveUpload, description: 'Nightly archive to your own storage' }, + { id: 'biometric-calibration', label: 'Biometric calibration', icon: Activity, description: 'Profiles, processing state and evaluation history' }, { id: 'device', label: 'Device', icon: Cog, description: 'Pod-wide hardware and maintenance' }, { id: 'gestures', label: 'Gestures', icon: Hand, description: 'What taps on the cover do' }, { id: 'homekit', label: 'HomeKit', icon: House, description: 'Apple Home bridge' }, diff --git a/src/components/Settings/tests/SleepStageRules.test.tsx b/src/components/Settings/tests/SleepStageRules.test.tsx index a371e1f16..ab8d37dd9 100644 --- a/src/components/Settings/tests/SleepStageRules.test.tsx +++ b/src/components/Settings/tests/SleepStageRules.test.tsx @@ -19,6 +19,7 @@ const trpcMock = vi.hoisted(() => { const invalidate = vi.fn() const state = { classifier: null as unknown as StageClassifier, + pending: null as { classifier: StageClassifier, generation: number, warning: string | null } | null, movementOnly: null as unknown as StageClassifier['ruleset'], isDefault: true, changes: [] as unknown[], @@ -45,6 +46,7 @@ const trpcMock = vi.hoisted(() => { ? undefined : { classifier: state.classifier, + pending: state.pending, isDefault: state.isDefault, changes: state.changes, presets: [ @@ -147,6 +149,7 @@ beforeEach(() => { trpcMock.invalidate.mockClear() Object.assign(trpcMock.state, { classifier: DEFAULT_STAGE_CLASSIFIER, + pending: null, isDefault: true, changes: [], preview: undefined, @@ -181,6 +184,16 @@ describe('SleepStageRules', () => { expect(screen.getByRole('button', { name: /Reset to default/ }).hasAttribute('disabled')).toBe(true) }) + it('edits pending rules while showing adoption status and allowing reset', () => { + trpcMock.state.pending = { classifier: { engine: 'rules', ruleset: MOVEMENT_ONLY_STAGE_RULESET }, generation: 7, warning: null } + renderRules() + expect(screen.getByRole('status').textContent).toContain('waiting for processor adoption') + expect(screen.getByRole('button', { name: 'Save' }).hasAttribute('disabled')).toBe(true) + expect(screen.queryByLabelText('Rule 2 stage')).toBeNull() + fireEvent.click(screen.getByRole('button', { name: /Reset to default/ })) + expect(trpcMock.setMutate).toHaveBeenCalledWith({ side: 'left', classifier: null }) + }) + it('opens a rule for editing from a dotted value and saves the edited threshold', () => { renderRules() open('Rule 2 condition 1 value') diff --git a/src/components/Settings/tests/sections.test.ts b/src/components/Settings/tests/sections.test.ts index d591532ae..305c90a0c 100644 --- a/src/components/Settings/tests/sections.test.ts +++ b/src/components/Settings/tests/sections.test.ts @@ -26,7 +26,7 @@ describe('resolveSection', () => { expect(resolveSection('bogus', 'also-bogus')).toBeNull() }) - it('lists all ten sections A–Z', () => { - expect(SECTIONS.map(s => s.id)).toEqual(['appearance', 'backup', 'device', 'gestures', 'homekit', 'mqtt', 'network', 'sides', 'sleep-stages', 'updates']) + it('lists all eleven sections A–Z', () => { + expect(SECTIONS.map(s => s.id)).toEqual(['appearance', 'backup', 'biometric-calibration', 'device', 'gestures', 'homekit', 'mqtt', 'network', 'sides', 'sleep-stages', 'updates']) }) }) diff --git a/src/components/biometrics/BiometricsPage.tsx b/src/components/biometrics/BiometricsPage.tsx index bffb0ddca..aa8b455df 100644 --- a/src/components/biometrics/BiometricsPage.tsx +++ b/src/components/biometrics/BiometricsPage.tsx @@ -7,6 +7,8 @@ */ 'use client' +import { Trans, useLingui } from '@lingui/react/macro' +import type { VitalProvenance } from '@/src/biometrics/profiles/vital-provenance' import { useTimeFormat } from '@/src/providers/TimeFormatProvider' import { useRef, useState, type ReactNode } from 'react' @@ -35,7 +37,7 @@ import { type Side = 'left' | 'right' -interface VitalRow { side: string, timestamp: Date, heartRate: number | null, hrv: number | null, breathingRate: number | null } +interface VitalRow { side: string, timestamp: Date, heartRate: number | null, hrv: number | null, breathingRate: number | null, processingRunId?: string | null, provenance?: VitalProvenance } const RANGES: Array<{ value: BiometricsRange, label: string }> = [ { value: 'night', label: 'Night' }, @@ -80,7 +82,8 @@ function BiometricsBody({ sectionSwitch, now }: { sectionSwitch?: ReactNode, now const referenceQ = trpc.biometrics.getReferenceVitals.useQuery({ side, startDate, endDate, limit: 20000 }, { refetchInterval: 5 * 60_000 }) const rows = (vitalsQ.data ?? []) as VitalRow[] - const points: VitalPoint[] = rows.map(r => ({ t: new Date(r.timestamp).getTime(), hr: r.heartRate, hrv: r.hrv, br: r.breathingRate })) + const points: VitalPoint[] = rows.map(r => ({ t: new Date(r.timestamp).getTime(), hr: r.heartRate, hrv: r.hrv, br: r.breathingRate, + provenanceKey: r.provenance?.processingRunId ?? r.processingRunId ?? 'legacy_unknown', hrvMetricId: r.provenance?.hrvMetricId ?? null })) const sessions = splitSessions(points) const bands = { hr: baselineBand(points, 'hr'), hrv: baselineBand(points, 'hrv'), br: baselineBand(points, 'br') } const reference = ((referenceQ.data ?? []) as Array<{ timestamp: Date | string, heartRate: number }>) @@ -226,6 +229,28 @@ function BiometricsBody({ sectionSwitch, now }: { sectionSwitch?: ReactNode, now
+
+

These are original recorded readings. Charts separate known processing runs and HRV definitions. Legacy readings with unknown provenance remain unverified.

+

The current pod algorithm reports a window IBI variability index, not clinical RMSSD. Older readings may have an unknown definition.

+ {summaryQ.data?.provenance + ? ( +

+ + Recorded configurations: + {' '} + {summaryQ.data.provenance.configurationCount} + ; readings with unknown configuration: + {' '} + {summaryQ.data.provenance.unknownConfigurationRows} + . + +

+ ) + : null} + {summaryQ.data?.provenance?.hrvAggregation === 'incompatible_definitions' + ?

HRV definitions differ or are partly unknown. A combined HRV average is unavailable.

+ : null} +
@@ -497,14 +522,17 @@ function SessionsCard({ sessions, rangeLabel, now, ongoing, stale, lastVitalAt, } function RawLog({ open, onClose, rows, loading, title }: { open: boolean, onClose: () => void, rows: VitalRow[], loading: boolean, title: string }) { + const { t } = useLingui() const columns: Array> = [ { key: 'timestamp', header: 'Time', render: r => {new Date(r.timestamp).toLocaleString()}, sortValue: r => new Date(r.timestamp).getTime() }, { key: 'heartRate', header: 'HR', align: 'right', render: r => fmtNum(r.heartRate), sortValue: r => r.heartRate ?? -1 }, { key: 'hrv', header: 'HRV', align: 'right', render: r => fmtNum(r.hrv), sortValue: r => r.hrv ?? -1 }, { key: 'breathingRate', header: 'BR', align: 'right', render: r => fmtNum(r.breathingRate, 1), sortValue: r => r.breathingRate ?? -1 }, + { key: 'processingRun', header: t`Processing run`, render: r => r.provenance?.processingRunId ?? r.processingRunId ?? t`Unknown` }, + { key: 'hrvDefinition', header: t`HRV definition`, render: r => r.provenance?.hrvMetricId ?? t`Unknown` }, ] return ( - + () + const [selected, setSelected] = useState() + const jobs = trpc.biometricTuning.listBacktests.useQuery({ side, beforeCreatedAtMs: cursor?.createdAtMs, beforeId: cursor?.id }, { refetchInterval: 5000 }) + return ( + + Backtest jobs} subtitle={Execution state is separate from the strength of biometric evidence.} /> + {jobs.error + ? {jobs.error.message} + : !jobs.data + ? + : ( + <> + {jobs.data.items.length === 0 ?

No backtest jobs for this side.

: null} +
    + {jobs.data.items.map(job => ( +
  • +

    {labels[job.state]}

    +

    {job.id}

    +

    {t`Attempt ${job.attempt}`}

    + + {selected === job.id ? : null} +
    • + ))} +
+ {cursor ? : null} + {jobs.data.nextCursor ? : null} + + )} + + ) +} +export function BacktestJobDetails({ id, side }: { id: string, side?: 'left' | 'right' }) { + const { t } = useLingui() + const labels = useJobLabels() + const [showReport, setShowReport] = useState(false) + const job = trpc.biometricTuning.getBacktest.useQuery({ id }, { refetchInterval: query => query.state.data && ['succeeded', 'failed', 'cancelled'].includes(query.state.data.state) ? false : 5000 }) + const cancel = trpc.biometricTuning.cancelBacktest.useMutation() + const utils = trpc.useUtils() + async function requestCancellation() { + const updated = await cancel.mutateAsync({ id }) + utils.biometricTuning.getBacktest.setData({ id }, updated) + await utils.biometricTuning.listBacktests.invalidate() + } + if (job.error) return {job.error.message} + if (!job.data) return + const data = job.data + const candidate = data.configurations.find(config => config.label === 'candidate') + const splits = { tuning: t`Tuning`, validation: t`Validation`, holdout: t`Holdout` } + const configurations: Record = { default: t`System defaults`, current: t`Current stage view`, candidate: t`Candidate` } + return ( +
+

{labels[data.state]}

+

+ {splits[data.split]} + {' — '} + {data.evaluationScope === 'retained_waveform_replay' ? Retained waveform replay : data.evaluationScope === 'stage_rules_on_original_readings' ? Stage rules on original readings : Original stored-reading comparison} +

+

{t`Progress: ${data.progress.completedNights} / ${data.progress.totalNights} nights`}

+ {data.progress.totalNights > 0 ? :

Night progress is not available yet.

} +

{data.progress.phase}

+ {data.error ? {data.error} : null} + {data.state === 'queued' ?

Waiting for a worker and resource admission. No execution is implied by a queued request.

: null} + {data.state === 'succeeded' ?

Execution completed. Review the report before drawing conclusions about agreement or improvement.

: null} + {['queued', 'running', 'interrupted'].includes(data.state) ? : null} + {cancel.error ? {cancel.error.message} : null} + {data.state === 'succeeded' ? : null} + {showReport && data.state === 'succeeded' ? : null} +
+ Pinned job inputs +
+
Dataset
+
{data.datasetId}
+
Dataset hash
+
{data.datasetHash}
+
Evaluator build hash
+
{data.evaluatorBuildHash}
+
+
    + {data.configurations.map(config => ( +
  • + {configurations[config.label] ?? config.label} + {': '} + {config.hash} + {' — '} + {config.revisionId ?? '—'} +
    • + ))} +
+
+
+ ) +} diff --git a/src/components/biometrics/Calibration/BacktestReport.tsx b/src/components/biometrics/Calibration/BacktestReport.tsx new file mode 100644 index 000000000..1a1304311 --- /dev/null +++ b/src/components/biometrics/Calibration/BacktestReport.tsx @@ -0,0 +1,166 @@ +'use client' + +import { useState } from 'react' +import { Trans, useLingui } from '@lingui/react/macro' +import { trpc } from '@/src/utils/trpc' +import { Button, InlineError, Skeleton } from '@/src/components/ds' +import { reportViewSchema, numericReportViewSchema, stageReportViewSchema } from './report-view' +import { NumericReport } from './NumericReport' +import { StageReport } from './StageReport' +import { NumericTrace } from './NumericTrace' +import { ObjectiveThresholds } from './ObjectiveThresholds' +import { ReplayResources } from './ReplayResources' +import { ValidatedProfileActivation } from './ValidatedProfileActivation' +import { StageTrace } from './StageTrace' + +export function BacktestReport({ id, candidate }: { id: string, candidate?: { side: 'left' | 'right', revisionId: string, hash: string } }) { + const result = trpc.biometricTuning.getBacktestSummary.useQuery({ id }, { staleTime: 60000 }) + const [reportHash, setReportHash] = useState() + if (result.data?.contentHash && result.data.contentHash !== reportHash) setReportHash(result.data.contentHash) + return ( +
+ {candidate && reportHash ? : null} + +
+ ) +} + +function BacktestReportContent({ id }: { id: string }) { + const { t, i18n } = useLingui() + const [confirmRelease, setConfirmRelease] = useState(false) + const [showTrace, setShowTrace] = useState(false) + const [showNumericTrace, setShowNumericTrace] = useState(false) + const result = trpc.biometricTuning.getBacktestSummary.useQuery({ id }, { staleTime: 60000 }) + const release = trpc.biometricTuning.releaseBacktestResult.useMutation() + const utils = trpc.useUtils() + if (result.error) return {result.error.message} + if (result.data === undefined) return + if (result.data === null) return

No committed report is available yet.

+ const descriptor = result.data + if (!descriptor.available) return ( +
+

This report was released. Its audit hash remains, but detailed results are unavailable.

+

{descriptor.contentHash}

+
+ ) + const parsed = reportViewSchema.safeParse(descriptor.report) + if (!parsed.success) return This report version or structure is not supported by this view. + const report = parsed.data + const statusLabels = { + insufficient_evidence: t`Insufficient evidence for validated improvement`, + improved: t`Improved agreement under the qualified evaluation policy`, + no_material_change: t`No material improvement under the evaluation policy`, + regressed: t`Regression under the evaluation policy`, + incompatible: t`Incompatible evaluation evidence`, + } + const waveform = report.protocol.mode === 'signal_replay' + const hasStages = report.protocol.mode === 'stage_replay' || (waveform && report.protocol.waveformStages === true) + const numeric = numericReportViewSchema.safeParse(descriptor.report) + const stages = stageReportViewSchema.safeParse(descriptor.report) + const labels: Record = { default: t`System defaults`, current: waveform ? t`Current baseline` : t`Current stage view`, candidate: t`Candidate` } + const splits = { tuning: t`Tuning`, validation: t`Validation`, holdout: t`Holdout` } + const uncertaintyLabels = { not_requested: t`Not requested`, insufficient_nights: t`Insufficient paired nights`, degenerate: t`No estimable variation`, estimated: t`Exploratory interval estimated` } + const percent = (value: number | null) => value === null ? t`Not scored` : new Intl.NumberFormat(i18n.locale || 'en', { style: 'percent', maximumFractionDigits: 2 }).format(value) + async function releaseReport() { + await release.mutateAsync({ id, expectedContentHash: descriptor.contentHash }) + setConfirmRelease(false) + await utils.biometricTuning.getBacktestSummary.invalidate({ id }) + } + return ( +
+

{statusLabels[report.status]}

+ {report.qualificationWarning ? {report.qualificationWarning} : null} + {report.status === 'improved' ?

This is the retained evaluation result, not current activation approval or clinical accuracy. Application checks current evidence and settings again.

: null} +

{splits[report.split]}

+

{t`Current holdout exposure count: ${descriptor.currentHoldoutUseCount}`}

+ {report.evidence === 'reused_holdout_is_validation' || descriptor.currentHoldoutUseCount > 1 ?

Repeated holdout exposure weakens independent evidence. Treat reused holdout results as validation, not untouched holdout evidence.

: null} + {hasStages ?

{t`Assumed stage calibration quality: ${report.calibration.quality}`}

: null} + {waveform ?

Numerical readings were recomputed from retained waveforms. Original stored readings remain unchanged.

: report.protocol.mode === 'comparison' ?

Original stored readings were compared; numerical parameter changes were not replayed.

: null} + {(report.protocol.objectivePolicies ?? (report.protocol.objectivePolicy ? [report.protocol.objectivePolicy] : [])).map(policy => ( + row.policy?.metric === policy.metric || (!row.policy && !report.protocol.objectivePolicies))} /> + ))} + {report.protocol.stageObjectivePolicy ? : null} + {report.limitations.map(limitation =>

{limitation}

)} + {report.stageNightSummary.map(summary => ( +
+

+ {labels[summary.candidate] ?? summary.candidate} + {' / '} + {labels[summary.baseline] ?? summary.baseline} +

+

{t`Scored nights: ${summary.scoredNights}; unscored: ${summary.unscoredNights}; coverage reduced: ${summary.coverageReducedNights}`}

+

{t`Higher agreement: ${summary.improvedAgreementNights}; lower: ${summary.regressedAgreementNights}; unchanged: ${summary.unchangedAgreementNights}`}

+

{t`Equal-night mean agreement difference: ${percent(summary.equalNightMeanAgreementDifference)}`}

+ + + + + + + + + + + + + + + + + + + + + +
Pooled common-support stage scores
ConfigurationAgreementMacro F1
{labels[summary.baseline] ?? summary.baseline}{percent(summary.pooledCommonSupport.baseline.agreement)}{percent(summary.pooledCommonSupport.baseline.macroF1)}
{labels[summary.candidate] ?? summary.candidate}{percent(summary.pooledCommonSupport.candidate.agreement)}{percent(summary.pooledCommonSupport.candidate.macroF1)}
+

Equal-night differences and pooled duration scores use different weighting. Coverage loss must be considered alongside score changes.

+

{uncertaintyLabels[summary.uncertainty.status]}

+ {summary.uncertainty.interval ?

{t`Exploratory 95% interval: ${percent(summary.uncertainty.interval.lower)} to ${percent(summary.uncertainty.interval.upper)}`}

: null} + {summary.uncertainty.limitations.map(limitation =>

{limitation}

)} +
+ ))} + {hasStages ? (stages.success ? : Stage details are unavailable or incompatible.) : null} + {waveform ? : null} + {numeric.success ? : Numerical details are unavailable or incompatible.} + {waveform && numeric.success + ? ( +
+ + {showNumericTrace ? : null} +
+ ) + : null} + {hasStages && numeric.success + ? ( +
+ + {showTrace ? : null} +
+ ) + : null} +
+ Report identity +
+
Report hash
+
{descriptor.contentHash}
+
Dataset hash
+
{report.datasetHash}
+
Protocol hash
+
{report.protocolHash}
+
Evaluator build hash
+
{report.evaluatorBuildHash}
+
+
+ {confirmRelease + ? ( +
+

Release detailed results permanently? The report hash, job history, and holdout exposure audit remain.

+ + +
+ ) + : } + {release.error ? {release.error.message} : null} +
+ ) +} diff --git a/src/components/biometrics/Calibration/BacktestSubmission.tsx b/src/components/biometrics/Calibration/BacktestSubmission.tsx new file mode 100644 index 000000000..f8ade9907 --- /dev/null +++ b/src/components/biometrics/Calibration/BacktestSubmission.tsx @@ -0,0 +1,102 @@ +'use client' + +import { useRef, useState } from 'react' +import { Trans, useLingui } from '@lingui/react/macro' +import { BacktestJobDetails } from './BacktestJobs' +import { trpc } from '@/src/utils/trpc' +import { Button, InlineError, Skeleton } from '@/src/components/ds' + +export function BacktestSubmission({ side, experimentId, datasetId }: { side: 'left' | 'right', experimentId: string, datasetId: string }) { + const { t } = useLingui() + const [cursor, setCursor] = useState<{ createdAtMs: number, id: string }>() + const [candidate, setCandidate] = useState<{ id: string, hash: string, generation: number }>() + const [split, setSplit] = useState<'tuning' | 'validation' | 'holdout'>('tuning') + const [holdoutAck, setHoldoutAck] = useState() + const [attempted, setAttempted] = useState() + const [saved, setSaved] = useState() + const request = useRef<{ fingerprint: string, key: string } | undefined>(undefined) + const profiles = trpc.biometricTuning.listProfileRevisions.useQuery({ side, beforeCreatedAtMs: cursor?.createdAtMs, beforeId: cursor?.id }) + const overview = trpc.biometricTuning.getProfileOverview.useQuery({ side }, { refetchInterval: 5000 }) + const dataset = trpc.biometricTuning.getDataset.useQuery({ id: datasetId }) + const enqueue = trpc.biometricTuning.enqueueBacktest.useMutation() + const utils = trpc.useUtils() + const error = profiles.error ?? overview.error ?? dataset.error + if (error) return {error.message} + if (!profiles.data || !overview.data || !dataset.data) return + if (!dataset.data.available) return

The retained inputs are unavailable for new evaluations.

+ const generation = overview.data.desired.binding.generation + const input = candidate ? { experimentId, candidateRevisionId: candidate.id, expectedBindingGeneration: candidate.generation, split } : undefined + const fingerprint = JSON.stringify(input) + const availableSplits = new Set(dataset.data.manifest.splitProtocol.assignments.map(cohort => cohort.split)) + const valid = candidate && (candidate.generation === generation || attempted === fingerprint) && availableSplits.has(split) && (split !== 'holdout' || holdoutAck === fingerprint) + async function submit() { + if (!input || !valid) return + if (request.current?.fingerprint !== fingerprint) request.current = { fingerprint, key: Array.from(crypto.getRandomValues(new Uint8Array(24)), byte => byte.toString(16).padStart(2, '0')).join('') } + setAttempted(fingerprint) + await enqueue.mutateAsync({ ...input, idempotencyKey: request.current.key }) + setSaved(fingerprint) + void utils.biometricTuning.listBacktests.invalidate({ side }) + void utils.biometricTuning.listExperiments.invalidate({ side }) + } + return ( +
+ {dataset.data.manifest.version === 'waveform-dataset-manifest-v1' ?

The server pins default, current baseline, and candidate configurations, plus the other side’s configuration. Replay uses retained signals and never replaces original observations.

:

The server captures default, current stage-view, and candidate configurations at submission. Stored readings cannot validate numerical parameter changes.

} +

{dataset.data.contentHash}

+
+ {profiles.data.items.length === 0 ?

No saved candidate revisions.

: null} +
    + {profiles.data.items.map(profile => ( +
  • + +
    • + ))} +
+ {cursor ? : null} + {profiles.data.nextCursor ? : null} + {candidate + ? ( +

+ {candidate.id} + {' — '} + {candidate.hash} +

+ ) + : null} + {candidate && candidate.generation !== generation ? : null} + + {split === 'holdout' + ? ( + + ) + : null} + +
+ {enqueue.error ? {enqueue.error.message} : null} + {enqueue.data + ? ( +
+

{t`Backtest request saved: ${enqueue.data.id}`}

+

A saved request does not mean execution has started or succeeded. Inspect the job for its current state.

+

{enqueue.data.evaluatorBuildHash}

+ +
+ ) + : null} +
+ ) +} diff --git a/src/components/biometrics/Calibration/BiometricCalibration.tsx b/src/components/biometrics/Calibration/BiometricCalibration.tsx new file mode 100644 index 000000000..c32e7a9ad --- /dev/null +++ b/src/components/biometrics/Calibration/BiometricCalibration.tsx @@ -0,0 +1,63 @@ +'use client' + +import { useState } from 'react' +import { Trans, useLingui } from '@lingui/react/macro' +import { Button, Card } from '@/src/components/ds' +import { SideTabs } from '@/src/components/Settings/SettingsLayout' +import { SleepStageRules } from '@/src/components/Settings/SleepStageRules' +import { CalibrationOverview } from './CalibrationOverview' +import { CalibrationReferences } from './CalibrationReferences' +import { ReferenceLibrary } from './ReferenceLibrary' +import { ProfileLibrary } from './ProfileLibrary' +import { ProfileRecovery } from './ProfileRecovery' +import { CalibrationBacktests } from './CalibrationBacktests' +import { CalibrationHistory } from './CalibrationHistory' +import { ShadowRuns } from './ShadowRuns' + +type Side = 'left' | 'right' +export function BiometricCalibration({ side, onSideChange }: { side: Side, onSideChange: (side: Side) => void }) { + const { t } = useLingui() + const [view, setView] = useState<'overview' | 'references' | 'backtests' | 'profiles' | 'history' | 'observations'>('overview') + return ( +
+ + + + + {view === 'overview' + ? ( +
+ + +
+ ) + : view === 'references' + ? ( +
+ + +
+ ) + : view === 'backtests' + ? + : view === 'observations' + ? + : view === 'history' + ? + : ( + <> + +

Direct sleep-stage changes are unvalidated manual settings. Saving requests a profile change; processor adoption is reported separately in Overview.

+ + + )} +
+ ) +} diff --git a/src/components/biometrics/Calibration/CalibrationBacktests.tsx b/src/components/biometrics/Calibration/CalibrationBacktests.tsx new file mode 100644 index 000000000..87ad6b528 --- /dev/null +++ b/src/components/biometrics/Calibration/CalibrationBacktests.tsx @@ -0,0 +1,17 @@ +'use client' + +import { BacktestJobs } from './BacktestJobs' +import { ExperimentLibrary } from './ExperimentLibrary' +import { DatasetLibrary } from './DatasetLibrary' +import { DatasetBuilder } from './DatasetBuilder' + +export function CalibrationBacktests({ side }: { side: 'left' | 'right' }) { + return ( +
+ + + + +
+ ) +} diff --git a/src/components/biometrics/Calibration/CalibrationHistory.tsx b/src/components/biometrics/Calibration/CalibrationHistory.tsx new file mode 100644 index 000000000..0d3212224 --- /dev/null +++ b/src/components/biometrics/Calibration/CalibrationHistory.tsx @@ -0,0 +1,67 @@ +'use client' + +import { useState } from 'react' +import { Trans, useLingui } from '@lingui/react/macro' +import { trpc } from '@/src/utils/trpc' +import { Button, Card, CardHeader, InlineError, Skeleton } from '@/src/components/ds' +import { useTimeFormat } from '@/src/providers/TimeFormatProvider' + +export function CalibrationHistory({ side }: { side: 'left' | 'right' }) { + const { t, i18n } = useLingui() + const timeFormat = useTimeFormat() + const [beforeGeneration, setBeforeGeneration] = useState() + const history = trpc.biometricTuning.listActivationHistory.useQuery({ side, beforeGeneration }, { refetchInterval: beforeGeneration === undefined ? 5000 : false }) + const settings = trpc.settings.getAll.useQuery({}, { staleTime: 60000 }) + const timezone = settings.data?.device.timezone ?? 'UTC' + const dates = new Intl.DateTimeFormat(i18n.locale || 'en', { dateStyle: 'medium', timeStyle: 'short', timeZone: timezone, hour12: timeFormat !== '24h' }) + const actions: Record = { activate: t`Apply profile`, reset: t`Use system defaults`, rollback: t`Restore previous profile`, migration: t`Migrate settings`, fallback: t`Compatibility fallback` } + const states: Record = { saved: t`Saved`, pending_delivery: t`Waiting for delivery`, pending_boundary: t`Waiting for a processing boundary`, applied: t`Applied`, failed: t`Failed` } + return ( + + Activation history} subtitle={Past acknowledgements do not establish which profile is running now.} /> +

+ {t`Times shown in ${timezone}`} +

+ {history.error + ? {history.error.message} + : !history.data + ? + : history.data.items.length === 0 + ?

No activation history for this side.

+ : ( +
    + {history.data.items.map(event => ( +
  1. +
    + {actions[event.action]} + {states[event.state]} +
    + +

    + {t`Generation ${event.generation}`} +

    + {event.reason ?

    {event.reason}

    : null} +
    + Exact profile transition +

    + {event.previousHash} + {' '} + → + {' '} + {event.nextHash} +

    +

    + {t`Requested by ${event.requester}`} +

    +
    +
    2. + ))} +
+ )} +
+ {beforeGeneration !== undefined ? : null} + {history.data?.nextGeneration != null ? : null} +
+ + ) +} diff --git a/src/components/biometrics/Calibration/CalibrationOverview.tsx b/src/components/biometrics/Calibration/CalibrationOverview.tsx new file mode 100644 index 000000000..26d68f645 --- /dev/null +++ b/src/components/biometrics/Calibration/CalibrationOverview.tsx @@ -0,0 +1,66 @@ +'use client' + +import { Trans, useLingui } from '@lingui/react/macro' +import { trpc } from '@/src/utils/trpc' +import { Card, CardHeader, InlineError, SettingRow, Skeleton } from '@/src/components/ds' +import { SectionColumns } from '@/src/components/Settings/SettingsLayout' + +export function CalibrationOverview({ side }: { side: 'left' | 'right' }) { + const { t } = useLingui() + const overview = trpc.biometricTuning.getProfileOverview.useQuery({ side }, { refetchInterval: 5000 }) + const activation = trpc.biometricTuning.getActivationStatus.useQuery({ side }, { refetchInterval: 5000 }) + if (overview.error) return {overview.error.message} + if (!overview.data) return + const data = overview.data + const states: Record = { unreported: t`Not reported`, current: t`Current`, stale: t`Stale`, failed: t`Failed`, + mixed_generations: t`Different versions reported`, clock_skew: t`Clock mismatch`, pending_delivery: t`Waiting for delivery`, + pending_boundary: t`Waiting for a processing boundary`, applied: t`Applied` } + const components: Record = { heart_rate: t`Heart rate`, breathing_rate: t`Breathing rate`, hrv_index: t`HRV index` } + return ( + + + Profile and processor} subtitle={Saved settings and processor reports are separate.} /> + Saved profile}>{data.desired.isDefault ? System defaults : Personalized profile} + Requested change}>{activation.data ? states[activation.data.observedState] : t`Not reported`} + Processor health}>{states[data.runtimeHealth.state]} + {activation.error ? {activation.error.message} : null} + {data.desired.warning ?

{data.desired.warning}

: null} + {data.runtimeHealth.components.map(component => ( + + {states[component.state]} + + ))} +
+ Configuration identity +

{data.desired.hash}

+

+ {t`Generation ${data.desired.binding.generation}`} +

+
+ + + What these metrics mean} /> +

The HRV index measures window IBI variability. It is not clinical beat-to-beat RMSSD, and incompatible reference metrics cannot validate it.

+

Historical sleep stages are recomputed with the current classifier. Original stored vitals remain separate from replay results.

+ {data.stageView.provenance.warning ?

{data.stageView.provenance.warning}

: null} + + + )} + right={( + + Measured sensor calibration} subtitle={Sensor measurements are separate from algorithm personalization.} /> + {data.sensorCalibration.length === 0 + ?

No sensor calibration has been reported for this side.

+ : data.sensorCalibration.map(sensor => ( + + {sensor.freshness === 'current' ? t`Current` : sensor.freshness === 'expired' ? t`Expired` : t`Freshness unknown`} + + ))} + {data.limitations.map(limitation =>

{limitation}

)} + + )} + /> + ) +} diff --git a/src/components/biometrics/Calibration/CalibrationReferences.tsx b/src/components/biometrics/Calibration/CalibrationReferences.tsx new file mode 100644 index 000000000..4b01d3a15 --- /dev/null +++ b/src/components/biometrics/Calibration/CalibrationReferences.tsx @@ -0,0 +1,109 @@ +'use client' + +import { useState } from 'react' +import { Trans, useLingui } from '@lingui/react/macro' +import { trpc } from '@/src/utils/trpc' +import { Button, Card, CardHeader, InlineError } from '@/src/components/ds' +import { ReferenceTimeSpan } from './ReferenceTimeSpan' +import { parseReferenceText } from '@/src/biometrics/references/parse' +import type { PreviewSummary, ReferenceImportInput } from '@/src/biometrics/references/contracts' + +export function CalibrationReferences({ side, correction }: { side: 'left' | 'right', correction?: { id: string, subjectId: string, source: string } }) { + const { t } = useLingui() + const [subjectId, setSubjectId] = useState(correction?.subjectId ?? '') + const [source, setSource] = useState(correction?.source ?? '') + const [format, setFormat] = useState<'csv' | 'json'>('csv') + const [text, setText] = useState('') + const [preview, setPreview] = useState<{ input: ReferenceImportInput, summary: PreviewSummary }>() + const [acceptRejected, setAcceptRejected] = useState(false) + const [error, setError] = useState() + const [saved, setSaved] = useState() + const utils = trpc.useUtils() + const previewImport = trpc.biometricTuning.previewImport.useMutation() + const commitImport = trpc.biometricTuning.commitImport.useMutation() + const busy = previewImport.isPending || commitImport.isPending + function invalidate() { + setPreview(undefined) + setAcceptRejected(false) + setError(undefined) + setSaved(undefined) + } + async function inspect() { + invalidate() + try { + const input = { side, subjectId, source, rows: parseReferenceText(format, text) } + const summary = await previewImport.mutateAsync(input) + setPreview({ input, summary }) + } + catch (cause) { setError(cause instanceof Error ? cause.message : t`Reference preview failed.`) } + } + async function save() { + if (!preview) return + setError(undefined) + try { + const result = await commitImport.mutateAsync({ import: preview.input, previewHash: preview.summary.contentHash, acceptRejectedRows: acceptRejected, ...(correction ? { supersedesImportId: correction.id } : {}) }) + setSaved(result.id) + void utils.biometricTuning.listImports.invalidate() + setPreview(undefined) + } + catch (cause) { setError(cause instanceof Error ? cause.message : t`Reference import failed.`) } + } + return ( + + Import reference measurements} subtitle={Preview normalized records before saving them on the pod.} /> +

Use offset-bearing ISO timestamps or Unix milliseconds. Each record needs recordKey, metric, value, unit, and timestamp (or start and end). JSON uses an array; CSV uses these field names as headers. Imports are limited to 2 MiB and 2000 records.

+ {correction ?

{t`Correcting import ${correction.id}. Supply the complete replacement export; the original revision and frozen datasets remain unchanged.`}

: null} + 
{'recordKey,metric,value,unit,timestamp\n1,heart_rate,60,bpm,2026-10-10T23:00:00Z'}
+
+ + + +