diff --git a/package.json b/package.json index 60ceecb6..7483513c 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "private": true, "scripts": { "prebuild": "node scripts/generate-git-info.mjs && lingui compile", - "build": "next build && node scripts/strip-turbopack-external-hashes.mjs && cp -r .next/static .next/standalone/.next/static && { [ ! -d public ] || cp -r public .next/standalone/public || true; }", + "build": "next build && node scripts/strip-turbopack-external-hashes.mjs && cp -r .next/static .next/standalone/.next/static && { [ ! -d public ] || cp -r public .next/standalone/public || true; } && node scripts/build/biometrics-worker.mjs", "dev": "next dev", "db:generate": "drizzle-kit generate", "db:migrate": "drizzle-kit migrate", @@ -27,7 +27,9 @@ "test:mutation": "stryker run", "test:mutation:incremental": "stryker run --incremental", "tsc": "tsc --noEmit", - "prepare": "git rev-parse --git-dir >/dev/null 2>&1 && git config core.hooksPath .githooks || true" + "prepare": "git rev-parse --git-dir >/dev/null 2>&1 && git config core.hooksPath .githooks || true", + "build:biometrics-worker": "node scripts/build/biometrics-worker.mjs", + "biometrics:experiment": "tsx scripts/algorithm-experiments/cli.ts" }, "lint-staged": { "*.{ts,tsx}": "eslint --fix", diff --git a/scripts/algorithm-experiments/README.md b/scripts/algorithm-experiments/README.md new file mode 100644 index 00000000..e9d95a1d --- /dev/null +++ b/scripts/algorithm-experiments/README.md @@ -0,0 +1,578 @@ +# Off-pod algorithm experiments + +`executePackageExperiment` runs an inert, hash-verified package in a reviewed +local Docker image selected by an operator policy. It is development tooling, +not a pod API, installation service, or qualification decision. Known pod DAC +socket paths are rejected. Never bundle this runner into the pod web server. + +The image must be operator-reviewed and selected by immutable local `sha256:` +ID. The package's dependency-lock artifact uses the +`biometric-python-environment-v1` JSON schema: exact image ID, Python version, +implementation, OS, architecture and the complete sorted installed distribution +inventory. A restricted probe container with no package files mounted collects +that inventory before candidate execution. Any mismatch rejects the experiment. +The result carries the exact lock-byte hash and observed environment hash. +Do not pass a candidate-selected image or policy. + +Use `inspectPythonEnvironment(imageId)` to capture a lock from a reviewed image, +then include those JSON bytes as a hashed package artifact. This is an +**environment lock**, not a pip or uv dependency-resolution file; existing uv +locks may also be retained as package artifacts for review. Preparing an image +from the source dependency lock remains an operator's reviewed build step. The +runner never installs dependencies or executes build hooks. The immutable image +ID binds the configuration and layers; inventory verification additionally +makes interpreter/platform/package-version mismatches explicit. See +[Docker image identities](https://docs.docker.com/reference/cli/docker/image/pull/) +and [Python distribution metadata](https://docs.python.org/3.12/library/importlib.metadata.html). +A matching lock does not establish that dependencies or the image are safe. + +The process runs as UID 65534 with no network, capabilities, privilege elevation, +or writable root filesystem. Only a fresh verified copy of the package is +mounted, read-only. No pod files, host credentials, Docker socket, or database +are mounted. Docker's default seccomp profile remains enabled. Memory, swap, +PID count, stdout/stderr bytes, input bytes, and a deadline are bounded. The +runner checks that the daemon reports the required limit and seccomp support. +These controls use Docker's [documented container options](https://docs.docker.com/engine/containers/run/). + +CPU is restricted to one core and the deadline is the smaller of requested CPU +and wall time, conservatively charging idle time against CPU budget. Scheduler +and kill latency can overshoot the deadline; execution metadata is not a CPU or +peak-memory qualification receipt. Experiments still require measured resource +qualification before any promotion. Use a dedicated external Docker host or VM +with no pod credentials; containers share its kernel and are not a substitute +for maintaining that host's isolation and security updates. + +The package entrypoint is a Python executable implementing JSON Lines: + +1. Read one `initialize` object with `request` and `runHash`; reset all state and + initialize deterministically from the explicit seed/configuration. +2. Read each `frame` object with `frame` and `frameHash`. +3. Write exactly one `biometric-metric-frame-v1` response line per input frame. +4. Exit successfully at stdin EOF. Logs may go to stderr within the same total + output budget. Additional stdout is an error. + +Only one frame is in flight. Host validation runs before sending a frame and +before retaining its response. The caller supplies a trusted synchronous frame iterator. Before starting the +candidate, the runner writes canonical frames to a private bounded file and +checks their domain-separated SHA-256 hash against the request's `inputHash`. +The file is not mounted into the container; only one frame at a time is sent +from that frozen copy. The hash encoding is UTF-8 +`biometric-package-input-v1\n` followed by each canonical frame plus LF. +Input hash validation does not establish a clinical reference or holdout split; +those associations still need the qualification dataset pipeline. +The host staging parent stays mode 0700, with frozen input mode 0600. Only its +`package/` subdirectory (0755, files 0444) is bind-mounted read-only. The container +UID can traverse that mount without granting other host users access through the +private parent. An opt-in real-Docker test verifies these modes after container +creation, successful execution as UID 65534, and removal after completion. +Package files and outputs are never imported as host code. Cleanup removes the +owned container and staging directory on normal completion and caught errors; +hard host termination is handled by the explicit container recovery sweep +described below, followed by ownership-checked private staging recovery. No arbitrary dependency installation occurs during execution. + +Run the opt-in Docker regression tests with a reviewed Python image: + +```sh +BIOMETRIC_EXPERIMENT_TEST_IMAGE=sha256: pnpm exec vitest run scripts/algorithm-experiments/container.test.ts +``` + +The tests cover a valid exchange, filesystem/network/environment/PID probes, +output floods, memory exhaustion, deadlines, and invalid results. They do not +qualify any biometric algorithm or prove resistance to kernel vulnerabilities. + +## Determinism evidence + +`checkPackageDeterminism` runs the same manifest, byte snapshots, request and +operator policy two to five times, with a fresh container each time. Its trusted +frame factory must reproduce the retained input: every run independently +verifies that hash. It hashes every response, including serialized state, in +order using the `biometric-package-output-v1` domain. Execution timings and +transport measurements are excluded from the output identity. + +The result records run/input/output identities and execution metadata. A +changing state fails even when numeric readings are identical. A deterministic +result is only one qualification dimension: it does not approve dependency +compatibility, resource fitness, artifact/noise behavior, held-out agreement, +shadow use, installation, or live activation. Those checks and reviewed release +promotion remain required. + +## File-based command line + +Run these commands on the external experiment host, from the repository: + +```sh +pnpm biometrics:experiment lock sha256: environment.lock.json +pnpm biometrics:experiment inspect proposal.json intake.json +pnpm biometrics:experiment input-hash frames.jsonl 33554432 input-identity.json +pnpm biometrics:experiment run proposal.json request.json frames.jsonl operator-policy.json experiment.json +pnpm biometrics:experiment determinism proposal.json request.json frames.jsonl operator-policy.json 2 determinism.json +``` + +The proposal is a single inert JSON envelope with `schema` equal to +`biometric-package-proposal-v1`, a complete `manifest`, and `files` containing +`{ "path": "engine.py", "base64": "..." }` entries for every declared artifact. +Include the captured environment lock as the manifest's `dependencyLock` file; +its exact bytes, like all package files, must match the artifact size and hash. +`inspect` verifies the proposal without launching Docker or executing its code. + +`frames.jsonl` contains one `biometric-signal-frame-v1` object per line. +`request.json` follows `packageRunRequestSchema` in `protocol.ts`: set its +`packageHash` from the intake report and `inputHash` from the input-identity +report, plus explicit hardware, sample rate, side, configuration, and seed. + +`operator-policy.json` is separate from candidate artifacts and contains the +reviewed image ID and positive integer ceilings: `maximumWallMs` (at most one +hour), `maximumMemoryBytes`, `maximumProcesses`, `maximumOutputBytes`, and +`maximumInputBytes`. Select these for the external host; they do not establish +that an algorithm fits the pod. Candidate resource requests must fit the policy. + +All file inputs must be bounded regular files, not symlinks, devices or FIFOs. +JSONL has a separate per-line cap. Reports are mode 0600, written completely +before atomic publication, and never overwrite an existing path. Errors return +nonzero and do not publish a success report. The report includes raw candidate +outputs and should be treated as private biometric data. No command sends it to +a reviewer or installs the package. A successful `run` is explicitly marked +`experiment_only`; use the remaining qualification checks and normal reviewed +release process before promotion. + +## Host resource observations + +Experiment reports now include `execution.measurements`, collected from the +Docker Engine stats API on the host, never from candidate output. The collector +uses the actual created container ID, validates each response's identity, and +records observed cumulative CPU milliseconds, memory bytes including cache, +and process counts. It attempts sampling every 250 ms without overlapping +requests, and requests a final sample while the candidate is still waiting for +stdin EOF. Reads have time and byte limits. + +The receipt records attempts, usable samples, and maximum gaps including +startup and shutdown. Missing, stale, or regressing samples leave it incomplete. Memory is an observed peak, +not an exact lifetime high-water mark; CPU is the largest observed cumulative +counter, not a guaranteed final lifetime total. A process may consume resources +between samples. The collector's overhead counts toward the experiment's wall +deadline. Qualification must inspect coverage and gaps and retain these limits. +No fit-for-pod decision is inferred from measurements on an external host. + +Currently measurement supports local Unix Docker endpoints and API 1.41 or +newer. Unsupported transports or failed observations leave nullable counters +and incomplete evidence; they do not fabricate zero usage or waive the existing +container resource limits. See the [Docker stats API](https://docs.docker.com/reference/api/engine/version/v1.51/). + +## Artifact/noise fixture suites + +```sh +pnpm biometrics:experiment fixtures proposal.json suite.json operator-policy.json fixture-report.json +``` + +A reviewed suite uses `biometric-package-fixtures-v1` and a `fixtures` array. +Each fixture has a unique `id`, `category`, full run `request`, inline signal +`frames`, and `expectations`. Its request must bind the proposal and actual +frame hashes. The suite file is bounded to 32 MiB; inline fixtures are intended +for focused signal tests, not whole-night held-out comparisons. + +Expectation windows use inclusive `startMs`, exclusive `endMs`, and a declared +`metric`. A `numeric` expectation specifies `minimum`, `maximum`, +`minimumReadings`, and `minimumQuality`. Missing/null readings cannot satisfy +numeric coverage. An `abstain` expectation permits absent or null output and +rejects numeric values. Duplicate readings, unexpected metrics/times, +out-of-range values, low quality, insufficient coverage and overlapping +expectations fail. Windows must fall inside the supplied input interval. +Clean-signal fixtures must require numeric output; empty-bed fixtures must +require abstention. + +Each fixture executes in a fresh isolated container. Reports retain the suite, +fixture, input and output identities, the complete frozen suite with frames and +expectations, execution output, independent resource observations and failures. +A crashed or invalid-output case remains a failed case in the report. + +Required category coverage is `clean_signal`, `pump`, `empty_bed`, `motion`, +`cross_side`, `harmonics`, `gaps`, `warmup`, and `boundary_reset`. Omitted categories +make an otherwise passing report `incomplete`. All covered checks passing yields +`fixture_checks_passed`, never automatic qualification or installation. Category +labels do not prove that a signal realistically represents that condition: +review the exact hashed suite and its expected behavior. The test suite includes +synthetic clean/empty-bed examples to verify this workflow; representative +shipped-algorithm fixtures, held-out agreement, compatibility and pod resource +qualification remain separate requirements. + +Each fixture suite also requires an explicit reviewed `resourcePolicy` with +schema `biometric-package-resource-policy-v1`, `maximumWallMs`, `maximumCpuMs`, +`maximumMemoryBytes`, `maximumProcesses`, `minimumSamples`, +`maximumObservationGapMs`, and `evidenceMode` equal to +`sampled_counters_and_enforced_limits`. There are no production default +thresholds. The policy is part of the frozen suite hash and every per-case +resource assessment records its own policy hash. + +An assessment can be `within_observed_limits`, `insufficient_evidence`, or +`failed`. It requires consistent host receipts, adequate sample count and gaps, +and enforced limits no wider than the reviewed budget. Known wall, observed +CPU/memory/process, or enforced-limit overruns fail even when sampling is also +incomplete. Missing measurements never mean zero cost. A passing signal check +with incomplete resource evidence makes the overall fixture report incomplete; +a resource failure makes it fail. These sampled observations plus hard limits +still do not establish exact lifetime costs or target-pod fitness. + +## Reference comparison + +```sh +pnpm biometrics:experiment compare comparison-input.json comparison-report.json +``` + +This command reuses the application's reference normalization, point/interval +alignment, and common-support numeric comparison. The input uses schema +`biometric-package-comparison-v1`, a `metricId`, whole-night `startMs`/`endMs` +(up to 24 hours), explicit `minimumQuality`, raw `referenceImport`, frozen +`alignment`, and exactly three `outputs` labelled `default`, `current`, and +`candidate`. Each output supplies its `manifest`, run `request`, and complete +executor `result`. Inputs are bounded to 32 MiB. Run IDs include configuration; +reported baseline/candidate hashes in the shared comparison are those run IDs. + +Each selected manifest metric must declare a `reference` contract with a +supported metric definition and nullable `windowMs`. HRV definitions require +an explicit window, and units must be canonical (`bpm`, `breaths/min`, or `ms`). +A free-text description or a coincidentally matching output name is insufficient. +All packages must agree on that contract, input identity, hardware, sample rate, +side and input-frame sequence. Different HRV definitions or windows cannot be +compared as the same metric. + +Reference rows are normalized using the shared adapter. Out-of-night rows and +compatibility exclusions are reported explicitly. Comparisons use identical, +unambiguous observation timestamps shared by all three runs before reference +pairing; rejected/low-quality readings remain missing support, never filled +values. MAE, bias and coverage differences are descriptive reference agreement. +The command does not trust file-provided results as execution attestations, +record holdout access, estimate whole-night uncertainty, designate an objective, +or authorize promotion. Those require the authoritative experiment/qualification +workflow, not a report assembled from arbitrary files. + +## Durable study ledger (orchestration component) + +`PackageExperimentLedger` maintains frozen study identities, exact trial +requests, idempotency keys, expiring worker leases and report hashes in a +**dedicated off-pod SQLite database**. Its generated schema and migrations are +under `src/db/package-experiment-*`; generate changes with +`pnpm exec drizzle-kit generate --config=drizzle.package-experiments.config.ts`. +Do not use either pod database for this ledger. Initialization refuses nonempty +foreign databases, uses an application identifier, and rejects future format +versions rather than migrating them backward. + +Study nights bind subject/side, start/end, input/reference hashes, split and +protocol hash. Splits must be chronological and nonoverlapping. Claims record +exposure transactionally before selected input is opened. Every attempt counts, +including failures, expired leases and retries; changed study IDs or corrected +input/reference hashes cannot erase overlap with an earlier attempt on the +same subject and side. A stale attempt cannot renew a replacement's lease or +publish over its report. Completed trials return their retained report identity +rather than silently running again. + +Claim-time `unexposedHoldoutAtClaim` is historical information. Re-read +`currentEvidence` for current eligibility: another overlapping trial can make a +previously unique holdout attempt non-unique. Keeping this ledger across studies +is required; a new/deleted ledger, relabelled subjects, or data inspected outside +its workflow cannot prove independent holdout use. Final qualification must +bind subjects and windows to retained datasets and recheck exposure while +publishing its decision. + +The exploratory `run`, `compare`, `determinism`, and `fixtures` commands do not +claim studies through this ledger and cannot issue independent-holdout promotion +evidence. `runManagedTrial` and the `managed` command integrate it as described +below. `assessManagedQualification` combines verified external evidence as +described below; installation remains a separate reviewed code/release step. + +## Managed whole-night runner + +`runManagedTrial` in `managed.ts` accepts a reserved ledger trial, its frozen +`biometric-managed-protocol-v1`, exactly three package variants, a private report +directory, and a trusted retained-night loader. The protocol freezes hardware, +sample rate, metric, quality threshold, reference alignment, whole-night +bootstrap settings and resource policy. Each trial variant binds the package +hash, runtime image, and `configHash({ config, seed })`. + +It verifies package/configuration/protocol identities before claiming a lease. +The loader runs only after all selected night exposures are committed. Loaded +references must match the study's subject, side and shared import content hash. +Frames must lie within their frozen night and match its actual input hash. All +three variants run in independent containers; results use the shared typed +comparison and paired whole-night uncertainty functions. Resource assessments +remain attached to every variant. + +Private per-night and trial reports are published under their content hashes, +then the ledger's live attempt fence selects the winning report. Heartbeats and +checks between expensive steps renew leases; lost leases abort active execution +or reject publication. Failures retain partial-night evidence and exposure. +An old worker may leave an unreferenced artifact but cannot overwrite the winning +ledger pointer. Restarted completed trials verify the complete retained report graph before +reusing it and never silently reread holdout input. Missing or corrupted +per-night artifacts fail rather than trigger a new evaluation. + +A successful run means the evaluation completed, not that its candidate passed +all promotion criteria. Check current ledger exposure, coverage, uncertainty, +resource assessments, deterministic/noise/compatibility evidence and reviewed +release admission. The runner does not install packages or claim pod fitness. +Hard process termination can still leave orphan containers/artifacts; use the +ownership-checked recovery procedures below. Combined external qualification +rechecks the retained evidence and current exposure without authorizing installation. + +## Managed descriptor command + +```sh +pnpm biometrics:experiment managed ./experiment-workspace descriptor.json receipt.json +``` + +The command creates a private workspace (0700) containing a dedicated +`experiments.sqlite` (0600) and private `reports/`. Reuse the same workspace +across studies; deleting or replacing it destroys the exposure history needed +for future decisions. Existing public directories, symlinked workspace/report +paths, symlinked database/sidecar files, and foreign databases are rejected. +No pod database is opened or migrated. + +Managed execution acquires an OS-released SQLite write lock in `.writer.sqlite` +for the entire workspace operation. Competing controllers fail before changing +the ledger or starting containers. Never unlink this lock file to resolve a busy +workspace: doing so would create a second lock inode. A killed controller releases +the database lock automatically; container and private-staging recovery are still +separate responsibilities. Low-level trial APIs are trusted building blocks; +`managed` is the bounded persistent-workspace entry point. + +The default workstation storage policy allows 2,048 report files, 1 GiB of total +report bytes, 32 MiB per report and a 64 MiB ledger, while requiring 1 GiB of free +space plus room for ledger growth/rollback and the next report. These are off-pod +storage limits, not production pod resource budgets. Supply a strict JSON policy +as the optional fourth managed argument to override them: + +```sh +pnpm biometrics:experiment managed WORKSPACE DESCRIPTOR RECEIPT STORAGE_POLICY +``` + +Policy fields are `maximumReportFiles`, `maximumReportBytes`, +`maximumSingleReportBytes`, `maximumDatabaseBytes`, and `minimumFreeBytes`. +Checks run before work, before each retained-night load, and before each report +publication. They bound managed report files, including abandoned temporary files, +and SQLite growth via `max_page_count` with DELETE-mode rollback journals. The +report directory must contain only private, owned, single-link regular files. +Symlinks, subdirectories and unsafe database/lock files are rejected. No existing +reports or exposure history are deleted to satisfy a quota. When publication is +refused, the failure-finalization behavior below still applies. + +Free-space checks are admission snapshots; unrelated applications can consume +space afterward. The ledger cap plus its rollback journal is accounted separately +from report bytes. Operator source files, the output receipt path, Docker storage, +and container input staging outside this workspace are not covered by these +workspace limits. Their existing per-input/output limits still apply; aggregate +staging quotas and crash recovery require separate controls. + +A descriptor uses `biometric-managed-descriptor-v1` with: + +- `study`: the frozen `biometric-package-study-v1`, including all chronological + night assignments and their input/reference hashes. +- `protocol`: the exact managed protocol whose canonical hash is the study's + `protocolHash`. +- `split` and an `idempotencyKey` for this exact trial. +- `variants`: exactly `default`, `current`, and `candidate`, each specifying + `proposal` (file path), `config`, `seed`, and the operator execution `policy`. +- `nights`: exactly the selected split's night IDs, each with `frames` (JSONL + path) and `referenceImport` (raw reference-import JSON path). + +Failure finalization also runs when the failure report cannot be published (for +example, a full report volume). If the ledger remains writable and the lease is +current, the attempt becomes `failed` with a null report hash. No nonexistent +artifact is claimed. Publication or ledger-finalization failures are returned +alongside the original error in an `AggregateError`; its cause remains the original +failure. Failed attempts retain their exposure history. A stale lease cannot end +or annotate a replacement attempt. A failed ledger write leaves state unconfirmed +and requires subsequent lease recovery; the CLI does not report successful cleanup. + +Relative source paths resolve from the descriptor's directory. The descriptor +is operator-owned input, not candidate-provided instructions. Source files must +pass the bounded regular-file reader. Study reference hashes use the shared +`previewReferenceImport(...).summary.contentHash`; input hashes use +`packageInputHash`. Keep retained sources and proposals alongside the study for +reproducibility; metadata alone does not prove a complete night or subject +identity. + +Night files are opened only after the ledger records exposure. The receipt +contains the winning trial/report identity, report path and current ledger +evidence. Repeating an unchanged completed descriptor verifies and reuses its +report without rereading night files. Reusing its idempotency key with changed +configuration or study fails. Review current exposure again at any later +promotion decision, since other trials can invalidate first-use evidence. +A receipt is not an installation or qualification approval. + +## Recovery after a terminated experiment controller + +```sh +pnpm biometrics:experiment recover recovery-report.json +``` + +Experiment and environment-probe containers carry immutable version, invoking +host UID, kind and expiry labels. Expiry includes the maximum execution deadline +plus two minutes for startup and cleanup. Run this command after restarting an +experiment host and periodically while accepting work. It checks labels, names +and full container IDs on the currently selected Docker daemon, then force-removes +only overdue owned containers by immutable ID. Unexpired, unlabelled and foreign +containers remain untouched. The private report records inspected count and +removed IDs; repeating a sweep is safe. A daemon outage fails the command and +must be retried. Legacy containers without these labels require manual review. + +Recovery does not retroactively qualify an interrupted trial or erase exposure. +The ledger's expired lease permits a fresh fenced attempt, with earlier holdout +access still counted. The container sweep does not delete retained reports; its subsequent staging +sweep is described below. This is not a background supervisor: the external host +must schedule it. Host clock correctness is required for absolute expiry. Aggregate +temporary-staging quotas and periodic recovery scheduling remain separate work. + +## Retained evidence verification + +`readManagedEvidence(ledger, trialId, reportDirectory)` starts from the successful +ledger-selected report hash. It verifies every linked night artifact, exact +study/request/protocol/attempt identities, the complete selected-night set, +reference content and subject/side, all three package/configuration/input/image +identities, and declared resource limits. It recomputes reference comparisons, +resource assessments and whole-night uncertainty, requiring both the per-night +and top-level summaries to match those calculations. Updating hashes around an +altered score does not make that score valid. + +Completed managed runs use this verifier before returning a cached result. It +reads reports only, without rerunning candidate code or reopening retained raw +night inputs. The returned evidence includes a fresh ledger exposure snapshot; +later overlapping access can invalidate a previously unique holdout attempt. +This does not prove that a local administrator has never replaced the ledger or +fabricated raw execution records. The experiment host and its retained history +remain trusted. Qualification must recheck current exposure when publishing its +decision and still requires objective criteria, determinism, representative +fixtures, compatibility, target-pod resources and reviewed release promotion. + +## Frozen accuracy criteria + +Managed protocols may include `accuracyPolicy` using schema +`biometric-package-accuracy-policy-v1`. Choose and review its explicit criteria +before freezing the study: `minimumMaeImprovement`, +`maximumNightMaeRegression`, `maximumNightP95Regression`, +`maximumNightAbsoluteBiasIncrease`, `minimumCommonPairsPerNight`, +`minimumReferenceCoverage`, `maximumReferenceCoverageLoss`, and +`maximumLostTimestampFraction`. Error thresholds use the selected metric's +canonical unit; coverage/loss thresholds are fractions. There are no implicit +production thresholds. Changing criteria changes the protocol hash and cannot +retrofit the original study. Legacy protocols without criteria remain readable +but cannot meet the accuracy gate. + +The managed receipt's `accuracy` assessment is recomputed from verified retained +evidence and current ledger exposure. It requires a unique current holdout +attempt, sufficient common pairs on every selected night, coverage and support +within the frozen limits, no excessive per-night MAE/tail/bias regression, and +an estimated whole-night confidence interval supporting the requested MAE +improvement against both default and current. Missing or degenerate uncertainty, +omitted nights, ambiguous support and absent criteria remain insufficient. +Known violations produce `criteria_not_met` even if other evidence is missing. + +`criteria_met` means only that these reference-agreement criteria passed. It is +not clinical validation or package qualification: determinism, resource/noise +fixtures, compatibility, pod fitness, prospective monitoring and reviewed release +promotion still apply. The assessment retains exact failures and missing evidence +and never installs code or changes live processing. + +## Combined external preflight + +```sh +pnpm biometrics:experiment preflight proposal.json plan.json suite.json operator-policy.json preflight-report.json +``` + +The plan uses `biometric-package-preflight-plan-v1`, `suiteHash` (the canonical +`dataHash` of the parsed fixture suite), `repetitions` (2–5), `configHash` +(`configHash({ config, seed })`), `hardwareId`, `sampleRateHz`, and `side`. +Every fixture must use that exact candidate configuration and seed, hardware, +rate and side. A changed suite or configuration is rejected before execution. +Use reviewed synthetic fixtures here, not undisclosed holdout data. + +The runner snapshots candidate bytes, suite and operator policy, executes the +fixture suite, then repeats every fixture in independent containers to check +full-output/state determinism. Each repeat also receives a resource assessment +under the suite's frozen policy. Execution failures and nondeterminism fail the +preflight. Missing fixture categories or incomplete resource observations keep +it incomplete. Cancellation propagates to active experiments and stops further +fixtures/repeats. Reports retain the exact plan and its hash, fixture report, +per-fixture determinism reports, resource assessments and bounded errors. + +`external_checks_passed` is external preflight only. Managed studies can bind +the preflight artifact as described below; standalone reports cannot prove +holdout independence or pod fitness. The final combined qualification decision +remains separate. The candidate still needs representative fixture review, current +held-out accuracy, target-pod compatibility/resources, and normal reviewed +release promotion before it can become live. + +## Binding preflight to a managed study + +Include `preflightPlan` in the managed protocol and `preflightSuite` (a suite-file +path relative to the descriptor) in the managed descriptor. Both are required +together. The study protocol hash freezes the plan, including suite identity, +repetitions and candidate configuration/seed. Its hardware, rate and side must +match the managed study. The runner snapshots the suite before asynchronous work +and rejects mismatches before claiming a trial. + +During the leased attempt, preflight runs under the same cancellation heartbeat +before night comparisons. Its complete report, including candidate manifest, is +published by content hash and referenced by the final trial report. Failed or +incomplete preflight remains visible alongside the night results; completing a +trial does not imply that preflight passed. Failure reports retain a completed +preflight reference if a later step fails. Legacy trials without a plan have no +preflight evidence and cannot supply that qualification dimension. + +The retained-evidence verifier reopens the bound preflight artifact and checks +its plan, candidate/image, exact fixture set, fixture inputs and output protocol, +resource limits and measurements, repeat count, run identities and determinism +hashes. It recomputes fixture outcomes, resource assessments, missing categories +and overall status. Missing, altered or inconsistent artifacts fail cached reuse. +Completed file workflows do not reread the original suite or night input files. +The managed receipt exposes verified preflight status alongside accuracy; target +pod compatibility, representative-fixture review and release promotion remain +required before an algorithm can become live. + +## Combined external qualification assessment + +`assessManagedQualification(ledger, trialId, reportDirectory)` verifies the full +retained graph once, reads current holdout exposure, assesses frozen accuracy, +and combines it with bound preflight and every night/variant's resource evidence. +The managed command includes this `qualification` object and an `assessmentHash` +in its receipt. The hash binds trial/report/protocol/request identities, current +exposure and all assessment dimensions. Re-reviewing unchanged evidence is stable; +later overlapping holdout use changes the snapshot and invalidates its independent +holdout claim even though the original execution report remains unchanged. + +The combined status is `failed`, `insufficient_evidence`, or +`external_criteria_met`. Known failures dominate missing evidence, with both +reason lists retained. Passing accuracy cannot waive absent/failed preflight, +missing or duplicated resource rows, substituted resource policy, budget excess, +or inadequate measurements for default/current/candidate on any selected night. + +This is an external evidence snapshot, not a durable installation authorization. +`installationAuthorized` remains false, and the receipt lists outstanding +representative-fixture review, target-pod compatibility and resource qualification, +prospective health/reference comparison, and reviewed shipped-registry release. +A future promotion step must recheck authoritative evidence at publication; an +old exported assessment must never substitute for that current check. + + +### Abandoned private staging + +Before freezing inputs, the executor writes a private, fsynced ownership receipt +inside its mode-0700 staging directory. It binds the directory name/device/inode, +host UID, controller PID, exact container name, and wall deadline plus two-minute +grace. These receipts require a Unix host UID (macOS or Linux). Package code cannot +read or modify them through its package-only mount. + +`recover OUTPUT` first performs the existing expired-container sweep, then scans +host temporary staging. Its output is `biometric-experiment-recovery-v1` with +separate `containers` and `staging` results. Staging is removed only when the +receipt is valid and expired, its controller PID is confirmed absent with ESRCH, +and Docker confirms the named container does not exist. A present container or +an unavailable daemon never proves absence. PID reuse/permission errors retain +staging conservatively. Directory and marker identities are rechecked before +removal; markers must be private, owned, bounded regular files with one link. +The sweep is bounded to 16,384 temporary-directory entries and 128 candidates. + +Normal cleanup also preserves staging if container removal fails. Legacy or +unmarked directories, damaged receipts, symlinks, and unproven ownership are left +untouched. A crash between creating the empty directory and writing its receipt +can leave an unmarked empty directory; recovery will not guess its ownership. +Reports and exposure ledgers are not touched. This is an explicit recovery sweep; +a periodic host service and aggregate temporary-staging quotas remain separate. diff --git a/scripts/algorithm-experiments/accuracy-policy.ts b/scripts/algorithm-experiments/accuracy-policy.ts new file mode 100644 index 00000000..d9414ffd --- /dev/null +++ b/scripts/algorithm-experiments/accuracy-policy.ts @@ -0,0 +1,15 @@ +import { z } from 'zod' + +/** Operator-reviewed criteria, frozen into the protocol before holdout access. + * There are deliberately no production defaults or clinical accuracy claims. */ +export const packageAccuracyPolicySchema = z.object({ + schema: z.literal('biometric-package-accuracy-policy-v1'), + minimumMaeImprovement: z.number().finite().positive(), + maximumNightMaeRegression: z.number().finite().nonnegative(), + maximumNightP95Regression: z.number().finite().nonnegative(), + maximumNightAbsoluteBiasIncrease: z.number().finite().nonnegative(), + minimumCommonPairsPerNight: z.number().int().positive().max(100000), + minimumReferenceCoverage: z.number().positive().max(1), + maximumReferenceCoverageLoss: z.number().min(0).max(1), + maximumLostTimestampFraction: z.number().min(0).max(1), +}).strict() diff --git a/scripts/algorithm-experiments/accuracy.test.ts b/scripts/algorithm-experiments/accuracy.test.ts new file mode 100644 index 00000000..b05be08e --- /dev/null +++ b/scripts/algorithm-experiments/accuracy.test.ts @@ -0,0 +1,82 @@ +// @vitest-environment node +import { createHash } from 'node:crypto' +import { describe, expect, it } from 'vitest' +import { compareNumericConfigurations } from '../../src/biometrics/backtest/numeric-comparison' +import { createNightUncertainty } from '../../src/biometrics/backtest/uncertainty-core' +import { normalizeReferenceImport } from '../../src/biometrics/references/normalize-core' +import { assessPackageAccuracy } from './accuracy' +import { managedProtocolSchema } from './managed-protocol' +import { fixtureResourceTestPolicy } from './resource-test-policy' +import { configHash } from '../../src/biometrics/algorithms/canonical' + +type Input = Parameters[0] +const policy = { schema: 'biometric-package-accuracy-policy-v1', minimumMaeImprovement: 0.5, maximumNightMaeRegression: 0, + maximumNightP95Regression: 0, maximumNightAbsoluteBiasIncrease: 0, minimumCommonPairsPerNight: 2, + minimumReferenceCoverage: 1, maximumReferenceCoverageLoss: 0, maximumLostTimestampFraction: 0 } +function fixture(): Input { + const protocol = managedProtocolSchema.parse({ schema: 'biometric-managed-protocol-v1', hardwareId: 'pod4', sampleRateHz: 500, metricId: 'heart_rate', minimumQuality: 0.8, + alignment: { version: 'reference-alignment-v1', pointToleranceMs: 0, clockOffsetMs: 0, offsetSource: 'fixed' }, + uncertainty: { version: 'paired-night-percentile-v1', seed: 'accuracy-test', resamples: 200, minimumNights: 3 }, + resourcePolicy: fixtureResourceTestPolicy, accuracyPolicy: policy }) + const reference = normalizeReferenceImport({ subjectId: 'fixture', side: 'left', source: 'synthetic', + rows: [0, 1].map(index => ({ recordKey: String(index), metric: 'heart_rate', value: 60, unit: 'bpm', timestampMs: 1700000000000 + index * 1000 })) }) + const nights = [0, 1, 2].map((index) => { + const outputs = (['default', 'current', 'candidate'] as const).map((label, variant) => ({ label, configHash: String(variant), + observations: [0, 1].map(point => ({ timestampMs: 1700000000000 + point * 1000, value: variant === 2 ? 60 : 64 + index + (1 - variant) })) })) + const comparison = { schema: 'biometric-package-comparison-report-v1' as const, inputHash: 'a', referenceHash: 'b', metricId: 'heart_rate', + contract: { metric: 'heart_rate' as const, windowMs: null }, alignment: protocol.alignment, startMs: 1700000000000, endMs: 1700000060000, + minimumQuality: 0.8, excludedBoundaryReferences: 0, compatibilityExclusions: {}, referenceSummary: reference.summary, + comparisons: compareNumericConfigurations('heart_rate', null, outputs, reference.measurements, protocol.alignment), + interpretation: 'descriptive_reference_agreement_not_independent_holdout_or_promotion_evidence' as const } + return { nightId: String(index), reportHash: 'c', comparison, resources: [] } + }) + const uncertainty = createNightUncertainty(text => createHash('sha256').update(text).digest(), 'equal_night_mean_common_mae_difference') + return { protocol, nights, uncertainty: (['default', 'current'] as const).map(baseline => ({ baseline, + report: uncertainty(nights.map(night => ({ cohortId: night.nightId, difference: night.comparison.comparisons.find(row => row.baseline === baseline && row.candidate === 'candidate')?.differences.commonMae ?? null })), protocol.uncertainty) })), + currentEvidence: { trialId: 'fixture', state: 'succeeded', reportHash: 'hash', otherExposures: [], uniqueHoldoutAttempt: true } } +} +describe('frozen package accuracy criteria', () => { + it('requires supported improvement against both baselines without authorizing installation', () => { + expect(assessPackageAccuracy(fixture())).toMatchObject({ status: 'criteria_met', failures: [], missing: [], interpretation: expect.stringContaining('requires_full_package') }) + const input = fixture() + const report = input.uncertainty[1].report + if (report.interval) report.interval.upper = 0 + expect(assessPackageAccuracy(input)).toMatchObject({ status: 'criteria_not_met', failures: ['current:improvement_not_supported'] }) + }) + it('cannot backfill criteria into an already frozen protocol or reuse exposed holdout', () => { + const input = fixture(), frozen = configHash(input.protocol) + delete input.protocol.accuracyPolicy + expect(configHash(input.protocol)).not.toBe(frozen) + expect(assessPackageAccuracy(input).missing).toContain('no_frozen_accuracy_policy') + input.currentEvidence.uniqueHoldoutAttempt = false + expect(assessPackageAccuracy(input).missing).toContain('no_unique_current_holdout') + }) + it('rejects coverage loss and individual-night regressions even with favorable aggregate intervals', () => { + const input = fixture() + const row = input.nights[0].comparison.comparisons.find(row => row.candidate === 'candidate' && row.baseline === 'default') + if (!row) throw new Error('Missing fixture comparison') + row.ownSupport.candidate.referenceCoverage = 0.5 + row.differences.referenceCoverage = -0.5 + row.lostObservationTimestamps = 1 + row.differences.commonMae = 1 + row.differences.commonP95AbsoluteError = 2 + if (row.commonSupport.candidate.metrics) row.commonSupport.candidate.metrics.signedBias = 99 + const result = assessPackageAccuracy(input) + expect(result.status).toBe('criteria_not_met') + expect(result.failures).toEqual(expect.arrayContaining(['0:default:reference_coverage_below_minimum', '0:default:reference_coverage_loss', + '0:default:observation_support_loss', '0:default:night_mae_regression', '0:default:night_tail_regression', '0:default:night_bias_regression'])) + }) + it('does not treat omitted nights, missing pairs or degenerate intervals as successful evidence', () => { + const input = fixture() + input.uncertainty[0].report.omittedNights = 1 + input.uncertainty[1].report = { ...input.uncertainty[1].report, status: 'degenerate', interval: null } + const row = input.nights[0].comparison.comparisons.find(row => row.candidate === 'candidate' && row.baseline === 'default') + if (!row) throw new Error('Missing fixture comparison') + row.commonSupport.candidate.pairedCount = 0 + row.ambiguousObservationTimestamps.candidate = 1 + expect(assessPackageAccuracy(input)).toMatchObject({ status: 'insufficient_evidence', missing: expect.arrayContaining([ + 'default:insufficient_whole_night_uncertainty', 'current:insufficient_whole_night_uncertainty', + '0:default:insufficient_common_pairs', '0:default:ambiguous_observation_support', + ]) }) + }) +}) diff --git a/scripts/algorithm-experiments/accuracy.ts b/scripts/algorithm-experiments/accuracy.ts new file mode 100644 index 00000000..900052d7 --- /dev/null +++ b/scripts/algorithm-experiments/accuracy.ts @@ -0,0 +1,66 @@ +import { configHash } from '../../src/biometrics/algorithms/canonical' +import { packageAccuracyPolicySchema } from './accuracy-policy' +import { readManagedEvidence } from './managed-evidence' + +type Evidence = ReturnType +type AccuracyInput = Pick + +/** Pure assessment of already verified evidence. The authoritative entry point + * below reads the retained graph and fresh ledger state first. */ +export function assessPackageAccuracy(evidence: AccuracyInput) { + const failures: string[] = [], missing: string[] = [] + const rawPolicy = evidence.protocol.accuracyPolicy + const policy = rawPolicy === undefined ? null : packageAccuracyPolicySchema.parse(rawPolicy) + if (!policy) missing.push('no_frozen_accuracy_policy') + if (!evidence.currentEvidence.uniqueHoldoutAttempt) missing.push('no_unique_current_holdout') + if (!evidence.nights.length) missing.push('no_nights') + for (const night of evidence.nights) { + for (const baseline of ['default', 'current'] as const) { + const key = `${night.nightId}:${baseline}` + const comparison = night.comparison.comparisons.find(row => row.candidate === 'candidate' && row.baseline === baseline) + if (!comparison) { + missing.push(`${key}:missing_comparison`) + continue + } + if (comparison.commonSupport.candidate.status === 'incompatible' || comparison.commonSupport.baseline.status === 'incompatible') { + failures.push(`${key}:incompatible_metric`) + continue + } + if (comparison.ambiguousObservationTimestamps.baseline || comparison.ambiguousObservationTimestamps.candidate) missing.push(`${key}:ambiguous_observation_support`) + if (!policy) continue + const candidate = comparison.commonSupport.candidate, previous = comparison.commonSupport.baseline + if (!candidate.metrics || !previous.metrics || candidate.pairedCount < policy.minimumCommonPairsPerNight + || previous.pairedCount < policy.minimumCommonPairsPerNight) missing.push(`${key}:insufficient_common_pairs`) + const own = comparison.ownSupport.candidate + if (own.referenceCoverage === null) missing.push(`${key}:missing_reference_coverage`) + else if (own.referenceCoverage < policy.minimumReferenceCoverage) failures.push(`${key}:reference_coverage_below_minimum`) + const coverageChange = comparison.differences.referenceCoverage + if (coverageChange === null) missing.push(`${key}:missing_coverage_difference`) + else if (coverageChange < -policy.maximumReferenceCoverageLoss) failures.push(`${key}:reference_coverage_loss`) + const baselineCount = comparison.ownSupport.baseline.validObservationCount + if (!baselineCount) missing.push(`${key}:missing_baseline_observations`) + else if (comparison.lostObservationTimestamps / baselineCount > policy.maximumLostTimestampFraction) failures.push(`${key}:observation_support_loss`) + if (comparison.differences.commonMae !== null && comparison.differences.commonMae > policy.maximumNightMaeRegression) failures.push(`${key}:night_mae_regression`) + if (comparison.differences.commonP95AbsoluteError !== null && comparison.differences.commonP95AbsoluteError > policy.maximumNightP95Regression) failures.push(`${key}:night_tail_regression`) + if (candidate.metrics && previous.metrics + && Math.abs(candidate.metrics.signedBias) - Math.abs(previous.metrics.signedBias) > policy.maximumNightAbsoluteBiasIncrease) failures.push(`${key}:night_bias_regression`) + } + } + for (const baseline of ['default', 'current'] as const) { + const report = evidence.uncertainty.find(row => row.baseline === baseline)?.report + if (!report || report.status !== 'estimated' || !report.interval || report.omittedNights > 0 + || report.scoredNights !== evidence.nights.length) missing.push(`${baseline}:insufficient_whole_night_uncertainty`) + else if (policy && (report.estimate === null || report.estimate > -policy.minimumMaeImprovement + || report.interval.upper > -policy.minimumMaeImprovement)) failures.push(`${baseline}:improvement_not_supported`) + } + return { schema: 'biometric-package-accuracy-assessment-v1' as const, + status: failures.length ? 'criteria_not_met' as const : missing.length ? 'insufficient_evidence' as const : 'criteria_met' as const, + policyHash: policy ? configHash(policy) : null, failures, missing, + interpretation: 'reference_agreement_criteria_only_requires_full_package_qualification_and_review' as const } +} + +export function assessManagedAccuracy(...args: Parameters) { + const evidence = readManagedEvidence(...args) + return { trialId: evidence.trialId, reportHash: evidence.reportHash, currentEvidence: evidence.currentEvidence, + assessment: assessPackageAccuracy(evidence), preflight: evidence.preflight } +} diff --git a/scripts/algorithm-experiments/cli.ts b/scripts/algorithm-experiments/cli.ts new file mode 100644 index 00000000..bbd3bbe0 --- /dev/null +++ b/scripts/algorithm-experiments/cli.ts @@ -0,0 +1,87 @@ +import { recoverExperimentStaging } from './staging-recovery' +import { runPackagePreflight } from './preflight' +import { recoverExpiredContainers } from './recovery' +import { pathToFileURL } from 'node:url' +import { z } from 'zod' +import { runManagedFiles } from './managed-files' +import { experimentPolicySchema as policySchema } from './policy' +import { comparePackageOutputs } from './comparison' +import { runPackageFixtures } from './fixtures' +import { packageInputHash } from './input' +import { inspectPythonEnvironment } from './environment' +import { executePackageExperiment } from './container' +import { checkPackageDeterminism } from './determinism' +import { publishJson, readJsonFile, readJsonLines, readPackageProposal } from './files' + +const positive = z.number().int().positive().max(Number.MAX_SAFE_INTEGER) +const usage = 'Usage: preflight PROPOSAL PLAN SUITE POLICY OUTPUT | recover OUTPUT | managed WORKSPACE DESCRIPTOR RECEIPT [STORAGE_POLICY] | compare INPUT OUTPUT | fixtures PROPOSAL SUITE POLICY OUTPUT | lock IMAGE OUTPUT | input-hash FRAMES MAX_BYTES OUTPUT | inspect PROPOSAL OUTPUT | run PROPOSAL REQUEST FRAMES POLICY OUTPUT | determinism PROPOSAL REQUEST FRAMES POLICY REPETITIONS OUTPUT' + +export async function runExperimentCli(args: string[]) { + const [command, ...paths] = args + if (command === 'preflight' && paths.length === 5) { + const proposal = readPackageProposal(paths[0]) + const plan = readJsonFile(paths[1], 16384) + const suite = readJsonFile(paths[2], 32 * 1024 * 1024) + const policy = policySchema.parse(readJsonFile(paths[3], 16384)) + await publishJson(paths[4], await runPackagePreflight({ ...proposal, policy }, plan, suite)) + return + } + if (command === 'recover' && paths.length === 1) { + const containers = await recoverExpiredContainers() + const staging = await recoverExperimentStaging() + await publishJson(paths[0], { schema: 'biometric-experiment-recovery-v1', containers, staging }) + return + } + if (command === 'managed' && (paths.length === 3 || paths.length === 4)) { + await publishJson(paths[2], await runManagedFiles(paths[0], paths[1], paths[3] ? readJsonFile(paths[3], 16384) : undefined)) + return + } + if (command === 'compare' && paths.length === 2) { + await publishJson(paths[1], comparePackageOutputs(readJsonFile(paths[0], 32 * 1024 * 1024))) + return + } + if (command === 'fixtures' && paths.length === 4) { + const proposal = readPackageProposal(paths[0]) + const suite = readJsonFile(paths[1], 32 * 1024 * 1024) + const policy = policySchema.parse(readJsonFile(paths[2], 16384)) + const report = await runPackageFixtures({ manifest: proposal.manifest, files: proposal.files, policy }, suite) + await publishJson(paths[3], report) + return + } + if (command === 'lock' && paths.length === 2) { + await publishJson(paths[1], await inspectPythonEnvironment(paths[0])) + return + } + if (command === 'input-hash' && paths.length === 3) { + const maximumBytes = positive.parse(Number(paths[1])) + const hash = packageInputHash(readJsonLines(paths[0], maximumBytes)) + await publishJson(paths[2], { schema: 'biometric-package-input-v1', hash }) + return + } + if (command === 'inspect' && paths.length === 2) { + const proposal = readPackageProposal(paths[0]) + await publishJson(paths[1], { schema: 'biometric-package-intake-v1', packageHash: proposal.packageHash, + status: proposal.status, manifest: proposal.manifest, totalBytes: proposal.totalBytes }) + return + } + if ((command === 'run' && paths.length === 5) || (command === 'determinism' && paths.length === 6)) { + const policy = policySchema.parse(readJsonFile(paths[3], 16384)) + const proposal = readPackageProposal(paths[0]) + const request = readJsonFile(paths[1], 65536) + const frames = () => readJsonLines(paths[2], policy.maximumInputBytes) + const options = { manifest: proposal.manifest, files: proposal.files, request, policy } + const report = command === 'run' + ? { schema: 'biometric-package-experiment-v1', status: 'experiment_only', ...await executePackageExperiment({ ...options, frames: frames() }) } + : await checkPackageDeterminism({ ...options, frames }, Number(paths[4])) + await publishJson(paths[paths.length - 1], report) + return + } + throw new Error(usage) +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + runExperimentCli(process.argv.slice(2)).catch((error: unknown) => { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) + process.exitCode = 1 + }) +} diff --git a/scripts/algorithm-experiments/comparison.test.ts b/scripts/algorithm-experiments/comparison.test.ts new file mode 100644 index 00000000..a645e6f1 --- /dev/null +++ b/scripts/algorithm-experiments/comparison.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from 'vitest' +import { configHash } from '../../src/biometrics/algorithms/canonical' +import { packageFixture } from '../../src/biometrics/packages/tests/fixture' +import { comparePackageOutputs } from './comparison' +const start = 1791673200000 +function fixture() { + const outputs = (['default', 'current', 'candidate'] as const).map((label, index) => { + const { manifest } = packageFixture() + const declared = { ...manifest, metrics: manifest.metrics.map(metric => ({ ...metric, reference: { metric: 'heart_rate', windowMs: null as number | null } })) } + const packageHash = configHash(declared) + const request = { schema: 'biometric-package-run-v1', packageHash, inputHash: 'a'.repeat(64), hardwareId: 'pod4', sampleRateHz: 500, side: 'left', seed: 1, config: { offset: index } } + const runHash = configHash(request) + return { label, manifest: declared, request, result: { packageHash, runHash, + frozenInput: { schema: 'biometric-package-input-v1', hash: request.inputHash, bytes: 1000, frameCount: 1 }, + results: [{ schema: 'biometric-metric-frame-v1', runHash, frameHash: 'b'.repeat(64), sequence: 0, state: {}, + readings: [0, 60000].map(offset => ({ timestampMs: start + offset, side: 'left', metric: 'heart_rate', value: 64 - index * 2 as number | null, quality: 1, flags: [] })) }] } } + }) + return { schema: 'biometric-package-comparison-v1', metricId: 'heart_rate', startMs: start, endMs: start + 120000, minimumQuality: 0.8, + referenceImport: { side: 'left', subjectId: 'fixture', source: 'synthetic', rows: [0, 60000].map((offset, index) => ({ recordKey: String(index), metric: 'heart_rate', value: 60, unit: 'bpm', timestampMs: start + offset })) }, + alignment: { version: 'reference-alignment-v1', pointToleranceMs: 0, clockOffsetMs: 0, offsetSource: 'fixed' }, outputs } +} +function rebind(output: ReturnType['outputs'][number]) { + output.request.packageHash = configHash(output.manifest) + output.result.packageHash = output.request.packageHash + output.result.runHash = configHash(output.request) + output.result.results.forEach((frame) => { + frame.runHash = output.result.runHash + }) +} +describe('package reference agreement', () => { + it('compares both baselines with shared pairing without granting promotion', () => { + const result = comparePackageOutputs(fixture()) + expect(result.comparisons.filter(row => row.candidate === 'candidate').map(row => row.differences.commonMae)).toEqual([-4, -2]) + expect(result.interpretation).toContain('not_independent_holdout_or_promotion') + }) + it('exposes dropped and low-quality values as lost support', () => { + for (const lowQuality of [false, true]) { + const input = fixture() + const row = input.outputs[2].result.results[0].readings[0] + if (lowQuality) row.quality = 0.1 + else row.value = null + const comparison = comparePackageOutputs(input).comparisons.find(row => row.candidate === 'candidate' && row.baseline === 'default') + expect(comparison).toMatchObject({ coverageReduced: true, lostObservationTimestamps: 1, commonObservationCount: 1 }) + } + }) + it('rejects substituted run, frame, input, side and duplicate baseline identities', () => { + for (const change of [ + (input: ReturnType) => { input.outputs[2].result.runHash = 'c'.repeat(64) }, + (input: ReturnType) => { input.outputs[2].result.results[0].frameHash = 'c'.repeat(64) }, + (input: ReturnType) => { input.outputs[2].request.side = 'right' }, + (input: ReturnType) => { input.outputs[2].request.inputHash = 'c'.repeat(64) }, + (input: ReturnType) => { input.outputs[2].label = 'default' }, + ]) { + const input = fixture() + change(input) + expect(() => comparePackageOutputs(input)).toThrow() + } + }) + it('rejects incompatible package definitions before scoring', () => { + const input = fixture() + input.outputs[2].manifest.metrics[0].reference = { metric: 'hrv_rmssd', windowMs: 300000 } + input.outputs[2].manifest.metrics[0].unit = 'ms' + rebind(input.outputs[2]) + expect(() => comparePackageOutputs(input)).toThrow('Incompatible package metric') + }) + it('does not score external RMSSD against the window IBI variability index', () => { + const input = fixture() + for (const output of input.outputs) { + output.manifest.metrics[0].reference = { metric: 'window_ibi_variability_v1', windowMs: 300000 } + output.manifest.metrics[0].unit = 'ms' + rebind(output) + } + input.referenceImport.rows = input.referenceImport.rows.map(row => ({ ...row, metric: 'hrv_rmssd', unit: 'ms', windowMs: 300000 })) + const report = comparePackageOutputs(input) + expect(report.compatibilityExclusions).toEqual({ incompatible_metric_definition: 2 }) + expect(report.comparisons.every(row => row.commonSupport.candidate.status === 'incompatible' && row.commonSupport.candidate.metrics === null)).toBe(true) + }) +}) diff --git a/scripts/algorithm-experiments/comparison.ts b/scripts/algorithm-experiments/comparison.ts new file mode 100644 index 00000000..f1c4a5cf --- /dev/null +++ b/scripts/algorithm-experiments/comparison.ts @@ -0,0 +1,76 @@ +import { z } from 'zod' +import { configHash, dataHash } from '../../src/biometrics/algorithms/canonical' +import { compareNumericConfigurations } from '../../src/biometrics/backtest/numeric-comparison' +import { algorithmPackageManifestSchema } from '../../src/biometrics/packages/manifest' +import { metricFrameSchema, packageRunRequestSchema } from '../../src/biometrics/packages/protocol' +import { referenceCompatibility } from '../../src/biometrics/references/alignment' +import { normalizeReferenceImport } from '../../src/biometrics/references/normalize-core' +import { referenceImportSchema, timestampMsSchema } from '../../src/biometrics/references/contracts' + +const hash = z.string().regex(/^[a-f0-9]{64}$/) +export const packageComparisonInputSchema = z.object({ + schema: z.literal('biometric-package-comparison-v1'), metricId: z.string().min(1).max(100), + startMs: timestampMsSchema, endMs: timestampMsSchema, minimumQuality: z.number().min(0).max(1), + referenceImport: referenceImportSchema, + alignment: z.object({ version: z.literal('reference-alignment-v1'), pointToleranceMs: z.number().int().min(0).max(300000), + clockOffsetMs: z.number().int().min(-3600000).max(3600000), offsetSource: z.enum(['fixed', 'training']) }).strict(), + outputs: z.array(z.object({ label: z.enum(['default', 'current', 'candidate']), manifest: algorithmPackageManifestSchema, + request: packageRunRequestSchema, + result: z.object({ packageHash: hash, runHash: hash, + frozenInput: z.object({ schema: z.literal('biometric-package-input-v1'), hash, bytes: z.number().int().positive(), frameCount: z.number().int().positive() }).strict(), + results: z.array(metricFrameSchema).max(100000), + }), + }).strict()).length(3), +}).strict().refine(value => value.endMs > value.startMs && value.endMs - value.startMs <= 86400000, 'Comparison requires a bounded whole-night interval') + +/** Descriptive scoring of host-produced experiment reports. This does not + * attest execution provenance or independent holdout use; promotion must use + * the authoritative experiment store and reviewed qualification pipeline. */ +export function comparePackageOutputs(raw: unknown) { + const input = packageComparisonInputSchema.parse(raw) + if (new Set(input.outputs.map(output => output.label)).size !== 3) throw new Error('Default, current and candidate outputs are required') + const reference = normalizeReferenceImport(input.referenceImport) + let commonInput: string | undefined + let frameIdentity: string | undefined + let contract: { metric: 'heart_rate' | 'breathing_rate' | 'window_ibi_variability_v1' | 'hrv_rmssd' | 'hrv_sdnn', windowMs: number | null } | undefined + const outputs = input.outputs.map((output) => { + const { manifest, request, result } = output + const packageHash = configHash(manifest) + if (packageHash !== request.packageHash || packageHash !== result.packageHash || configHash(request) !== result.runHash + || result.frozenInput.hash !== request.inputHash || result.frozenInput.frameCount !== result.results.length) throw new Error('Package comparison identity mismatch') + if (request.side !== input.referenceImport.side) throw new Error('Reference side mismatch') + const identity = configHash({ inputHash: request.inputHash, side: request.side, hardwareId: request.hardwareId, sampleRateHz: request.sampleRateHz }) + if (commonInput !== undefined && commonInput !== identity) throw new Error('Packages did not process the same input and side') + commonInput = identity + const frameHash = dataHash(result.results.map(frame => frame.frameHash)) + if (frameIdentity !== undefined && frameIdentity !== frameHash) throw new Error('Packages received different frames') + frameIdentity = frameHash + const metric = manifest.metrics.find(metric => metric.id === input.metricId) + if (!metric?.reference) throw new Error('Package lacks a comparable metric contract') + if (contract && configHash(contract) !== configHash(metric.reference)) throw new Error('Incompatible package metric definitions or windows') + contract = metric.reference + const observations = [] + for (const [index, frame] of result.results.entries()) { + if (frame.runHash !== result.runHash || frame.sequence !== index) throw new Error('Package output identity mismatch') + for (const reading of frame.readings) { + if (reading.side !== request.side) throw new Error('Package output side mismatch') + if (reading.metric !== input.metricId) continue + if (reading.value !== null && (reading.value < metric.minimum || reading.value > metric.maximum)) throw new Error('Package metric outside declared range') + if (reading.timestampMs >= input.startMs && reading.timestampMs < input.endMs) observations.push({ timestampMs: reading.timestampMs, value: reading.quality >= input.minimumQuality ? reading.value : null }) + } + } + return { label: output.label, configHash: result.runHash, observations } + }) + if (!contract) throw new Error('No comparable metric contract') + const selected = reference.measurements.filter(row => row.startMs >= input.startMs && row.startMs < input.endMs && row.endMs <= input.endMs) + const compatibilityExclusions: Record = {} + for (const row of selected) { + const reason = referenceCompatibility(contract.metric, contract.windowMs, row) + if (reason) compatibilityExclusions[reason] = (compatibilityExclusions[reason] ?? 0) + 1 + } + const comparisons = compareNumericConfigurations(contract.metric, contract.windowMs, outputs, selected, input.alignment) + return { schema: 'biometric-package-comparison-report-v1' as const, inputHash: dataHash(input), referenceHash: dataHash(reference.measurements), + metricId: input.metricId, contract, alignment: input.alignment, startMs: input.startMs, endMs: input.endMs, + minimumQuality: input.minimumQuality, excludedBoundaryReferences: reference.measurements.length - selected.length, compatibilityExclusions, + referenceSummary: reference.summary, comparisons, interpretation: 'descriptive_reference_agreement_not_independent_holdout_or_promotion_evidence' as const } +} diff --git a/scripts/algorithm-experiments/container.test.ts b/scripts/algorithm-experiments/container.test.ts new file mode 100644 index 00000000..fc75db6f --- /dev/null +++ b/scripts/algorithm-experiments/container.test.ts @@ -0,0 +1,221 @@ +import { fixtureResourceTestPolicy } from './resource-test-policy' +// @vitest-environment node +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { dirname, join } from 'node:path' +import { statSync } from 'node:fs' +import * as frozenInput from './input' +import * as resourceReader from './resources' +import { tmpdir } from 'node:os' +import { runPackageFixtures } from './fixtures' +import { runExperimentCli } from './cli' +import { createHash } from 'node:crypto' +import { beforeAll, describe, expect, it, vi } from 'vitest' +import { canonicalJson, configHash } from '../../src/biometrics/algorithms/canonical' +import { packageFixture } from '../../src/biometrics/packages/tests/fixture' +import { inspectPythonEnvironment } from './environment' +import { checkPackageDeterminism } from './determinism' +import { packageInputHash } from './input' +import { executePackageExperiment } from './container' + +const image = process.env.BIOMETRIC_EXPERIMENT_TEST_IMAGE +let lockBytes: Buffer +beforeAll(async () => { + if (image) lockBytes = Buffer.from(JSON.stringify(await inspectPythonEnvironment(image))) +}) +const program = `import json, sys +init = json.loads(sys.stdin.readline()) +for line in sys.stdin: + message = json.loads(line) + frame = message['frame'] + print(json.dumps({'schema': 'biometric-metric-frame-v1', 'runHash': init['runHash'], 'frameHash': message['frameHash'], 'sequence': frame['sequence'], 'readings': [], 'state': {}}), flush=True) +` +function options(code: string) { + const { manifest, files } = packageFixture() + files.set('engine.py', Buffer.from(code)) + files.set(manifest.dependencyLock, lockBytes) + manifest.artifacts = [...files].map(([path, bytes]) => ({ path, bytes: bytes.length, sha256: createHash('sha256').update(bytes).digest('hex') })) + manifest.resources = { cpuMs: 5000, wallMs: 5000, memoryBytes: 64 * 1024 * 1024, processes: 1, outputBytes: 16384 } + const frames = [{ schema: 'biometric-signal-frame-v1', sequence: 0, timestampMs: 1000, samples: { left: [1, 2, 3], right: null }, context: { occupied: null, pumpActive: null } }] + frames.push({ ...frames[0], sequence: 1, timestampMs: 1006 }) + return { manifest, files, + request: { schema: 'biometric-package-run-v1', packageHash: configHash(manifest), inputHash: packageInputHash(frames), hardwareId: 'pod4', sampleRateHz: 500, side: 'left', seed: 123, config: {} }, + frames, + policy: { image: image ?? '', maximumWallMs: 10000, maximumMemoryBytes: 128 * 1024 * 1024, maximumProcesses: 4, maximumOutputBytes: 32768, maximumInputBytes: 32768 } } +} + +describe.skipIf(!image)('real external Docker experiments', () => { + it('runs the frame protocol with read-only input, no network, no extra processes or host environment', async () => { + const probes = `import os, socket +assert os.getuid() == 65534 +assert 'BIOMETRIC_EXPERIMENT_TEST_IMAGE' not in os.environ +assert not os.path.exists('/var/run/docker.sock') +assert not os.path.exists('/root/.ssh') +for path in ['/package/engine.py', '/tmp/write-attempt']: + try: + open(path, 'w') + except OSError: + pass + else: + raise AssertionError('Writable filesystem') +try: + os.fork() +except OSError: + pass +else: + raise AssertionError('Fork succeeded') +s = socket.socket() +s.settimeout(0.2) +try: + s.connect(('1.1.1.1', 443)) +except OSError: + pass +else: + raise AssertionError('Network available') +s.close() +` + const input = options(probes + program) + const execution = executePackageExperiment(input) + input.policy.image = 'sha256:' + 'b'.repeat(64) + const result = await execution + expect(result.execution.environment.image).toBe(image) + expect(result.execution.measurements.samples).toBeGreaterThan(0) + expect(result.execution.measurements.maximumObservedCpuMs).toBeGreaterThan(0) + expect(result.execution.measurements.peakObservedMemoryBytes).toBeGreaterThan(0) + expect(result.execution.measurements).toMatchObject({ source: 'host_docker_engine_stats', exactPeakMemory: false, exactLifetimeCpu: false }) + expect(result.results).toHaveLength(2) + expect(result.execution.isolation).toBe('docker-restricted-v1') + }, 20000) + it('keeps the host staging parent private while the unprivileged container reads its package mount', async () => { + let parent: string | undefined + let modes: number[] | undefined + const freeze = frozenInput.freezePackageInput + const readResources = resourceReader.dockerResourceReader + const capture = vi.spyOn(frozenInput, 'freezePackageInput').mockImplementation(async (path, frames, maximumBytes) => { + parent = dirname(path) + return freeze(path, frames, maximumBytes) + }) + const observe = vi.spyOn(resourceReader, 'dockerResourceReader').mockImplementation(async (id) => { + if (!parent) throw new Error('Missing staged input') + // This runs after Docker create, when all host permissions are final. + modes = [parent, join(parent, 'input.jsonl'), join(parent, 'package')].map(path => statSync(path).mode & 0o777) + return readResources(id) + }) + try { + expect((await executePackageExperiment(options(program))).results).toHaveLength(2) + expect(observe).toHaveBeenCalledOnce() + expect(modes).toEqual([0o700, 0o600, 0o755]) + const stagedParent = parent + if (!stagedParent) throw new Error('Missing staging') + expect(() => statSync(stagedParent)).toThrow() + } + finally { + capture.mockRestore() + observe.mockRestore() + } + }, 20000) + it('runs a complete file-based CLI experiment and publishes a reviewable report', async () => { + const root = await mkdtemp(join(tmpdir(), 'package-cli-test-')) + try { + const input = options(program) + const proposal = { schema: 'biometric-package-proposal-v1', manifest: input.manifest, + files: [...input.files].map(([path, bytes]) => ({ path, base64: bytes.toString('base64') })) } + for (const [name, value] of [['proposal', proposal], ['request', input.request], ['policy', input.policy]] as const) { + await writeFile(join(root, name), JSON.stringify(value)) + } + await writeFile(join(root, 'frames'), input.frames.map(frame => JSON.stringify(frame)).join('\n') + '\n') + await runExperimentCli(['run', ...['proposal', 'request', 'frames', 'policy', 'report'].map(name => join(root, name))]) + const report = JSON.parse(await readFile(join(root, 'report'), 'utf8')) + expect(report).toMatchObject({ schema: 'biometric-package-experiment-v1', status: 'experiment_only', packageHash: input.request.packageHash, + frozenInput: { hash: input.request.inputHash, frameCount: 2 } }) + expect(report.results).toHaveLength(2) + } + finally { await rm(root, { recursive: true, force: true }) } + }, 20000) + it('checks numeric coverage and catches false vitals during an empty-bed fixture', async () => { + const reading = '[{\'timestampMs\': frame[\'timestampMs\'], \'side\': \'left\', \'metric\': \'heart_rate\', \'value\': 60, \'quality\': 1, \'flags\': []}]' + const makeSuite = (input: ReturnType) => ({ schema: 'biometric-package-fixtures-v1', resourcePolicy: fixtureResourceTestPolicy, fixtures: [true, false].map((occupied) => { + const frames = input.frames.map(frame => ({ ...frame, context: { ...frame.context, occupied } })) + return { id: occupied ? 'clean' : 'empty', category: occupied ? 'clean_signal' : 'empty_bed', + request: { ...input.request, inputHash: packageInputHash(frames) }, frames, + expectations: [{ metric: 'heart_rate', startMs: 1000, endMs: 1012, ...(occupied + ? { kind: 'numeric', minimum: 59, maximum: 61, minimumReadings: 2, minimumQuality: 0.8 } + : { kind: 'abstain' }) }] } + }) }) + const candidate = options(program.replace('\'readings\': []', `'readings': ${reading} if frame['context']['occupied'] else []`)) + const suite = makeSuite(candidate) + const running = runPackageFixtures(candidate, suite) + suite.fixtures[0].frames[0].samples.left[0] = 999 + const report = await running + expect(report.suite.fixtures[0].frames[0].samples.left?.[0]).toBe(1) + expect(report.status).toBe('incomplete') + expect(report.results.every(result => result.assessment.passed)).toBe(true) + expect(report.missingCategories).toContain('pump') + const strictBudget = makeSuite(candidate) + strictBudget.resourcePolicy = { ...fixtureResourceTestPolicy, maximumWallMs: 1 } + const overBudget = await runPackageFixtures(candidate, strictBudget) + expect(overBudget.results.every(result => result.assessment.passed)).toBe(true) + expect(overBudget.results.every(result => result.resourceAssessment?.status === 'failed')).toBe(true) + expect(overBudget.status).toBe('failed') + const bad = options(program.replace('\'readings\': []', `'readings': ${reading}`)) + const regression = await runPackageFixtures(bad, makeSuite(bad)) + expect(regression.status).toBe('failed') + expect(regression.results[1].assessment.failures).toContain('failed_abstention:heart_rate:1000') + }, 30000) + it('repeats in fresh containers and detects changing state even with identical readings', async () => { + const input = options(program) + const stable = await checkPackageDeterminism({ ...input, frames: () => input.frames }) + expect(stable.deterministic).toBe(true) + expect(stable.runs).toHaveLength(2) + const unstable = options(program.replace('import json, sys', 'import json, sys, secrets').replace('\'state\': {}', '\'state\': {\'nonce\': secrets.token_hex(16)}')) + expect((await checkPackageDeterminism({ ...unstable, frames: () => unstable.frames })).deterministic).toBe(false) + }, 20000) + it('rejects a locked Python version that differs from the actual image before running code', async () => { + const input = options('raise AssertionError("candidate must not run")') + const wrong = JSON.parse(lockBytes.toString()) + wrong.pythonVersion = '0.0.0' + const bytes = Buffer.from(JSON.stringify(wrong)) + input.files.set(input.manifest.dependencyLock, bytes) + input.manifest.artifacts = input.manifest.artifacts.map(artifact => artifact.path === input.manifest.dependencyLock + ? { ...artifact, bytes: bytes.length, sha256: createHash('sha256').update(bytes).digest('hex') } + : artifact) + input.request.packageHash = configHash(input.manifest) + await expect(executePackageExperiment(input)).rejects.toThrow('differs from dependency lock') + }, 20000) + it('streams an input whose frozen size exactly meets the operator limit', async () => { + const input = options(program) + input.policy.maximumInputBytes = input.frames.reduce((bytes, frame) => bytes + Buffer.byteLength(canonicalJson(frame) + '\n'), 0) + const result = await executePackageExperiment(input) + expect(result.frozenInput.bytes).toBe(input.policy.maximumInputBytes) + expect(result.execution.inputBytes).toBeGreaterThan(input.policy.maximumInputBytes) + expect(result.results).toHaveLength(2) + }, 20000) + it('terminates an output flood without buffering it', async () => { + await expect(executePackageExperiment(options('import sys\nsys.stdout.write(\'x\' * 1000000)\nsys.stdout.flush()\n'))).rejects.toThrow('Output transport limit') + }, 20000) + it('terminates a package that never responds', async () => { + const input = options('while True: pass\n') + input.manifest.resources.cpuMs = 1000 + input.request.packageHash = configHash(input.manifest) + await expect(executePackageExperiment(input)).rejects.toThrow('deadline') + }, 20000) + it('stops a cancelled experiment and removes its container', async () => { + const controller = new AbortController() + const timer = setTimeout(() => controller.abort(), 1000) + try { + await expect(executePackageExperiment({ ...options('import time\ntime.sleep(60)\n'), signal: controller.signal })).rejects.toThrow(/abort/i) + } + finally { clearTimeout(timer) } + }, 20000) + it('terminates memory exhaustion', async () => { + await expect(executePackageExperiment(options('data = bytearray(256 * 1024 * 1024)\n'))).rejects.toThrow() + }, 20000) + it('rejects invalid output identities and operator limit excess', async () => { + await expect(executePackageExperiment(options(program.replace('init[\'runHash\']', '\'b\' * 64')))).rejects.toThrow('identity') + const substituted = options(program) + substituted.frames[0].samples.left[0]++ + await expect(executePackageExperiment(substituted)).rejects.toThrow('input identity mismatch') + const input = options(program) + input.policy.maximumMemoryBytes = 1 + await expect(executePackageExperiment(input)).rejects.toThrow('operator resource policy') + }, 20000) +}) diff --git a/scripts/algorithm-experiments/container.ts b/scripts/algorithm-experiments/container.ts new file mode 100644 index 00000000..45d39429 --- /dev/null +++ b/scripts/algorithm-experiments/container.ts @@ -0,0 +1,200 @@ +import { recordExperimentStaging } from './staging-recovery' +import { recoveryLabelArguments } from './recovery' +import { spawn, execFile } from 'node:child_process' +import { existsSync } from 'node:fs' +import { randomUUID } from 'node:crypto' +import { mkdtemp, mkdir, writeFile, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' +import { promisify } from 'node:util' +import { performance } from 'node:perf_hooks' +import { verifyAlgorithmPackage } from '../../src/biometrics/packages/manifest' +import { PackageFrameValidator } from '../../src/biometrics/packages/protocol' +import { dockerResourceReader, startResourceMonitor } from './resources' +import { inspectPythonEnvironment, matchPythonEnvironment, parsePythonEnvironmentLock } from './environment' +import { freezePackageInput, packageTransportCeiling, readFrozenPackageInput } from './input' +import { dataHash } from '../../src/biometrics/algorithms/canonical' + +const exec = promisify(execFile) +async function removeContainer(name: string) { + try { + await exec('docker', ['rm', '-f', name], { timeout: 10000, maxBuffer: 65536 }) + } + catch (error) { + if (!(error instanceof Error) || !error.message.includes('No such container:')) throw error + } +} +/** Operator-owned policy, never accepted from a candidate or pod settings. */ +export interface ExperimentPolicy { + image: string + maximumWallMs: number + maximumMemoryBytes: number + maximumProcesses: number + maximumOutputBytes: number + maximumInputBytes: number +} + +/** Off-pod CLI only. Do not import from the web application or run on a pod. + * The image must be a reviewed Python runtime with locked dependencies already + * installed; dependency installation and network access are not permitted here. */ +export async function executePackageExperiment(options: { + manifest: unknown + files: ReadonlyMap + request: unknown + frames: Iterable + policy: ExperimentPolicy + signal?: AbortSignal +}) { + options.signal?.throwIfAborted() + if (existsSync('/persistent/deviceinfo/dac.sock') || existsSync('/deviceinfo/dac.sock')) throw new Error('Algorithm experiments must run off-pod') + const policy = structuredClone(options.policy) + if (!/^sha256:[a-f0-9]{64}$/.test(policy.image)) throw new Error('A local immutable image ID is required') + for (const value of [policy.maximumWallMs, policy.maximumMemoryBytes, policy.maximumProcesses, policy.maximumOutputBytes, policy.maximumInputBytes]) { + if (!Number.isSafeInteger(value) || value <= 0) throw new Error('Invalid experiment policy') + } + verifyAlgorithmPackage(options.manifest, options.files) + const files = new Map([...options.files].map(([path, bytes]) => [path, Buffer.from(bytes)])) + const verified = verifyAlgorithmPackage(options.manifest, files) + const validator = new PackageFrameValidator(verified.manifest, options.request) + const resources = verified.manifest.resources + if (resources.wallMs > policy.maximumWallMs || resources.memoryBytes > policy.maximumMemoryBytes + || resources.processes > policy.maximumProcesses || resources.outputBytes > policy.maximumOutputBytes) throw new Error('Package exceeds operator resource policy') + if (resources.wallMs > 3600000) throw new Error('Experiment deadline exceeds one hour') + const info = JSON.parse((await exec('docker', ['info', '--format', '{{json .}}'], { timeout: 10000, maxBuffer: 1024 * 1024 })).stdout) + if (info.OSType !== 'linux' || !info.MemoryLimit || !info.PidsLimit || !info.CpuCfsQuota || !info.SecurityOptions?.some((item: string) => item.startsWith('name=seccomp'))) throw new Error('Docker lacks required isolation controls') + const lockBytes = files.get(verified.manifest.dependencyLock) + if (!lockBytes) throw new Error('Missing dependency lock') + parsePythonEnvironmentLock(lockBytes, policy.image) + const environment = matchPythonEnvironment(lockBytes, await inspectPythonEnvironment(policy.image), policy.image) + const root = await mkdtemp(join(tmpdir(), 'sleepypod-package-')) + const name = `sleepypod-experiment-${randomUUID()}` + let created = false + try { + await recordExperimentStaging(root, name, Math.min(resources.wallMs, resources.cpuMs)) + const inputPath = join(root, 'input.jsonl') + const frozenInput = await freezePackageInput(inputPath, options.frames, policy.maximumInputBytes) + if (frozenInput.hash !== validator.request.inputHash) throw new Error('Experiment input identity mismatch') + const initialize = { type: 'initialize', request: validator.request, runHash: validator.runHash } + const transportLimit = packageTransportCeiling(frozenInput, Buffer.byteLength(JSON.stringify(initialize) + '\n')) + const packagePath = join(root, 'package') + await mkdir(packagePath, { mode: 0o755 }) + // Docker mounts only packagePath; its container UID needs access inside that + // mount, not traversal of the host parent. Keep the mkdtemp root at 0700 so + // host users cannot list package source or the private frozen sensor input. + for (const artifact of verified.manifest.artifacts) { + const path = join(packagePath, artifact.path) + await mkdir(dirname(path), { recursive: true, mode: 0o755 }) + // Copy to privately owned staging so caller mutation cannot alter the mount. + await writeFile(path, files.get(artifact.path) as Uint8Array, { flag: 'wx', mode: 0o444 }) + } + // Bound aggregate CPU conservatively: one CPU and a wall deadline no longer + // than the CPU budget. Idle time consumes that budget too. This is not a + // measured CPU receipt and cannot qualify a package's runtime performance. + const deadlineMs = Math.min(resources.wallMs, resources.cpuMs) + options.signal?.throwIfAborted() + created = true + const creation = await exec('docker', ['create', '--name', name, ...recoveryLabelArguments('experiment', deadlineMs), '--network', 'none', '--ipc', 'none', '--read-only', + '--cap-drop', 'ALL', '--security-opt', 'no-new-privileges=true', '--user', '65534:65534', + '--pids-limit', String(resources.processes), '--memory', String(resources.memoryBytes), + '--memory-swap', String(resources.memoryBytes), '--cpus', '1', '--ulimit', 'core=0', + '--log-driver', 'none', '--restart', 'no', '--no-healthcheck', '-i', + '--mount', `type=bind,source=${packagePath},target=/package,readonly`, '--workdir', '/package', + '--entrypoint', 'python', policy.image, '-I', '-B', `/package/${verified.manifest.entrypoint}`], + { timeout: 10000, maxBuffer: 65536 }) + const resourceReader = await dockerResourceReader(creation.stdout.trim()).catch(() => null) + const started = performance.now() + const resourceMonitor = startResourceMonitor(resourceReader, started) + const child = spawn('docker', ['start', '-ai', name], { stdio: ['pipe', 'pipe', 'pipe'] }) + let failure: Error | null = null + let pending = Buffer.alloc(0), outputBytes = 0, inputBytes = 0 + let waiting: { resolve: (line: string) => void, reject: (error: Error) => void } | null = null + let stopPromise: Promise | null = null + const fail = (error: Error) => { + if (failure) return + failure = error + waiting?.reject(error) + waiting = null + child.stdin.destroy() + stopPromise = exec('docker', ['kill', name], { timeout: 10000, maxBuffer: 65536 }).catch(() => undefined) + child.kill('SIGKILL') + } + const abort = () => fail(new Error('Experiment aborted')) + options.signal?.addEventListener('abort', abort, { once: true }) + if (options.signal?.aborted) abort() + child.stdin.on('error', error => fail(error)) + child.on('error', error => fail(error)) + child.stdout.on('data', (chunk: Buffer) => { + if (failure) return + outputBytes += chunk.length + if (outputBytes > resources.outputBytes) return fail(new Error('Output transport limit exceeded')) + pending = Buffer.concat([pending, chunk]) + let newline: number + while ((newline = pending.indexOf(10)) >= 0) { + if (!waiting) return fail(new Error('Unexpected package output')) + const line = pending.subarray(0, newline).toString('utf8') + pending = pending.subarray(newline + 1) + const receiver = waiting + waiting = null + receiver.resolve(line) + } + }) + child.stderr.on('data', (chunk: Buffer) => { + outputBytes += chunk.length + if (outputBytes > resources.outputBytes) fail(new Error('Output transport limit exceeded')) + }) + const completion = new Promise((resolve, reject) => { + child.once('close', (code) => { + if (waiting) fail(new Error('Package exited before responding')) + if (failure) reject(failure) + else if (code !== 0 || pending.length) reject(new Error(`Package exited with invalid output (exit ${code})`)) + else resolve() + }) + }) + // Observe immediately, even if a frame iterator is still pending. + void completion.catch(() => undefined) + const timer = setTimeout(() => fail(new Error('Package deadline exceeded')), deadlineMs) + const send = async (value: unknown) => { + if (failure) throw failure + const bytes = JSON.stringify(value) + '\n' + inputBytes += Buffer.byteLength(bytes) + if (inputBytes > transportLimit) throw new Error('Input transport limit exceeded') + await new Promise((resolve, reject) => child.stdin.write(bytes, error => error ? reject(error) : resolve())) + } + const results = [] + try { + await send(initialize) + for await (const input of readFrozenPackageInput(inputPath)) { + const { frame } = validator.validateFrame(input) + const response = new Promise((resolve, reject) => { + waiting = { resolve, reject } + }) + void response.catch(() => undefined) + await send({ type: 'frame', frame, frameHash: dataHash(frame) }) + const result = validator.validateStep(frame, JSON.parse(await response)).result + results.push(result) + } + await resourceMonitor.sample() + child.stdin.end() + await completion + const measurements = await resourceMonitor.stop() + return { packageHash: verified.packageHash, runHash: validator.runHash, frozenInput, results, + execution: { limits: resources, measurements, environment, image: policy.image, wallMs: performance.now() - started, outputBytes, inputBytes, isolation: 'docker-restricted-v1' as const } } + } + catch (error) { + fail(error instanceof Error ? error : new Error(String(error))) + await completion.catch(() => undefined) + throw error + } + finally { + options.signal?.removeEventListener('abort', abort) + clearTimeout(timer) + await resourceMonitor.stop() + if (stopPromise) await stopPromise + } + } + finally { + // Keep ownership metadata and files if Docker cleanup is unconfirmed. + if (created) await removeContainer(name) + await rm(root, { recursive: true, force: true }) + } +} diff --git a/scripts/algorithm-experiments/determinism.ts b/scripts/algorithm-experiments/determinism.ts new file mode 100644 index 00000000..85b6ff0c --- /dev/null +++ b/scripts/algorithm-experiments/determinism.ts @@ -0,0 +1,28 @@ +import { createHash } from 'node:crypto' +import { canonicalJson } from '../../src/biometrics/algorithms/canonical' +import { verifyAlgorithmPackage } from '../../src/biometrics/packages/manifest' +import { executePackageExperiment } from './container' + +type ExperimentOptions = Parameters[0] +/** Independent containers provide a fresh process and filesystem for every run. + * The trusted caller supplies a fresh iterator over the same retained inputs. */ +export async function checkPackageDeterminism(options: Omit & { + frames: () => Iterable +}, repetitions = 2) { + if (!Number.isInteger(repetitions) || repetitions < 2 || repetitions > 5) throw new Error('Determinism requires 2–5 independent runs') + verifyAlgorithmPackage(options.manifest, options.files) + const stable = { manifest: structuredClone(options.manifest), request: structuredClone(options.request), + policy: structuredClone(options.policy), signal: options.signal, files: new Map([...options.files].map(([path, bytes]) => [path, Buffer.from(bytes)])) } + type Result = Awaited> + const runs: (Pick & { input: Result['frozenInput'], outputHash: string, responseCount: number })[] = [] + for (let index = 0; index < repetitions; index++) { + const result = await executePackageExperiment({ ...stable, frames: options.frames() }) + const hash = createHash('sha256').update('biometric-package-output-v1\n') + for (const frame of result.results) hash.update(canonicalJson(frame) + '\n') + runs.push({ packageHash: result.packageHash, runHash: result.runHash, input: result.frozenInput, + outputHash: hash.digest('hex'), responseCount: result.results.length, execution: result.execution }) + } + const deterministic = runs.every(run => run.runHash === runs[0].runHash && run.outputHash === runs[0].outputHash) + return { schema: 'biometric-package-determinism-v1' as const, deterministic, runs, + qualification: 'requires_resource_noise_holdout_compatibility_checks_and_review' as const } +} diff --git a/scripts/algorithm-experiments/environment.test.ts b/scripts/algorithm-experiments/environment.test.ts new file mode 100644 index 00000000..c258e346 --- /dev/null +++ b/scripts/algorithm-experiments/environment.test.ts @@ -0,0 +1,26 @@ +// @vitest-environment node +import { describe, expect, it } from 'vitest' +import { matchPythonEnvironment, parsePythonEnvironmentLock, pythonEnvironmentLockSchema } from './environment' +const image = 'sha256:' + 'a'.repeat(64) +const lock = { schema: 'biometric-python-environment-v1', image, implementation: 'cpython', pythonVersion: '3.12.13', system: 'Linux', machine: 'aarch64', distributions: [{ name: 'numpy', version: '2.0.0' }, { name: 'scipy', version: '1.0.0' }] } +const bytes = Buffer.from(JSON.stringify(lock)) +describe('locked Python environment', () => { + it('binds the image, interpreter, architecture and complete distribution inventory', () => { + expect(matchPythonEnvironment(bytes, lock, image)).toMatchObject({ image, schema: 'biometric-python-environment-verification-v1' }) + for (const actual of [{ ...lock, image: 'sha256:' + 'b'.repeat(64) }, { ...lock, machine: 'x86_64' }, { ...lock, pythonVersion: '3.13.0' }, + { ...lock, distributions: lock.distributions.slice(0, 1) }, { ...lock, distributions: [{ name: 'numpy', version: '2.1.0' }, lock.distributions[1]] }]) { + expect(() => matchPythonEnvironment(bytes, actual, image)).toThrow('differs') + } + }) + it('rejects ambiguous names, ordering, duplicates and extra lock fields', () => { + for (const invalid of [{ ...lock, distributions: [...lock.distributions].reverse() }, { ...lock, distributions: [lock.distributions[0], lock.distributions[0]] }, + { ...lock, distributions: [{ name: 'Num_Py', version: '2.0.0' }] }, { ...lock, command: 'pip install latest' }]) { + expect(pythonEnvironmentLockSchema.safeParse(invalid).success).toBe(false) + } + }) + it('rejects unpinned images, foreign image locks, invalid encoding and oversized locks', () => { + expect(() => parsePythonEnvironmentLock(bytes, 'python:latest')).toThrow('operator image') + expect(() => parsePythonEnvironmentLock(Buffer.alloc(256 * 1024 + 1), image)).toThrow('byte limit') + expect(() => parsePythonEnvironmentLock(Buffer.from([255]), image)).toThrow() + }) +}) diff --git a/scripts/algorithm-experiments/environment.ts b/scripts/algorithm-experiments/environment.ts new file mode 100644 index 00000000..b15de579 --- /dev/null +++ b/scripts/algorithm-experiments/environment.ts @@ -0,0 +1,65 @@ +import { recoveryLabelArguments } from './recovery' +import { execFile } from 'node:child_process' +import { createHash, randomUUID } from 'node:crypto' +import { promisify } from 'node:util' +import { z } from 'zod' +import { configHash } from '../../src/biometrics/algorithms/canonical' + +const exec = promisify(execFile) +const imageId = z.string().regex(/^sha256:[a-f0-9]{64}$/) +export const pythonEnvironmentLockSchema = z.object({ + schema: z.literal('biometric-python-environment-v1'), image: imageId, + implementation: z.literal('cpython'), pythonVersion: z.string().regex(/^\d+\.\d+\.\d+$/), + system: z.literal('Linux'), machine: z.string().regex(/^[a-zA-Z0-9_+-]{1,40}$/), + distributions: z.array(z.object({ name: z.string().regex(/^[a-z0-9]+(?:-[a-z0-9]+)*$/), version: z.string().min(1).max(100) }).strict()).max(512), +}).strict().superRefine((value, context) => { + const names = value.distributions.map(item => item.name) + if (names.some((name, index) => index > 0 && name <= names[index - 1])) context.addIssue({ code: 'custom', message: 'Distributions must be unique and sorted by normalized name' }) +}) + +const probe = `import importlib.metadata, json, platform, re, sys +packages = sorted([{'name': re.sub(r'[-_.]+', '-', dist.metadata['Name']).lower(), 'version': dist.version} for dist in importlib.metadata.distributions()], key=lambda item: item['name']) +print(json.dumps({'implementation': sys.implementation.name, 'pythonVersion': platform.python_version(), 'system': platform.system(), 'machine': platform.machine(), 'distributions': packages})) +` + +/** Probe an operator-reviewed immutable image, with no candidate code or mounts. + * Captures installed versions and pins all runtime bytes through the image ID. + * This is not a dependency resolver or an approval of the image's contents. */ +export async function inspectPythonEnvironment(image: string) { + imageId.parse(image) + const name = `sleepypod-environment-${randomUUID()}` + try { + await exec('docker', ['create', '--name', name, ...recoveryLabelArguments('environment', 10000), '--network', 'none', '--ipc', 'none', '--read-only', + '--cap-drop', 'ALL', '--security-opt', 'no-new-privileges=true', '--user', '65534:65534', + '--pids-limit', '16', '--memory', '134217728', '--memory-swap', '134217728', '--cpus', '1', + '--ulimit', 'core=0', '--ulimit', 'cpu=5', '--log-driver', 'none', '--restart', 'no', '--no-healthcheck', + '--entrypoint', 'python', image, '-I', '-B', '-c', probe], { timeout: 10000, maxBuffer: 65536 }) + const { stdout } = await exec('docker', ['start', '-a', name], { timeout: 10000, maxBuffer: 256 * 1024 }) + return pythonEnvironmentLockSchema.parse({ ...JSON.parse(stdout), schema: 'biometric-python-environment-v1', image }) + } + finally { + await removeProbe(name) + } +} +async function removeProbe(name: string) { + try { + await exec('docker', ['rm', '-f', name], { timeout: 10000, maxBuffer: 65536 }) + } + catch (error) { + if (!(error instanceof Error) || !error.message.includes('No such container:')) throw error + } +} + +export function parsePythonEnvironmentLock(bytes: Uint8Array, image: string) { + if (bytes.byteLength > 256 * 1024) throw new Error('Environment lock byte limit exceeded') + const lock = pythonEnvironmentLockSchema.parse(JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes))) + if (lock.image !== image) throw new Error('Environment lock does not match operator image') + return lock +} +export function matchPythonEnvironment(bytes: Uint8Array, actual: unknown, image: string) { + const locked = parsePythonEnvironmentLock(bytes, image) + const observed = pythonEnvironmentLockSchema.parse(actual) + if (configHash(locked) !== configHash(observed)) throw new Error('Installed Python environment differs from dependency lock') + return { schema: 'biometric-python-environment-verification-v1' as const, image, + lockHash: createHash('sha256').update(bytes).digest('hex'), environmentHash: configHash(observed) } +} diff --git a/scripts/algorithm-experiments/files.test.ts b/scripts/algorithm-experiments/files.test.ts new file mode 100644 index 00000000..2ede06f8 --- /dev/null +++ b/scripts/algorithm-experiments/files.test.ts @@ -0,0 +1,62 @@ +// @vitest-environment node +import { mkdtemp, readFile, readdir, rm, stat, symlink, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { packageFixture } from '../../src/biometrics/packages/tests/fixture' +import { publishJson, readJsonFile, readJsonLines, readPackageProposal } from './files' +import { runExperimentCli } from './cli' +let root: string +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'package-files-test-')) +}) +afterEach(async () => { + await rm(root, { recursive: true, force: true }) +}) +function proposal() { + const { files, manifest } = packageFixture() + return { schema: 'biometric-package-proposal-v1', manifest, files: [...files].map(([path, bytes]) => ({ path, base64: bytes.toString('base64') })) } +} +describe('experiment CLI file boundary', () => { + it('inspects an inert proposal and creates a complete private report without running its code', async () => { + const path = join(root, 'proposal.json'), report = join(root, 'report.json') + await writeFile(path, JSON.stringify(proposal())) + await runExperimentCli(['inspect', path, report]) + expect(JSON.parse(await readFile(report, 'utf8'))).toMatchObject({ status: 'inert_proposal', manifest: { engine: { id: 'test.hr' } } }) + expect((await stat(report)).mode & 0o777).toBe(0o600) + await expect(publishJson(report, { replaced: true })).rejects.toThrow() + expect(await readdir(root)).toEqual(['proposal.json', 'report.json']) + }) + it('rejects changed bytes, duplicate files and noncanonical base64', async () => { + const original = proposal() + for (const changed of [{ ...original, files: [original.files[0], original.files[0]] }, + { ...original, files: [{ ...original.files[0], base64: 'cGFzcw==' }, original.files[1]] }, + { ...original, files: [{ ...original.files[0], base64: original.files[0].base64 + '\n' }, original.files[1]] }]) { + const path = join(root, 'proposal.json') + await writeFile(path, JSON.stringify(changed)) + expect(() => readPackageProposal(path)).toThrow() + } + }) + it('rejects symlinks, nonfiles and byte limits for JSON and JSONL', async () => { + const path = join(root, 'input'), link = join(root, 'symlink') + await writeFile(path, '{"value":1}\n{"value":2}\n') + await symlink(path, link) + expect([...readJsonLines(path, 100)]).toEqual([{ value: 1 }, { value: 2 }]) + for (const invalid of [root, link]) { + expect(() => readJsonFile(invalid, 100)).toThrow() + expect(() => [...readJsonLines(invalid, 100)]).toThrow() + } + expect(() => [...readJsonLines(path, 5)]).toThrow() + expect(() => [...readJsonLines(path, 100, 2)]).toThrow('line byte limit') + expect(() => readJsonFile(path, 5)).toThrow() + }) + it('rejects malformed UTF-8, blank lines and incorrect CLI arguments', async () => { + const path = join(root, 'input') + await writeFile(path, Buffer.from([255])) + expect(() => readJsonFile(path, 100)).toThrow() + expect(() => [...readJsonLines(path, 100)]).toThrow() + await writeFile(path, '\n') + expect(() => [...readJsonLines(path, 100)]).toThrow('Empty JSON') + await expect(runExperimentCli(['run'])).rejects.toThrow('Usage:') + }) +}) diff --git a/scripts/algorithm-experiments/files.ts b/scripts/algorithm-experiments/files.ts new file mode 100644 index 00000000..c743add1 --- /dev/null +++ b/scripts/algorithm-experiments/files.ts @@ -0,0 +1,99 @@ +import { constants, closeSync, fstatSync, openSync, readSync } from 'node:fs' +import { link, open, unlink } from 'node:fs/promises' +import { randomUUID } from 'node:crypto' +import { z } from 'zod' +import { algorithmPackageManifestSchema, maximumPackageBytes, verifyAlgorithmPackage } from '../../src/biometrics/packages/manifest' + +function openRegular(path: string, maximumBytes: number) { + if (!Number.isSafeInteger(maximumBytes) || maximumBytes <= 0) throw new Error('Invalid file byte limit') + const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK) + try { + const stat = fstatSync(fd) + if (!stat.isFile() || stat.size > maximumBytes) throw new Error('Expected a bounded regular file') + return fd + } + catch (error) { + closeSync(fd) + throw error + } +} + +/** Symlinks, devices and FIFOs are never followed. Size is checked both before + * and during reads so a growing input cannot bypass its transport limit. */ +export function* readJsonLines(path: string, maximumBytes: number, maximumLineBytes = 8 * 1024 * 1024) { + const fd = openRegular(path, maximumBytes) + const chunk = Buffer.alloc(65536) + let total = 0, pending = Buffer.alloc(0) + try { + let count: number + while ((count = readSync(fd, chunk)) !== 0) { + total += count + if (total > maximumBytes) throw new Error('File byte limit exceeded') + pending = Buffer.concat([pending, chunk.subarray(0, count)]) + let newline: number + while ((newline = pending.indexOf(10)) >= 0) { + if (newline > maximumLineBytes) throw new Error('JSON line byte limit exceeded') + const line = pending.subarray(0, newline) + pending = pending.subarray(newline + 1) + if (!line.length) throw new Error('Empty JSON line') + yield JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(line)) as unknown + } + if (pending.length > maximumLineBytes) throw new Error('JSON line byte limit exceeded') + } + if (pending.length) yield JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(pending)) as unknown + } + finally { closeSync(fd) } +} + +export function readJsonFile(path: string, maximumBytes: number): unknown { + const fd = openRegular(path, maximumBytes) + try { + const bytes = Buffer.alloc(maximumBytes + 1) + let offset = 0, count: number + while ((count = readSync(fd, bytes, offset, bytes.length - offset, null)) !== 0) { + offset += count + if (offset > maximumBytes) throw new Error('File byte limit exceeded') + } + return JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes.subarray(0, offset))) + } + finally { closeSync(fd) } +} + +const proposalSchema = z.object({ schema: z.literal('biometric-package-proposal-v1'), manifest: algorithmPackageManifestSchema, + files: z.array(z.object({ path: z.string().min(1).max(240), base64: z.string().max(Math.ceil(maximumPackageBytes / 3) * 4) }).strict()).max(128), +}).strict() + +/** A single inert JSON envelope avoids archive extraction and filesystem paths + * derived from candidate data. Every byte is verified against the manifest. */ +export function readPackageProposal(path: string) { + const proposal = proposalSchema.parse(readJsonFile(path, 48 * 1024 * 1024)) + if (new Set(proposal.files.map(file => file.path)).size !== proposal.files.length) throw new Error('Duplicate proposal file') + const files = new Map() + let total = 0 + for (const file of proposal.files) { + const bytes = Buffer.from(file.base64, 'base64') + if (bytes.toString('base64') !== file.base64) throw new Error('Expected canonical base64') + total += bytes.length + if (total > maximumPackageBytes) throw new Error('Proposal byte limit exceeded') + files.set(file.path, bytes) + } + const verified = verifyAlgorithmPackage(proposal.manifest, files) + return { ...verified, files } +} + +/** Publish only complete JSON, mode 0600, and never replace an existing report. + * Hard-link publication is atomic and refuses existing files or symlinks. */ +export async function publishJson(path: string, value: unknown) { + const staging = `${path}.${randomUUID()}.tmp` + const file = await open(staging, 'wx', 0o600) + try { + await file.writeFile(JSON.stringify(value, null, 2) + '\n') + await file.sync() + await file.close() + await link(staging, path) + } + finally { + await file.close() + await unlink(staging) + } +} diff --git a/scripts/algorithm-experiments/fixtures.test.ts b/scripts/algorithm-experiments/fixtures.test.ts new file mode 100644 index 00000000..b7f4e791 --- /dev/null +++ b/scripts/algorithm-experiments/fixtures.test.ts @@ -0,0 +1,39 @@ +import { fixtureResourceTestPolicy } from './resource-test-policy' +import { describe, expect, it } from 'vitest' +import { assessFixtureReadings, packageFixtureSuiteSchema } from './fixtures' +const numeric = { kind: 'numeric' as const, metric: 'heart_rate', startMs: 1000, endMs: 2000, minimum: 59, maximum: 61, minimumReadings: 2, minimumQuality: 0.8 } +const reading = { timestampMs: 1000, side: 'left' as const, metric: 'heart_rate', value: 60, quality: 1, flags: [] } +describe('artifact/noise fixture scoring', () => { + it('requires coverage and rejects dropping difficult readings', () => { + expect(assessFixtureReadings([numeric], [reading, { ...reading, timestampMs: 1500 }]).passed).toBe(true) + expect(assessFixtureReadings([numeric], [reading]).failures).toContain('insufficient_coverage:heart_rate:0') + expect(assessFixtureReadings([numeric], [{ ...reading, value: null }]).passed).toBe(false) + expect(assessFixtureReadings([numeric], [reading, reading]).passed).toBe(false) + }) + it('rejects out-of-range, low quality, unknown metric and out-of-window readings', () => { + for (const changed of [{ ...reading, value: 120 }, { ...reading, quality: 0.2 }, { ...reading, timestampMs: 2000 }, { ...reading, metric: 'breathing_rate' }]) { + expect(assessFixtureReadings([{ ...numeric, minimumReadings: 1 }], [changed]).passed).toBe(false) + } + }) + it('accepts explicit null or absent output for abstention, but rejects invented vitals', () => { + const expected = { kind: 'abstain' as const, metric: 'heart_rate', startMs: 1000, endMs: 2000 } + expect(assessFixtureReadings([expected], []).passed).toBe(true) + expect(assessFixtureReadings([expected], [{ ...reading, value: null }]).passed).toBe(true) + expect(assessFixtureReadings([expected], [reading]).failures).toContain('failed_abstention:heart_rate:1000') + }) + it('rejects overlapping, vacuous, out-of-input and duplicate fixture definitions', () => { + const fixture = { id: 'clean', category: 'clean_signal', + request: { schema: 'biometric-package-run-v1', packageHash: 'a'.repeat(64), inputHash: 'b'.repeat(64), hardwareId: 'pod4', sampleRateHz: 1, side: 'left', seed: 1, config: {} }, + frames: [{ schema: 'biometric-signal-frame-v1', sequence: 0, timestampMs: 1000, samples: { left: [1], right: null }, context: { occupied: true, pumpActive: false } }], expectations: [numeric] } + const suite = (fixtures: unknown[]) => ({ schema: 'biometric-package-fixtures-v1', resourcePolicy: fixtureResourceTestPolicy, fixtures }) + expect(packageFixtureSuiteSchema.safeParse(suite([fixture])).success).toBe(true) + for (const fixtures of [[fixture, fixture], [{ ...fixture, expectations: [numeric, numeric] }], + [{ ...fixture, expectations: [{ ...numeric, startMs: 0 }] }], [{ ...fixture, category: 'empty_bed' }], + [{ ...fixture, expectations: [{ kind: 'abstain', metric: 'heart_rate', startMs: 1000, endMs: 2000 }] }]]) { + expect(packageFixtureSuiteSchema.safeParse(suite(fixtures)).success).toBe(false) + } + }) + it('rejects empty suites', () => { + expect(packageFixtureSuiteSchema.safeParse({ schema: 'biometric-package-fixtures-v1', resourcePolicy: fixtureResourceTestPolicy, fixtures: [] }).success).toBe(false) + }) +}) diff --git a/scripts/algorithm-experiments/fixtures.ts b/scripts/algorithm-experiments/fixtures.ts new file mode 100644 index 00000000..ca892776 --- /dev/null +++ b/scripts/algorithm-experiments/fixtures.ts @@ -0,0 +1,110 @@ +import { z } from 'zod' +import { dataHash } from '../../src/biometrics/algorithms/canonical' +import { packageRunRequestSchema, signalFrameSchema } from '../../src/biometrics/packages/protocol' +import { verifyAlgorithmPackage } from '../../src/biometrics/packages/manifest' +import { assessPackageResources, packageResourcePolicySchema } from './resource-assessment' +import { packageInputHash } from './input' +import { executePackageExperiment } from './container' + +const time = z.number().finite().nonnegative().max(Number.MAX_SAFE_INTEGER) +const common = { metric: z.string().min(1).max(100), startMs: time, endMs: time } +const expectationSchema = z.discriminatedUnion('kind', [ + z.object({ ...common, kind: z.literal('numeric'), minimum: z.number().finite(), maximum: z.number().finite(), + minimumReadings: z.number().int().positive().max(20000), minimumQuality: z.number().min(0).max(1) }).strict(), + z.object({ ...common, kind: z.literal('abstain') }).strict(), +]) +export const fixtureCategories = ['clean_signal', 'pump', 'empty_bed', 'motion', 'cross_side', 'harmonics', 'gaps', 'warmup', 'boundary_reset'] as const +const fixtureSchema = z.object({ + id: z.string().regex(/^[a-z0-9][a-z0-9_-]{0,99}$/), category: z.enum(fixtureCategories), + request: packageRunRequestSchema, frames: z.array(signalFrameSchema).min(1).max(256), + expectations: z.array(expectationSchema).min(1).max(256), +}).strict().superRefine((fixture, context) => { + if (!fixture.frames.length) return + const firstMs = fixture.frames[0].timestampMs + const last = fixture.frames[fixture.frames.length - 1] + const endMs = last.timestampMs + (last.samples[fixture.request.side]?.length ?? 0) * 1000 / fixture.request.sampleRateHz + if (fixture.category === 'clean_signal' && !fixture.expectations.some(expected => expected.kind === 'numeric')) context.addIssue({ code: 'custom', message: 'Clean-signal fixture requires numeric coverage' }) + if (fixture.category === 'empty_bed' && fixture.expectations.some(expected => expected.kind !== 'abstain')) context.addIssue({ code: 'custom', message: 'Empty-bed fixture must require abstention' }) + for (let index = 0; index < fixture.expectations.length; index++) { + const expected = fixture.expectations[index] + if (expected.startMs < firstMs || expected.endMs > endMs) context.addIssue({ code: 'custom', message: 'Expectation outside fixture input interval' }) + if (expected.startMs >= expected.endMs || (expected.kind === 'numeric' && expected.minimum > expected.maximum)) { + context.addIssue({ code: 'custom', message: 'Invalid fixture expectation range' }) + } + if (fixture.expectations.slice(0, index).some(other => other.metric === expected.metric && other.startMs < expected.endMs && expected.startMs < other.endMs)) { + context.addIssue({ code: 'custom', message: 'Overlapping fixture expectations' }) + } + } +}) +export const packageFixtureSuiteSchema = z.object({ schema: z.literal('biometric-package-fixtures-v1'), resourcePolicy: packageResourcePolicySchema, + fixtures: z.array(fixtureSchema).min(1).max(64), +}).strict().refine(suite => new Set(suite.fixtures.map(fixture => fixture.id)).size === suite.fixtures.length, 'Duplicate fixture identity') + +type Expectation = z.infer +type Reading = Awaited>['results'][number]['readings'][number] +/** Expectations come from reviewed fixtures, never from package responses. + * Every returned reading must belong to a declared expectation window. */ +export function assessFixtureReadings(expectations: Expectation[], readings: Reading[]) { + const failures: string[] = [] + const counts = expectations.map(() => 0) + const identities = new Set() + for (const reading of readings) { + const identity = `${reading.metric}:${reading.timestampMs}` + if (identities.has(identity)) failures.push(`duplicate_reading:${identity}`) + identities.add(identity) + const index = expectations.findIndex(expected => expected.metric === reading.metric && expected.startMs <= reading.timestampMs && reading.timestampMs < expected.endMs) + if (index < 0) { + failures.push(`unexpected_reading:${identity}`) + continue + } + const expected = expectations[index] + if (expected.kind === 'abstain') { + if (reading.value !== null) failures.push(`failed_abstention:${identity}`) + } + else if (reading.value !== null) { + if (reading.value < expected.minimum || reading.value > expected.maximum || reading.quality < expected.minimumQuality) failures.push(`out_of_range_or_low_quality:${identity}`) + else counts[index]++ + } + } + expectations.forEach((expected, index) => { + if (expected.kind === 'numeric' && counts[index] < expected.minimumReadings) failures.push(`insufficient_coverage:${expected.metric}:${index}`) + }) + return { passed: failures.length === 0, failures, usableReadings: counts } +} + +/** This is one qualification dimension, not a held-out improvement claim. + * Fresh containers also prevent state leaking between independent fixtures. */ +export async function runPackageFixtures(options: Pick[0], 'manifest' | 'files' | 'policy' | 'signal'>, suiteInput: unknown) { + const suite = packageFixtureSuiteSchema.parse(structuredClone(suiteInput)) + const suiteHash = dataHash(suite) + const verified = verifyAlgorithmPackage(options.manifest, options.files) + const stable = { manifest: verified.manifest, files: new Map([...options.files].map(([path, bytes]) => [path, Buffer.from(bytes)])), policy: structuredClone(options.policy), signal: options.signal } + for (const fixture of suite.fixtures) { + for (const expected of fixture.expectations) { + if (!verified.manifest.metrics.some(metric => metric.id === expected.metric)) throw new Error('Fixture expects an undeclared metric') + } + if (fixture.request.packageHash !== verified.packageHash || fixture.request.inputHash !== packageInputHash(fixture.frames)) throw new Error('Fixture input/package identity mismatch') + } + const results = [] + for (const fixture of suite.fixtures) { + options.signal?.throwIfAborted() + const identity = { fixtureId: fixture.id, category: fixture.category, fixtureHash: dataHash(fixture) } + try { + const execution = await executePackageExperiment({ ...stable, request: fixture.request, frames: fixture.frames }) + const assessment = assessFixtureReadings(fixture.expectations, execution.results.flatMap(frame => frame.readings)) + results.push({ ...identity, assessment, resourceAssessment: assessPackageResources(execution.execution, suite.resourcePolicy), execution, error: null }) + } + catch (error) { + options.signal?.throwIfAborted() + results.push({ ...identity, assessment: { passed: false, failures: ['execution_failed'], usableReadings: [] }, resourceAssessment: null, execution: null, + error: error instanceof Error ? error.message.slice(0, 1000) : 'Experiment failed' }) + } + } + const covered = new Set(suite.fixtures.map(fixture => fixture.category)) + const missingCategories = fixtureCategories.filter(category => !covered.has(category)) + return { schema: 'biometric-package-fixture-report-v1' as const, packageHash: verified.packageHash, suiteHash, suite, + status: results.some(result => !result.assessment.passed || result.resourceAssessment?.status === 'failed') + ? 'failed' + : missingCategories.length || results.some(result => result.resourceAssessment?.status !== 'within_observed_limits') ? 'incomplete' : 'fixture_checks_passed', + missingCategories, results, promotion: 'requires_determinism_resource_holdout_compatibility_and_release_review' as const } +} diff --git a/scripts/algorithm-experiments/input.test.ts b/scripts/algorithm-experiments/input.test.ts new file mode 100644 index 00000000..2a475c9c --- /dev/null +++ b/scripts/algorithm-experiments/input.test.ts @@ -0,0 +1,56 @@ +// @vitest-environment node +import { mkdtemp, rm } from 'node:fs/promises' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { describe, expect, it } from 'vitest' +import { dataHash } from '../../src/biometrics/algorithms/canonical' +import { signalFrameSchema } from '../../src/biometrics/packages/protocol' +import { freezePackageInput, packageInputHash, packageTransportCeiling, readFrozenPackageInput } from './input' + +const frame = { schema: 'biometric-signal-frame-v1', sequence: 0, timestampMs: 1000, + samples: { left: [1, 2, 3], right: null }, context: { occupied: null, pumpActive: null } } + +describe('frozen experiment input', () => { + it('binds sample bytes, context, ordering and timestamps to a streamed identity', () => { + const original = packageInputHash([frame]) + for (const changed of [{ ...frame, timestampMs: 1001 }, { ...frame, context: { occupied: true, pumpActive: null } }, + { ...frame, samples: { left: [1, 2, 4], right: null } }]) expect(packageInputHash([changed])).not.toBe(original) + expect(packageInputHash([frame, { ...frame, sequence: 1 }])).not.toBe(packageInputHash([{ ...frame, sequence: 1 }, frame])) + expect(packageInputHash([{ context: frame.context, samples: frame.samples, timestampMs: 1000, sequence: 0, schema: frame.schema }])).toBe(original) + }) + it('reads the frozen copy after source mutation and enforces storage bounds', async () => { + const root = await mkdtemp(join(tmpdir(), 'package-input-test-')) + try { + const source = structuredClone(frame) + const next = { ...frame, sequence: 1, timestampMs: 1006 } + const frozen = await freezePackageInput(join(root, 'input'), [source, next], 10000) + source.samples.left[0] = 999 + const collected = [] + for await (const value of readFrozenPackageInput(join(root, 'input'))) collected.push(value) + expect(collected).toEqual([frame, next]) + expect(frozen).toMatchObject({ hash: packageInputHash([frame, next]), frameCount: 2 }) + await expect(freezePackageInput(join(root, 'input'), [frame], 10000)).rejects.toThrow() + await expect(freezePackageInput(join(root, 'small'), [frame], 10)).rejects.toThrow('byte limit') + await expect(freezePackageInput(join(root, 'empty'), [], 10000)).rejects.toThrow('at least one') + } + finally { await rm(root, { recursive: true, force: true }) } + }) + it('derives a transport ceiling that covers the initialize line and every frame envelope of a frozen input', async () => { + const root = await mkdtemp(join(tmpdir(), 'package-input-test-')) + try { + const frames = [frame, { ...frame, sequence: 1, timestampMs: 1006, samples: { left: [0.1, 1e21, 5e-324], right: null } }] + const frozen = await freezePackageInput(join(root, 'input'), frames, 100000) + const initialize = JSON.stringify({ type: 'initialize', request: {}, runHash: 'a'.repeat(64) }) + '\n' + let transport = Buffer.byteLength(initialize) + for await (const input of readFrozenPackageInput(join(root, 'input'))) { + const parsed = signalFrameSchema.parse(input) + transport += Buffer.byteLength(JSON.stringify({ type: 'frame', frame: parsed, frameHash: dataHash(parsed) }) + '\n') + } + const ceiling = packageTransportCeiling(frozen, Buffer.byteLength(initialize)) + expect(ceiling).toBeGreaterThanOrEqual(transport) + expect(packageTransportCeiling(await freezePackageInput(join(root, 'integers'), [frame], 100000), Buffer.byteLength(initialize))) + .toBe(Buffer.byteLength(initialize) + Buffer.byteLength(JSON.stringify({ type: 'frame', frame, frameHash: dataHash(frame) }) + '\n')) + } + finally { await rm(root, { recursive: true, force: true }) } + }) +}) diff --git a/scripts/algorithm-experiments/input.ts b/scripts/algorithm-experiments/input.ts new file mode 100644 index 00000000..322c659b --- /dev/null +++ b/scripts/algorithm-experiments/input.ts @@ -0,0 +1,59 @@ +import { createHash } from 'node:crypto' +import { open } from 'node:fs/promises' +import { createReadStream } from 'node:fs' +import { createInterface } from 'node:readline' +import { canonicalJson } from '../../src/biometrics/algorithms/canonical' +import { signalFrameSchema } from '../../src/biometrics/packages/protocol' + +const domain = 'biometric-package-input-v1\n' +/** Streaming identity; canonical frame JSON followed by LF, prefixed by domain. + * These are actual experiment frames, not a user-supplied dataset name/hash. */ +export function packageInputHash(frames: Iterable) { + const hash = createHash('sha256').update(domain) + for (const frame of frames) hash.update(canonicalJson(signalFrameSchema.parse(frame)) + '\n') + return hash.digest('hex') +} + +/** Freeze bounded input before any candidate starts. The owned file is never + * mounted into the container: the executor sends one frame at a time. */ +export async function freezePackageInput(path: string, frames: Iterable, maximumBytes: number) { + if (!Number.isSafeInteger(maximumBytes) || maximumBytes <= 0) throw new Error('Invalid input byte limit') + const file = await open(path, 'wx', 0o600) + let bytes = 0, frameCount = 0 + const hash = createHash('sha256').update(domain) + try { + for (const frame of frames) { + const encoded = canonicalJson(signalFrameSchema.parse(frame)) + '\n' + bytes += Buffer.byteLength(encoded) + if (bytes > maximumBytes) throw new Error('Frozen input byte limit exceeded') + hash.update(encoded) + await file.writeFile(encoded) + frameCount++ + } + if (!frameCount) throw new Error('Experiment requires at least one frame') + } + finally { + await file.close() + } + return { schema: 'biometric-package-input-v1' as const, hash: hash.digest('hex'), bytes, frameCount } +} + +/** Transport bytes the executor needs to stream a frozen input: the initialize + * line plus each canonical frame line wrapped in its frame envelope. Canonical + * number expansion is never shorter than JSON.stringify, so this is a ceiling. */ +export function packageTransportCeiling(frozen: { bytes: number, frameCount: number }, initializeBytes: number) { + return initializeBytes + frozen.bytes + frozen.frameCount * frameEnvelopeOverhead +} +const frameEnvelopeOverhead = Buffer.byteLength(JSON.stringify({ type: 'frame', frame: 0, frameHash: '0'.repeat(64) })) - 1 + +export async function* readFrozenPackageInput(path: string) { + const stream = createReadStream(path) + const lines = createInterface({ input: stream, crlfDelay: Infinity }) + try { + for await (const line of lines) yield JSON.parse(line) as unknown + } + finally { + lines.close() + stream.destroy() + } +} diff --git a/scripts/algorithm-experiments/ledger.test.ts b/scripts/algorithm-experiments/ledger.test.ts new file mode 100644 index 00000000..176cece3 --- /dev/null +++ b/scripts/algorithm-experiments/ledger.test.ts @@ -0,0 +1,142 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { mkdtempSync, rmSync } from 'node:fs' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { PackageExperimentLedger, packageStudySchema } from './ledger' +const start = 1791673200000 +function study() { + return { schema: 'biometric-package-study-v1', subjectId: 'fixture', side: 'left', protocolHash: 'a'.repeat(64), nights: [ + { id: 'night', startMs: start, endMs: start + 28800000, inputHash: 'b'.repeat(64), referenceHash: 'c'.repeat(64), split: 'holdout' }, + ] } +} +function trial(studyId: string, split = 'holdout') { + return { studyId, split, variants: ['default', 'current', 'candidate'].map(label => ({ label, packageHash: 'd'.repeat(64), configHash: 'e'.repeat(64), image: 'sha256:' + 'f'.repeat(64) })) } +} +let db: Database.Database, ledger: PackageExperimentLedger, now: number +beforeEach(() => { + db = new Database(':memory:') + now = 1000 + ledger = new PackageExperimentLedger(db, () => now) +}) +afterEach(() => db.close()) +describe('dedicated package experiment ledger', () => { + it('freezes studies and reserves exactly one trial per unchanged request key', () => { + const first = ledger.createStudy(study()) + expect(ledger.createStudy(study()).id).toBe(first.id) + const id = ledger.reserveTrial(trial(first.id), 'request') + expect(ledger.reserveTrial(trial(first.id), 'request')).toBe(id) + const changed = trial(first.id) + changed.variants[2].configHash = '0'.repeat(64) + expect(() => ledger.reserveTrial(changed, 'request')).toThrow('idempotency') + expect(ledger.currentEvidence(id).uniqueHoldoutAttempt).toBe(false) + }) + it('records exposure before execution and fences expired workers and their renewals', () => { + const id = ledger.reserveTrial(trial(ledger.createStudy(study()).id), 'request') + const first = ledger.claim(id, 1000) + expect(first.unexposedHoldoutAtClaim).toBe(true) + expect(() => ledger.claim(id, 1000)).toThrow('leased') + now = 2000 + const retry = ledger.claim(id, 1000) + expect(retry).toMatchObject({ unexposedHoldoutAtClaim: false, priorExposures: [{ nightId: 'night', count: 1 }] }) + expect(() => ledger.finish(first, 'succeeded', '1'.repeat(64))).toThrow('Stale') + expect(() => ledger.renew(first, 1000)).toThrow('Stale') + ledger.renew(retry, 2000) + now = 3500 + ledger.finish(retry, 'succeeded', '2'.repeat(64)) + expect(ledger.currentEvidence(id)).toMatchObject({ reportHash: '2'.repeat(64), uniqueHoldoutAttempt: false }) + expect(() => ledger.claim(id, 1000)).toThrow('completed') + }) + it('detects overlap across changed studies, hashes and splits, including failed attempts', () => { + const tuning = study() + tuning.nights[0].split = 'tuning' + const tuningId = ledger.reserveTrial(trial(ledger.createStudy(tuning).id, 'tuning'), 'tuning') + ledger.finish(ledger.claim(tuningId, 1000), 'failed', '1'.repeat(64)) + const changed = study() + changed.nights[0] = { ...changed.nights[0], id: 'renamed', startMs: start + 1000, inputHash: '0'.repeat(64), referenceHash: '1'.repeat(64) } + const next = ledger.reserveTrial(trial(ledger.createStudy(changed).id), 'new') + expect(ledger.claim(next, 1000)).toMatchObject({ unexposedHoldoutAtClaim: false, priorExposures: [{ nightId: 'renamed', count: 1 }] }) + }) + it('invalidates a previous first-use claim when later overlapping trials run', () => { + const studyId = ledger.createStudy(study()).id + const first = ledger.reserveTrial(trial(studyId), 'first') + ledger.finish(ledger.claim(first, 1000), 'succeeded', '1'.repeat(64)) + expect(ledger.currentEvidence(first).uniqueHoldoutAttempt).toBe(true) + const next = ledger.reserveTrial(trial(studyId), 'next') + ledger.claim(next, 1000) + expect(ledger.currentEvidence(first).uniqueHoldoutAttempt).toBe(false) + expect(new PackageExperimentLedger(db, () => now).currentEvidence(first).otherExposures[0].count).toBe(1) + }) + it('preserves independent sides and disjoint nights', () => { + const original = ledger.reserveTrial(trial(ledger.createStudy(study()).id), 'original') + ledger.claim(original, 1000) + for (const [key, changed] of [['right', { ...study(), side: 'right' }], ['later', { ...study(), nights: study().nights.map(night => ({ ...night, startMs: start + 86400000, endMs: start + 86400000 + 28800000 })) }]] as const) { + const id = ledger.reserveTrial(trial(ledger.createStudy(changed).id), key) + expect(ledger.claim(id, 1000).unexposedHoldoutAtClaim).toBe(true) + } + }) + it('rejects nonchronological and overlapping split plans', () => { + const value = study() + const later = { ...value.nights[0], id: 'later', startMs: start + 86400000, endMs: start + 86400000 + 28800000, split: 'tuning' } + expect(packageStudySchema.safeParse({ ...value, nights: [...value.nights, later] }).success).toBe(false) + expect(packageStudySchema.safeParse({ ...value, nights: [...value.nights, { ...value.nights[0], id: 'overlap' }] }).success).toBe(false) + }) + it('persists exposure and fences competing database connections across reopening', () => { + const directory = mkdtempSync(join(tmpdir(), 'package-ledger-test-')) + const path = join(directory, 'experiments.db') + const firstDb = new Database(path), secondDb = new Database(path) + try { + const first = new PackageExperimentLedger(firstDb, () => now), second = new PackageExperimentLedger(secondDb, () => now) + const studyId = first.createStudy(study()).id + const trialId = first.reserveTrial(trial(studyId), 'shared') + expect(second.reserveTrial(trial(studyId), 'shared')).toBe(trialId) + const lease = first.claim(trialId, 1000) + expect(() => second.claim(trialId, 1000)).toThrow('leased') + now += 1000 + const replacement = second.claim(trialId, 1000) + expect(() => first.finish(lease, 'succeeded', '1'.repeat(64))).toThrow('Stale') + second.finish(replacement, 'succeeded', '2'.repeat(64)) + firstDb.close() + secondDb.close() + const reopened = new Database(path) + try { + expect(new PackageExperimentLedger(reopened).currentEvidence(trialId)).toMatchObject({ uniqueHoldoutAttempt: false, reportHash: '2'.repeat(64) }) + } + finally { reopened.close() } + } + finally { + if (firstDb.open) firstDb.close() + if (secondDb.open) secondDb.close() + rmSync(directory, { recursive: true, force: true }) + } + }) + it('resumes interrupted initial migration branding and rejects unknown future versions', () => { + db.pragma('user_version = 0') + expect(() => new PackageExperimentLedger(db)).not.toThrow() + db.pragma('user_version = 2') + expect(() => new PackageExperimentLedger(db)).toThrow('Unsupported') + expect(db.pragma('user_version', { simple: true })).toBe(2) + }) + it('refuses another application database without changing its schema or data', () => { + const other = new Database(':memory:') + try { + other.exec('CREATE TABLE vitals(value INTEGER); INSERT INTO vitals VALUES (60)') + expect(() => new PackageExperimentLedger(other)).toThrow('non-experiment') + expect(other.prepare('SELECT value FROM vitals').get()).toEqual({ value: 60 }) + expect(other.pragma('application_id', { simple: true })).toBe(0) + } + finally { other.close() } + }) +}) + +it('allows a failed attempt without a report while refusing success without evidence', () => { + const id = ledger.reserveTrial(trial(ledger.createStudy(study()).id), 'missing-report') + const lease = ledger.claim(id, 1000) + expect(() => ledger.finish(lease, 'succeeded', null)).toThrow() + expect(ledger.trial(id).state).toBe('running') + ledger.finish(lease, 'failed', null) + expect(ledger.trial(id)).toMatchObject({ state: 'failed', report_hash: null, lease_until: null }) + expect(ledger.claim(id, 1000).unexposedHoldoutAtClaim).toBe(false) + expect(() => ledger.finish(lease, 'failed', null)).toThrow('Stale') +}) diff --git a/scripts/algorithm-experiments/ledger.ts b/scripts/algorithm-experiments/ledger.ts new file mode 100644 index 00000000..39e9dc29 --- /dev/null +++ b/scripts/algorithm-experiments/ledger.ts @@ -0,0 +1,140 @@ +import type Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { fileURLToPath } from 'node:url' +import { randomUUID } from 'node:crypto' +import { z } from 'zod' +import { configHash, dataHash } from '../../src/biometrics/algorithms/canonical' +import { timestampMsSchema } from '../../src/biometrics/references/contracts' + +const hash = z.string().regex(/^[a-f0-9]{64}$/) +const key = z.string().min(1).max(200) +const split = z.enum(['tuning', 'validation', 'holdout']) +export const packageStudySchema = z.object({ schema: z.literal('biometric-package-study-v1'), subjectId: key, side: z.enum(['left', 'right']), + protocolHash: hash, nights: z.array(z.object({ id: key, startMs: timestampMsSchema, endMs: timestampMsSchema, inputHash: hash, referenceHash: hash, split }).strict()).min(1).max(60), +}).strict().superRefine((study, context) => { + const order = { tuning: 0, validation: 1, holdout: 2 } + if (new Set(study.nights.map(night => night.id)).size !== study.nights.length) context.addIssue({ code: 'custom', message: 'Duplicate night identity' }) + const sorted = [...study.nights].sort((a, b) => a.startMs - b.startMs) + for (const [index, night] of sorted.entries()) { + if (night.endMs <= night.startMs || night.endMs - night.startMs > 86400000) context.addIssue({ code: 'custom', message: 'Invalid whole-night interval' }) + if (index > 0 && (sorted[index - 1].endMs > night.startMs || order[sorted[index - 1].split] > order[night.split])) context.addIssue({ code: 'custom', message: 'Nights must be disjoint chronological splits' }) + } +}) +const trialRequestSchema = z.object({ studyId: z.string().uuid(), split, + variants: z.array(z.object({ label: z.enum(['default', 'current', 'candidate']), packageHash: hash, configHash: hash, image: z.string().regex(/^sha256:[a-f0-9]{64}$/) }).strict()).length(3), +}).strict().refine(request => new Set(request.variants.map(variant => variant.label)).size === 3, 'All three variants are required') +interface StudyRow { id: string, hash: string, payload: string } +interface TrialRow { id: string, request_hash: string, request: string, state: 'pending' | 'running' | 'succeeded' | 'failed', attempt_id: string | null, lease_until: number | null, report_hash: string | null } +export interface PackageTrialLease { trialId: string, attemptId: string } +const applicationId = 0x53504558 + +/** Dedicated off-pod metadata database, never the pod's configuration or vitals + * database. Retain it across experiments: a fresh ledger cannot prove that a + * night was never observed elsewhere. Publication is fenced by attempt ID. */ +export class PackageExperimentLedger { + constructor(private readonly db: Database.Database, private readonly now = Date.now) { + const id = db.pragma('application_id', { simple: true }) + const version = db.pragma('user_version', { simple: true }) + if (id !== applicationId) { + const tables = db.prepare('SELECT name FROM sqlite_master WHERE name NOT LIKE \'sqlite_%\'').all() + if (id !== 0 || version !== 0 || tables.length) throw new Error('Refusing a non-experiment database') + db.pragma(`application_id = ${applicationId}`) + } + else if (version !== 0 && version !== 1) throw new Error('Unsupported experiment ledger version') + migrate(drizzle(db), { migrationsFolder: fileURLToPath(new URL('../../src/db/package-experiment-migrations', import.meta.url)) }) + db.pragma('user_version = 1') + } + + createStudy(input: unknown) { + const study = packageStudySchema.parse(input) + const hash = dataHash(study) + return this.db.transaction(() => { + const existing = this.db.prepare('SELECT * FROM package_studies WHERE hash = ?').get(hash) as StudyRow | undefined + if (existing) return { id: existing.id, hash, study } + const id = randomUUID() + this.db.prepare('INSERT INTO package_studies VALUES (?, ?, ?)').run(id, hash, JSON.stringify(study)) + return { id, hash, study } + }).immediate() + } + + reserveTrial(input: unknown, idempotencyKey: string) { + const request = trialRequestSchema.parse(input) + key.parse(idempotencyKey) + const requestHash = configHash(request) + const study = this.study(request.studyId) + if (!study.nights.some(night => night.split === request.split)) throw new Error('Selected split has no nights') + return this.db.transaction(() => { + const previous = this.db.prepare('SELECT * FROM package_trials WHERE idempotency_key = ?').get(idempotencyKey) as TrialRow | undefined + if (previous) { + if (previous.request_hash !== requestHash) throw new Error('Trial idempotency conflict') + return previous.id + } + const id = randomUUID() + this.db.prepare('INSERT INTO package_trials(id, idempotency_key, request_hash, request, state) VALUES (?, ?, ?, ?, \'pending\')').run(id, idempotencyKey, requestHash, JSON.stringify(request)) + return id + }).immediate() + } + + study(id: string) { + const row = this.db.prepare('SELECT * FROM package_studies WHERE id = ?').get(id) as StudyRow | undefined + if (!row) throw new Error('Unknown experiment study') + const study = packageStudySchema.parse(JSON.parse(row.payload)) + if (dataHash(study) !== row.hash) throw new Error('Study integrity mismatch') + return study + } + + trial(id: string) { + const row = this.db.prepare('SELECT * FROM package_trials WHERE id = ?').get(id) as TrialRow | undefined + if (!row) throw new Error('Unknown package trial') + const request = trialRequestSchema.parse(JSON.parse(row.request)) + if (configHash(request) !== row.request_hash) throw new Error('Trial integrity mismatch') + return { ...row, request } + } + + /** Exposure is committed before opening any selected input. Failed or expired + * attempts remain exposures; changing study/input/reference hashes cannot + * erase prior use of an overlapping night for the same subject and side. */ + claim(trialId: string, leaseMs: number) { + z.number().int().min(1000).max(3600000).parse(leaseMs) + return this.db.transaction(() => { + const trial = this.trial(trialId), now = this.now() + if (trial.state === 'succeeded') throw new Error('Trial already completed; reuse its report') + if (trial.state === 'running' && (trial.lease_until ?? 0) > now) throw new Error('Trial already leased') + const study = this.study(trial.request.studyId) + const nights = study.nights.filter(night => night.split === trial.request.split) + const priorExposures = nights.map(night => ({ nightId: night.id, count: (this.db.prepare(`SELECT COUNT(DISTINCT attempt_id) AS count FROM package_exposures + WHERE subject_id = ? AND side = ? AND start_ms < ? AND end_ms > ?`).get(study.subjectId, study.side, night.endMs, night.startMs) as { count: number }).count })) + const attemptId = randomUUID() + for (const night of nights) this.db.prepare('INSERT INTO package_exposures VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)').run(attemptId, trialId, night.id, study.subjectId, study.side, night.startMs, night.endMs, trial.request.split, now) + this.db.prepare('UPDATE package_trials SET state = \'running\', attempt_id = ?, lease_until = ?, report_hash = NULL WHERE id = ?').run(attemptId, now + leaseMs, trialId) + return { trialId, attemptId, priorExposures, unexposedHoldoutAtClaim: trial.request.split === 'holdout' && priorExposures.every(night => night.count === 0) } + }).immediate() + } + + /** Recomputed when evidence is reviewed; never rely on the claim-time flag + * after other experiments have accessed overlapping nights. */ + currentEvidence(trialId: string) { + return this.db.transaction(() => { + const trial = this.trial(trialId) + const study = this.study(trial.request.studyId) + const otherExposures = study.nights.filter(night => night.split === trial.request.split).map(night => ({ nightId: night.id, + count: (this.db.prepare(`SELECT COUNT(DISTINCT attempt_id) AS count FROM package_exposures + WHERE subject_id = ? AND side = ? AND start_ms < ? AND end_ms > ? AND attempt_id != ?`).get(study.subjectId, study.side, night.endMs, night.startMs, trial.attempt_id ?? '') as { count: number }).count })) + return { trialId, state: trial.state, reportHash: trial.report_hash, otherExposures, + uniqueHoldoutAttempt: trial.state === 'succeeded' && trial.request.split === 'holdout' && otherExposures.every(night => night.count === 0) } + }).deferred() + } + + renew(lease: PackageTrialLease, leaseMs: number) { + z.number().int().min(1000).max(3600000).parse(leaseMs) + const now = this.now() + if (this.db.prepare('UPDATE package_trials SET lease_until = ? WHERE id = ? AND attempt_id = ? AND state = \'running\' AND lease_until > ?').run(now + leaseMs, lease.trialId, lease.attemptId, now).changes !== 1) throw new Error('Stale trial lease') + } + + finish(lease: PackageTrialLease, status: 'succeeded' | 'failed', reportHash: string | null) { + z.enum(['succeeded', 'failed']).parse(status) + if (status === 'succeeded' || reportHash !== null) hash.parse(reportHash) + if (this.db.prepare('UPDATE package_trials SET state = ?, report_hash = ?, lease_until = NULL WHERE id = ? AND attempt_id = ? AND state = \'running\' AND lease_until > ?').run(status, reportHash, lease.trialId, lease.attemptId, this.now()).changes !== 1) throw new Error('Stale trial lease') + } +} diff --git a/scripts/algorithm-experiments/managed-evidence.ts b/scripts/algorithm-experiments/managed-evidence.ts new file mode 100644 index 00000000..7116d854 --- /dev/null +++ b/scripts/algorithm-experiments/managed-evidence.ts @@ -0,0 +1,100 @@ +import { verifyPreflightEvidence } from './preflight-evidence' +import { createHash } from 'node:crypto' +import { join } from 'node:path' +import { z } from 'zod' +import { configHash, dataHash } from '../../src/biometrics/algorithms/canonical' +import { createNightUncertainty } from '../../src/biometrics/backtest/uncertainty-core' +import { previewReferenceImport } from '../../src/biometrics/references/normalize' +import { comparePackageOutputs, packageComparisonInputSchema } from './comparison' +import { readJsonFile } from './files' +import { packageStudySchema, type PackageExperimentLedger } from './ledger' +import { managedProtocolSchema } from './managed-protocol' +import { assessPackageResources } from './resource-assessment' + +const hash = z.string().regex(/^[a-f0-9]{64}$/) +const summarySchema = z.object({ nightId: z.string(), reportHash: hash, comparison: z.unknown(), resources: z.unknown() }).strict() +const trialSchema = z.object({ schema: z.literal('biometric-managed-trial-v1'), trialId: z.string().uuid(), attemptId: z.string().uuid(), + study: packageStudySchema, protocol: managedProtocolSchema, request: z.unknown(), nights: z.array(summarySchema).min(1).max(60), + preflight: z.object({ reportHash: hash, status: z.enum(['failed', 'incomplete', 'external_checks_passed']) }).strict().optional(), + uncertainty: z.unknown(), exposureAtClaim: z.unknown(), + interpretation: z.literal('requires_current_ledger_exposure_check_and_full_qualification_review'), +}).strict() +const outputSchema = packageComparisonInputSchema.shape.outputs.element +const nightSchema = z.object({ schema: z.literal('biometric-managed-night-v1'), trialId: z.string().uuid(), attemptId: z.string().uuid(), + night: packageStudySchema.shape.nights.element, referenceImport: packageComparisonInputSchema.shape.referenceImport, + outputs: z.array(outputSchema.extend({ result: outputSchema.shape.result.extend({ execution: z.object({ + image: z.string(), limits: z.unknown(), measurements: z.unknown(), wallMs: z.unknown(), + }).passthrough() }).passthrough() })).length(3), comparison: z.unknown(), resources: z.unknown(), +}).strict() +const uncertainty = createNightUncertainty(text => createHash('sha256').update(text).digest(), 'equal_night_mean_common_mae_difference') +function retained(directory: string, identity: string) { + hash.parse(identity) + const value = readJsonFile(join(directory, `${identity}.json`), 32 * 1024 * 1024) + if (dataHash(value) !== identity) throw new Error('Retained trial report integrity mismatch') + return value +} +function same(actual: unknown, expected: unknown, description: string) { + if (dataHash(actual) !== dataHash(expected)) throw new Error(`Retained ${description} integrity mismatch`) +} + +/** Verifies the ledger-selected report graph without reopening night inputs or + * executing candidate code. Derived scores are recomputed, never accepted from + * the summary. This is an evidence snapshot, not an installation approval. */ +export function readManagedEvidence(ledger: PackageExperimentLedger, trialId: string, directory: string) { + const trial = ledger.trial(trialId) + if (trial.state !== 'succeeded' || !trial.report_hash) throw new Error('Trial has no successful retained evidence') + const report = trialSchema.parse(retained(directory, trial.report_hash)) + if (report.trialId !== trial.id || report.attemptId !== trial.attempt_id) throw new Error('Retained trial report identity mismatch') + const study = ledger.study(trial.request.studyId) + same(report.study, study, 'study') + same(report.request, trial.request, 'request') + if (configHash(report.protocol) !== study.protocolHash) throw new Error('Retained protocol integrity mismatch') + let preflight: ReturnType | null = null + if (report.protocol.preflightPlan) { + const plan = report.protocol.preflightPlan + const candidate = trial.request.variants.find(variant => variant.label === 'candidate') + if (!candidate || !report.preflight || plan.configHash !== candidate.configHash || plan.side !== study.side + || plan.hardwareId !== report.protocol.hardwareId || plan.sampleRateHz !== report.protocol.sampleRateHz) throw new Error('Retained preflight binding mismatch') + preflight = verifyPreflightEvidence(retained(directory, report.preflight.reportHash), { plan, packageHash: candidate.packageHash, image: candidate.image }) + same(preflight.status, report.preflight.status, 'preflight status') + } + else if (report.preflight) throw new Error('Unexpected retained preflight evidence') + const selected = study.nights.filter(night => night.split === trial.request.split) + if (report.nights.length !== selected.length || new Set(report.nights.map(night => night.nightId)).size !== selected.length + || selected.some(night => !report.nights.some(row => row.nightId === night.id))) throw new Error('Retained night set integrity mismatch') + const nights = report.nights.map((summary) => { + const night = nightSchema.parse(retained(directory, summary.reportHash)) + const expected = selected.find(row => row.id === summary.nightId) + same(night.night, expected, 'night') + if (night.trialId !== trial.id || night.attemptId !== trial.attempt_id) throw new Error('Retained night attempt integrity mismatch') + if (night.referenceImport.side !== study.side || night.referenceImport.subjectId !== study.subjectId + || previewReferenceImport(night.referenceImport).summary.contentHash !== night.night.referenceHash) throw new Error('Retained reference integrity mismatch') + for (const output of night.outputs) { + const variant = trial.request.variants.find(row => row.label === output.label) + if (!variant || output.request.packageHash !== variant.packageHash + || configHash({ config: output.request.config, seed: output.request.seed }) !== variant.configHash + || output.request.inputHash !== night.night.inputHash || output.request.side !== study.side + || output.request.hardwareId !== report.protocol.hardwareId || output.request.sampleRateHz !== report.protocol.sampleRateHz + || output.result.execution.image !== variant.image) throw new Error('Retained variant integrity mismatch') + same(output.result.execution.limits, output.manifest.resources, 'resource limits') + } + const comparison = comparePackageOutputs({ schema: 'biometric-package-comparison-v1', metricId: report.protocol.metricId, + startMs: night.night.startMs, endMs: night.night.endMs, minimumQuality: report.protocol.minimumQuality, + referenceImport: night.referenceImport, alignment: report.protocol.alignment, outputs: night.outputs }) + const resources = night.outputs.map(output => ({ label: output.label, assessment: assessPackageResources(output.result.execution, report.protocol.resourcePolicy) })) + same(night.comparison, comparison, 'night comparison') + same(summary.comparison, comparison, 'summary comparison') + same(night.resources, resources, 'night resources') + same(summary.resources, resources, 'summary resources') + return { nightId: summary.nightId, reportHash: summary.reportHash, comparison, resources } + }) + const uncertaintyReports = (['default', 'current'] as const).map(baseline => ({ baseline, + report: uncertainty(nights.map(night => ({ cohortId: night.nightId, + difference: night.comparison.comparisons.find(row => row.candidate === 'candidate' && row.baseline === baseline)?.differences.commonMae ?? null })), report.protocol.uncertainty) })) + same(report.uncertainty, uncertaintyReports, 'uncertainty') + const currentEvidence = ledger.currentEvidence(trialId) + if (currentEvidence.state !== 'succeeded' || currentEvidence.reportHash !== trial.report_hash) throw new Error('Trial changed during evidence verification') + return { schema: 'biometric-managed-evidence-v1' as const, trialId, reportHash: trial.report_hash, + study, protocol: report.protocol, request: trial.request, preflight, nights, uncertainty: uncertaintyReports, currentEvidence, + interpretation: 'verified_retained_evidence_requires_full_qualification_and_review' as const } +} diff --git a/scripts/algorithm-experiments/managed-files.test.ts b/scripts/algorithm-experiments/managed-files.test.ts new file mode 100644 index 00000000..ba57d87e --- /dev/null +++ b/scripts/algorithm-experiments/managed-files.test.ts @@ -0,0 +1,77 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { chmod, mkdir, mkdtemp, readFile, rm, symlink, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { configHash } from '../../src/biometrics/algorithms/canonical' +import { packageFixture } from '../../src/biometrics/packages/tests/fixture' +import { fixtureResourceTestPolicy } from './resource-test-policy' +import { managedDescriptorSchema, runManagedFiles } from './managed-files' +let root: string +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'managed-storage-test-')) +}) +afterEach(async () => { + await rm(root, { recursive: true, force: true }) +}) +async function descriptor() { + const { manifest, files } = packageFixture() + await writeFile(join(root, 'proposal.json'), JSON.stringify({ schema: 'biometric-package-proposal-v1', manifest, + files: [...files].map(([path, bytes]) => ({ path, base64: bytes.toString('base64') })) })) + const protocol = { schema: 'biometric-managed-protocol-v1', hardwareId: 'pod4', sampleRateHz: 500, metricId: 'heart_rate', minimumQuality: 0.8, + alignment: { version: 'reference-alignment-v1', pointToleranceMs: 0, clockOffsetMs: 0, offsetSource: 'fixed' }, + uncertainty: { version: 'paired-night-percentile-v1', seed: 'fixture', resamples: 200, minimumNights: 3 }, resourcePolicy: fixtureResourceTestPolicy } + const value = { schema: 'biometric-managed-descriptor-v1', study: { schema: 'biometric-package-study-v1', subjectId: 'fixture', side: 'left', protocolHash: configHash(protocol), + nights: [{ id: 'night', startMs: 1791673200000, endMs: 1791673260000, inputHash: 'a'.repeat(64), referenceHash: 'b'.repeat(64), split: 'holdout' }] }, + protocol, split: 'holdout', idempotencyKey: 'test', variants: ['default', 'current', 'candidate'].map(label => ({ label, proposal: 'proposal.json', config: {}, seed: 1, + policy: { image: 'sha256:' + 'c'.repeat(64), maximumWallMs: 5000, maximumMemoryBytes: 100000000, maximumProcesses: 1, maximumOutputBytes: 10000, maximumInputBytes: 10000 } })), + nights: [{ id: 'night', frames: 'absent-frames', referenceImport: 'absent-reference' }] } + await writeFile(join(root, 'descriptor.json'), JSON.stringify(value)) + return value +} +describe('managed workspace boundary', () => { + it('rejects missing/extra night sources and protocol substitutions before opening inputs', async () => { + const value = await descriptor() + expect(managedDescriptorSchema.safeParse(value).success).toBe(true) + for (const changed of [{ ...value, nights: [] }, { ...value, nights: [...value.nights, value.nights[0]] }, + { ...value, protocol: { ...value.protocol, minimumQuality: 0.1 } }]) { + expect(managedDescriptorSchema.safeParse(changed).success).toBe(false) + } + }) + it('refuses public or symlinked workspaces', async () => { + await descriptor() + const publicPath = join(root, 'public') + await mkdir(publicPath) + await chmod(publicPath, 0o755) + await expect(runManagedFiles(publicPath, join(root, 'descriptor.json'))).rejects.toThrow('private owned directory') + const link = join(root, 'link') + await symlink(root, link) + await expect(runManagedFiles(link, join(root, 'descriptor.json'))).rejects.toThrow('private owned directory') + }) + it('refuses linked databases and sidecars without changing their targets', async () => { + await descriptor() + const target = join(root, 'target') + await writeFile(target, 'preserve me') + for (const name of ['experiments.sqlite', 'experiments.sqlite-wal', 'experiments.sqlite-shm', 'experiments.sqlite-journal']) { + const workspace = join(root, name + '-workspace') + await mkdir(workspace, { mode: 0o700 }) + await symlink(target, join(workspace, name)) + await expect(runManagedFiles(workspace, join(root, 'descriptor.json'))).rejects.toThrow('Unsafe experiment database') + expect(await readFile(target, 'utf8')).toBe('preserve me') + } + }) + it('refuses a foreign database and preserves its contents', async () => { + await descriptor() + const workspace = join(root, 'workspace') + await mkdir(workspace, { mode: 0o700 }) + const path = join(workspace, 'experiments.sqlite') + const database = new Database(path) + database.exec('CREATE TABLE vitals(value INTEGER); INSERT INTO vitals VALUES (60)') + database.close() + await chmod(path, 0o600) + const before = await readFile(path) + await expect(runManagedFiles(workspace, join(root, 'descriptor.json'))).rejects.toThrow('non-experiment database') + expect(await readFile(path)).toEqual(before) + }) +}) diff --git a/scripts/algorithm-experiments/managed-files.ts b/scripts/algorithm-experiments/managed-files.ts new file mode 100644 index 00000000..088a3267 --- /dev/null +++ b/scripts/algorithm-experiments/managed-files.ts @@ -0,0 +1,95 @@ +import { withExperimentWorkspace, defaultWorkspacePolicy } from './workspace' +import { assessManagedQualification } from './qualification' +import Database from 'better-sqlite3' +import { constants } from 'node:fs' +import { lstat, mkdir, open } from 'node:fs/promises' +import { dirname, join, resolve } from 'node:path' +import { z } from 'zod' +import { configHash } from '../../src/biometrics/algorithms/canonical' +import { packageStudySchema, PackageExperimentLedger } from './ledger' +import { managedProtocolSchema, runManagedTrial } from './managed' +import { experimentPolicySchema } from './policy' +import { readJsonFile, readJsonLines, readPackageProposal } from './files' + +const sourcePath = z.string().min(1).max(4096) +export const managedDescriptorSchema = z.object({ schema: z.literal('biometric-managed-descriptor-v1'), + study: packageStudySchema, protocol: managedProtocolSchema, split: z.enum(['tuning', 'validation', 'holdout']), + preflightSuite: sourcePath.optional(), + idempotencyKey: z.string().min(1).max(200), + variants: z.array(z.object({ label: z.enum(['default', 'current', 'candidate']), proposal: sourcePath, + config: z.record(z.string(), z.unknown()), seed: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER), policy: experimentPolicySchema }).strict()).length(3), + nights: z.array(z.object({ id: z.string().min(1).max(200), frames: sourcePath, referenceImport: sourcePath }).strict()).min(1).max(60), +}).strict().superRefine((value, context) => { + if (Boolean(value.protocol.preflightPlan) !== Boolean(value.preflightSuite)) context.addIssue({ code: 'custom', message: 'Preflight plan and suite must be supplied together' }) + if (new Set(value.variants.map(variant => variant.label)).size !== 3) context.addIssue({ code: 'custom', message: 'All three variants are required' }) + const selected = value.study.nights.filter(night => night.split === value.split) + if (new Set(value.nights.map(night => night.id)).size !== value.nights.length || selected.length !== value.nights.length + || selected.some(night => !value.nights.some(source => source.id === night.id))) context.addIssue({ code: 'custom', message: 'Night sources must exactly match the selected split' }) + if (configHash(value.protocol) !== value.study.protocolHash) context.addIssue({ code: 'custom', message: 'Study protocol hash mismatch' }) +}) + +async function privateDirectory(path: string) { + await mkdir(path, { recursive: true, mode: 0o700 }) + const stat = await lstat(path) + if (!stat.isDirectory() || stat.isSymbolicLink() || (stat.mode & 0o077) !== 0 + || (process.getuid && stat.uid !== process.getuid())) throw new Error('Experiment workspace must be a private owned directory, not a symlink') +} + +/** CLI storage boundary. Candidate bytes never choose this workspace or its + * database path; the operator passes the dedicated private workspace argument. */ +export async function runManagedFiles(workspace: string, descriptorPath: string, storagePolicy: unknown = defaultWorkspacePolicy) { + const descriptor = managedDescriptorSchema.parse(readJsonFile(descriptorPath, 2 * 1024 * 1024)) + const base = dirname(resolve(descriptorPath)) + const variants = descriptor.variants.map((variant) => { + const proposal = readPackageProposal(resolve(base, variant.proposal)) + return { ...variant, manifest: proposal.manifest, files: proposal.files, packageHash: proposal.packageHash } + }) + await privateDirectory(workspace) + return withExperimentWorkspace(workspace, storagePolicy, async (owned) => { + const reports = join(workspace, 'reports') + await privateDirectory(reports) + const databasePath = join(workspace, 'experiments.sqlite') + // Refuse symlinked SQLite files and sidecars before opening SQLite. The + // private owned directory excludes mutation by candidate/other-user code. + for (const path of [databasePath, databasePath + '-wal', databasePath + '-shm', databasePath + '-journal']) { + try { + const stat = await lstat(path) + if (!stat.isFile() || stat.isSymbolicLink() || stat.nlink !== 1 || (stat.mode & 0o077) !== 0 || (process.getuid && stat.uid !== process.getuid())) throw new Error('Unsafe experiment database file') + } + catch (error) { + if (!(error instanceof Error) || !('code' in error) || error.code !== 'ENOENT') throw error + } + } + try { + const created = await open(databasePath, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY | constants.O_NOFOLLOW, 0o600) + await created.close() + } + catch (error) { + if (!(error instanceof Error) || !('code' in error) || error.code !== 'EEXIST') throw error + } + const db = new Database(databasePath) + try { + owned.configureDatabase(db) + const ledger = new PackageExperimentLedger(db) + const study = ledger.createStudy(descriptor.study) + const trialId = ledger.reserveTrial({ studyId: study.id, split: descriptor.split, + variants: variants.map(variant => ({ label: variant.label, packageHash: variant.packageHash, + configHash: configHash({ config: variant.config, seed: variant.seed }), image: variant.policy.image })) }, descriptor.idempotencyKey) + const maximumInputBytes = Math.min(...variants.map(variant => variant.policy.maximumInputBytes)) + const result = await runManagedTrial({ ledger, trialId, protocol: descriptor.protocol, variants, reportDirectory: reports, publishReport: owned.publishReport, + preflightSuite: descriptor.preflightSuite && ledger.trial(trialId).state !== 'succeeded' ? readJsonFile(resolve(base, descriptor.preflightSuite), 32 * 1024 * 1024) : undefined, + loadNight: async (id) => { + await owned.checkCapacity() + const source = descriptor.nights.find(night => night.id === id) + if (!source) throw new Error('Missing retained night source') + return { referenceImport: readJsonFile(resolve(base, source.referenceImport), 2 * 1024 * 1024), + frames: () => readJsonLines(resolve(base, source.frames), maximumInputBytes) } + } }) + const reviewed = assessManagedQualification(ledger, trialId, reports) + return { schema: 'biometric-managed-receipt-v1' as const, studyId: study.id, trialId, ...result, + currentEvidence: reviewed.currentEvidence, accuracy: reviewed.qualification.accuracy, preflight: reviewed.qualification.preflight, + qualification: reviewed.qualification, assessmentHash: reviewed.assessmentHash, reportFile: join(reports, `${result.reportHash}.json`) } + } + finally { db.close() } + }) +} diff --git a/scripts/algorithm-experiments/managed-protocol.ts b/scripts/algorithm-experiments/managed-protocol.ts new file mode 100644 index 00000000..74bfc07d --- /dev/null +++ b/scripts/algorithm-experiments/managed-protocol.ts @@ -0,0 +1,14 @@ +import { packagePreflightPlanSchema } from './preflight' +import { packageAccuracyPolicySchema } from './accuracy-policy' +import { z } from 'zod' +import { nightBootstrapSchema } from '../../src/biometrics/backtest/uncertainty-policy' +import { packageResourcePolicySchema } from './resource-assessment' + +export const managedProtocolSchema = z.object({ schema: z.literal('biometric-managed-protocol-v1'), + hardwareId: z.string().min(1).max(100), sampleRateHz: z.number().int().positive(), metricId: z.string().min(1).max(100), + minimumQuality: z.number().min(0).max(1), + alignment: z.object({ version: z.literal('reference-alignment-v1'), pointToleranceMs: z.number().int().min(0).max(300000), + clockOffsetMs: z.number().int().min(-3600000).max(3600000), offsetSource: z.enum(['fixed', 'training']) }).strict(), + accuracyPolicy: packageAccuracyPolicySchema.optional(), preflightPlan: packagePreflightPlanSchema.optional(), + uncertainty: nightBootstrapSchema, resourcePolicy: packageResourcePolicySchema, +}).strict() diff --git a/scripts/algorithm-experiments/managed.test.ts b/scripts/algorithm-experiments/managed.test.ts new file mode 100644 index 00000000..a0256a3d --- /dev/null +++ b/scripts/algorithm-experiments/managed.test.ts @@ -0,0 +1,299 @@ +import { defaultWorkspacePolicy } from './workspace' +// @vitest-environment node +import Database from 'better-sqlite3' +import { createHash } from 'node:crypto' +import { mkdtemp, readFile, rm, writeFile, stat } from 'node:fs/promises' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import * as reportFiles from './files' +import { beforeAll, describe, expect, it, vi } from 'vitest' +import { configHash, dataHash } from '../../src/biometrics/algorithms/canonical' +import { previewReferenceImport } from '../../src/biometrics/references/normalize' +import { packageFixture } from '../../src/biometrics/packages/tests/fixture' +import { inspectPythonEnvironment } from './environment' +import { packageInputHash } from './input' +import { PackageExperimentLedger } from './ledger' +import { runManagedFiles } from './managed-files' +import { runManagedTrial } from './managed' +import { readManagedEvidence } from './managed-evidence' +import { runPackagePreflight } from './preflight' +import { verifyPreflightEvidence } from './preflight-evidence' +import { assessManagedQualification } from './qualification' +import { fixtureResourceTestPolicy } from './resource-test-policy' +const image = process.env.BIOMETRIC_EXPERIMENT_TEST_IMAGE +let environmentBytes: Buffer +beforeAll(async () => { + if (image) environmentBytes = Buffer.from(JSON.stringify(await inspectPythonEnvironment(image))) +}) +const start = 1791673200000 +function fixture(ledger: PackageExperimentLedger, withPreflight = false) { + const { manifest, files } = packageFixture() + const code = `import json, sys +init = json.loads(sys.stdin.readline()) +for line in sys.stdin: + msg = json.loads(line) + frame = msg['frame'] + print(json.dumps({'schema': 'biometric-metric-frame-v1', 'runHash': init['runHash'], 'frameHash': msg['frameHash'], 'sequence': frame['sequence'], 'state': {}, 'readings': [{'timestampMs': frame['timestampMs'], 'side': 'left', 'metric': 'heart_rate', 'value': 60 + init['request']['config']['offset'], 'quality': 1, 'flags': []}]}), flush=True) +` + files.set('engine.py', Buffer.from(code)) + if (environmentBytes) files.set(manifest.dependencyLock, environmentBytes) + const declared = { ...manifest, metrics: manifest.metrics.map(metric => ({ ...metric, reference: { metric: 'heart_rate', windowMs: null } })), + resources: { cpuMs: 5000, wallMs: 5000, memoryBytes: 64 * 1024 * 1024, processes: 1, outputBytes: 16384 }, + artifacts: [...files].map(([path, bytes]) => ({ path, bytes: bytes.length, sha256: createHash('sha256').update(bytes).digest('hex') })) } + const frames = [0, 1].map(sequence => ({ schema: 'biometric-signal-frame-v1', sequence, timestampMs: start + sequence * 1000, + samples: { left: [1, 2, 3], right: null }, context: { occupied: true, pumpActive: false } })) + const referenceImport = { side: 'left' as const, subjectId: 'fixture', source: 'synthetic', rows: [0, 1].map(index => ({ recordKey: String(index), metric: 'heart_rate', value: 60, unit: 'bpm', timestampMs: start + index * 1000 })) } + const preflightSuite = { schema: 'biometric-package-fixtures-v1', resourcePolicy: fixtureResourceTestPolicy, + fixtures: [{ id: 'clean', category: 'clean_signal', frames, + request: { schema: 'biometric-package-run-v1', packageHash: configHash(declared), inputHash: packageInputHash(frames), + hardwareId: 'pod4', sampleRateHz: 500, side: 'left', config: { offset: 0 }, seed: 1 }, + expectations: [{ kind: 'numeric', metric: 'heart_rate', startMs: start, endMs: start + 1006, + minimum: 59, maximum: 61, minimumReadings: 2, minimumQuality: 0.8 }] }] } + const preflightPlan = { schema: 'biometric-package-preflight-plan-v1', suiteHash: dataHash(preflightSuite), repetitions: 2, + configHash: configHash({ config: { offset: 0 }, seed: 1 }), hardwareId: 'pod4', sampleRateHz: 500, side: 'left' } + const protocol = { schema: 'biometric-managed-protocol-v1', hardwareId: 'pod4', sampleRateHz: 500, metricId: 'heart_rate', minimumQuality: 0.8, ...(withPreflight ? { preflightPlan } : {}), + alignment: { version: 'reference-alignment-v1', pointToleranceMs: 0, clockOffsetMs: 0, offsetSource: 'fixed' }, + uncertainty: { version: 'paired-night-percentile-v1', seed: 'fixture', resamples: 200, minimumNights: 3 }, resourcePolicy: fixtureResourceTestPolicy } + const study = ledger.createStudy({ schema: 'biometric-package-study-v1', subjectId: 'fixture', side: 'left', protocolHash: configHash(protocol), + nights: [{ id: 'night', startMs: start, endMs: start + 120000, inputHash: packageInputHash(frames), referenceHash: previewReferenceImport(referenceImport).summary.contentHash, split: 'holdout' }] }) + const variants = (['default', 'current', 'candidate'] as const).map((label, index) => ({ label, manifest: declared, files, + config: { offset: 4 - index * 2 }, seed: 1, + policy: { image: image ?? 'sha256:' + 'f'.repeat(64), maximumWallMs: 10000, maximumMemoryBytes: 128 * 1024 * 1024, maximumProcesses: 4, maximumOutputBytes: 32768, maximumInputBytes: 32768 } })) + const request = { studyId: study.id, split: 'holdout', variants: variants.map(variant => ({ label: variant.label, packageHash: configHash(declared), + configHash: configHash({ config: variant.config, seed: variant.seed }), image: variant.policy.image })) } + return { trialId: ledger.reserveTrial(request, 'first'), request, protocol, variants, referenceImport, frames, ...(withPreflight ? { preflightSuite } : {}) } +} +describe.skipIf(!image)('managed package trials through real Docker', () => { + it('executes combined fixture and determinism preflight for the exact candidate configuration', async () => { + const db = new Database(':memory:') + try { + const input = fixture(new PackageExperimentLedger(db)), candidate = input.variants[2] + const suite = { schema: 'biometric-package-fixtures-v1', resourcePolicy: fixtureResourceTestPolicy, + fixtures: [{ id: 'clean', category: 'clean_signal', frames: input.frames, + request: { schema: 'biometric-package-run-v1', packageHash: configHash(candidate.manifest), inputHash: packageInputHash(input.frames), + hardwareId: 'pod4', sampleRateHz: 500, side: 'left', config: candidate.config, seed: candidate.seed }, + expectations: [{ kind: 'numeric', metric: 'heart_rate', startMs: start, endMs: start + 1006, + minimum: 59, maximum: 61, minimumReadings: 2, minimumQuality: 0.8 }] }] } + const plan = { schema: 'biometric-package-preflight-plan-v1', suiteHash: dataHash(suite), repetitions: 2, + configHash: configHash({ config: candidate.config, seed: candidate.seed }), hardwareId: 'pod4', sampleRateHz: 500, side: 'left' } + const result = await runPackagePreflight(candidate, plan, suite) + expect(result).toMatchObject({ status: 'incomplete', packageHash: configHash(candidate.manifest), planHash: configHash(plan), + fixtureReport: { missingCategories: expect.arrayContaining(['empty_bed', 'motion']) }, + determinism: [{ fixtureId: 'clean', error: null, report: { deterministic: true, runs: expect.any(Array) } }] }) + expect(verifyPreflightEvidence(result, { plan, packageHash: configHash(candidate.manifest), image: candidate.policy.image }).status).toBe('incomplete') + const changed = structuredClone(result) + changed.status = 'external_checks_passed' + expect(() => verifyPreflightEvidence(changed, { plan, packageHash: result.packageHash, image: result.image })).toThrow('integrity mismatch') + const changedRuns = structuredClone(result) + const repeated = changedRuns.determinism[0].report + if (!repeated) throw new Error('Missing determinism fixture') + repeated.runs[1].outputHash = 'f'.repeat(64) + expect(() => verifyPreflightEvidence(changedRuns, { plan, packageHash: result.packageHash, image: result.image })).toThrow('integrity mismatch') + expect(result.determinism[0].report?.runs).toHaveLength(2) + expect(result.determinism[0].resources).toHaveLength(2) + const failingSuite = structuredClone(suite) + failingSuite.fixtures[0].expectations[0].minimum = 58 + failingSuite.fixtures[0].expectations[0].maximum = 59 + expect(await runPackagePreflight(candidate, { ...plan, suiteHash: dataHash(failingSuite) }, failingSuite)).toMatchObject({ + status: 'failed', fixtureReport: { status: 'failed' }, + }) + for (const change of [{ configHash: 'f'.repeat(64) }, { side: 'right' }, { hardwareId: 'other' }, { sampleRateHz: 100 }, { suiteHash: 'f'.repeat(64) }]) { + await expect(runPackagePreflight(candidate, { ...plan, ...change }, suite)).rejects.toThrow(/mismatch/) + } + const controller = new AbortController() + controller.abort() + await expect(runPackagePreflight({ ...candidate, signal: controller.signal }, plan, suite)).rejects.toThrow() + } + finally { db.close() } + }, 30000) + + it('records exposure before loading, compares all variants, publishes verified artifacts and reuses completion', async () => { + const db = new Database(':memory:'), root = await mkdtemp(join(tmpdir(), 'managed-trial-test-')) + try { + const ledger = new PackageExperimentLedger(db) + const input = fixture(ledger, true) + let loads = 0 + const options = { ...input, ledger, reportDirectory: root, loadNight: async () => { + loads++ + expect(ledger.trial(input.trialId).state).toBe('running') + expect((db.prepare('SELECT COUNT(*) AS count FROM package_exposures').get() as { count: number }).count).toBe(1) + return { referenceImport: input.referenceImport, frames: () => input.frames } + } } + const result = await runManagedTrial(options) + expect(result).toMatchObject({ state: 'succeeded', evidence: { uniqueHoldoutAttempt: true } }) + const report = JSON.parse(await readFile(join(root, `${result.reportHash}.json`), 'utf8')) + expect(dataHash(report)).toBe(result.reportHash) + expect(report.preflight.status).toBe('incomplete') + const preflightPath = join(root, `${report.preflight.reportHash}.json`) + const preflightBytes = await readFile(preflightPath) + await rm(preflightPath) + await expect(runManagedTrial(options)).rejects.toThrow() + expect(loads).toBe(1) + await writeFile(preflightPath, preflightBytes) + expect(readManagedEvidence(ledger, input.trialId, root).preflight?.status).toBe('incomplete') + expect(report.uncertainty.map((row: { report: { estimate: number } }) => row.report.estimate)).toEqual([-4, -2]) + const night = JSON.parse(await readFile(join(root, `${report.nights[0].reportHash}.json`), 'utf8')) + expect(dataHash(night)).toBe(report.nights[0].reportHash) + expect(night.outputs).toHaveLength(3) + expect(await runManagedTrial(options)).toEqual({ state: 'already_completed', reportHash: result.reportHash }) + expect(loads).toBe(1) + const qualification = assessManagedQualification(ledger, input.trialId, root) + expect(assessManagedQualification(ledger, input.trialId, root).assessmentHash).toBe(qualification.assessmentHash) + expect(qualification.qualification.installationAuthorized).toBe(false) + expect(readManagedEvidence(ledger, input.trialId, root)).toMatchObject({ + reportHash: result.reportHash, currentEvidence: { uniqueHoldoutAttempt: true }, + }) + // A valid top-level hash does not excuse a missing or corrupted child. + const childPath = join(root, `${report.nights[0].reportHash}.json`) + await rm(childPath) + await expect(runManagedTrial(options)).rejects.toThrow() + await writeFile(childPath, JSON.stringify({ ...night, outputs: [] })) + await expect(runManagedTrial(options)).rejects.toThrow('integrity mismatch') + await writeFile(childPath, JSON.stringify(night)) + // Even a rehashed report graph cannot substitute a derived score. The + // verifier recomputes from the retained outputs and frozen protocol. + const forgedChild = structuredClone(night) + forgedChild.comparison.comparisons[0].differences.commonMae = -999 + const forgedChildHash = dataHash(forgedChild) + await writeFile(join(root, `${forgedChildHash}.json`), JSON.stringify(forgedChild)) + const forged = structuredClone(report) + forged.nights[0].reportHash = forgedChildHash + forged.nights[0].comparison = forgedChild.comparison + const forgedHash = dataHash(forged) + await writeFile(join(root, `${forgedHash}.json`), JSON.stringify(forged)) + db.prepare('UPDATE package_trials SET report_hash = ? WHERE id = ?').run(forgedHash, input.trialId) + expect(() => readManagedEvidence(ledger, input.trialId, root)).toThrow('night comparison integrity') + // Retained request/uncertainty substitutions are rejected independently. + for (const replacement of [{ ...report, uncertainty: [] }, { ...report, request: { ...report.request, split: 'tuning' } }]) { + const replacementHash = dataHash(replacement) + await writeFile(join(root, `${replacementHash}.json`), JSON.stringify(replacement)) + db.prepare('UPDATE package_trials SET report_hash = ? WHERE id = ?').run(replacementHash, input.trialId) + expect(() => readManagedEvidence(ledger, input.trialId, root)).toThrow('integrity mismatch') + } + db.prepare('UPDATE package_trials SET report_hash = ? WHERE id = ?').run(result.reportHash, input.trialId) + await writeFile(join(root, `${result.reportHash}.json`), JSON.stringify({ ...report, trialId: 'changed' })) + await expect(runManagedTrial(options)).rejects.toThrow('integrity mismatch') + expect(loads).toBe(1) + await writeFile(join(root, `${result.reportHash}.json`), JSON.stringify(report)) + const second = ledger.reserveTrial(input.request, 'second') + ledger.claim(second, 30000) + expect(ledger.currentEvidence(input.trialId).uniqueHoldoutAttempt).toBe(false) + expect(readManagedEvidence(ledger, input.trialId, root).currentEvidence.uniqueHoldoutAttempt).toBe(false) + const afterExposure = assessManagedQualification(ledger, input.trialId, root) + expect(afterExposure.assessmentHash).not.toBe(qualification.assessmentHash) + expect(afterExposure.qualification.accuracy.missing).toContain('no_unique_current_holdout') + expect(loads).toBe(1) + } + finally { + db.close() + await rm(root, { recursive: true, force: true }) + } + }, 30000) + it('runs a managed descriptor through persistent storage and reuses it without reopening night files', async () => { + const db = new Database(':memory:'), root = await mkdtemp(join(tmpdir(), 'managed-files-test-')) + try { + const ledger = new PackageExperimentLedger(db), input = fixture(ledger, true) + const variant = input.variants[0] + const proposal = { schema: 'biometric-package-proposal-v1', manifest: variant.manifest, + files: [...variant.files].map(([path, bytes]) => ({ path, base64: bytes.toString('base64') })) } + const descriptor = { schema: 'biometric-managed-descriptor-v1', study: ledger.study(input.request.studyId), protocol: input.protocol, + split: 'holdout', idempotencyKey: 'managed-files', preflightSuite: 'suite.json', + variants: input.variants.map(variant => ({ label: variant.label, proposal: 'proposal.json', config: variant.config, seed: variant.seed, policy: variant.policy })), + nights: [{ id: 'night', frames: 'frames.jsonl', referenceImport: 'references.json' }] } + await writeFile(join(root, 'proposal.json'), JSON.stringify(proposal)) + await writeFile(join(root, 'descriptor.json'), JSON.stringify(descriptor)) + await writeFile(join(root, 'suite.json'), JSON.stringify(input.preflightSuite)) + await writeFile(join(root, 'references.json'), JSON.stringify(input.referenceImport)) + await writeFile(join(root, 'frames.jsonl'), input.frames.map(frame => JSON.stringify(frame)).join('\n')) + const workspace = join(root, 'workspace') + const first = await runManagedFiles(workspace, join(root, 'descriptor.json')) + expect(first).toMatchObject({ state: 'succeeded', currentEvidence: { uniqueHoldoutAttempt: true }, + accuracy: { status: 'insufficient_evidence', missing: expect.arrayContaining(['no_frozen_accuracy_policy']) } }) + expect((await stat(join(workspace, 'experiments.sqlite'))).mode & 0o777).toBe(0o600) + expect(dataHash(JSON.parse(await readFile(first.reportFile, 'utf8')))).toBe(first.reportHash) + expect(first.preflight?.status).toBe('incomplete') + expect(first.qualification).toMatchObject({ status: 'insufficient_evidence', installationAuthorized: false }) + expect(first.assessmentHash).toMatch(/^[a-f0-9]{64}$/) + await rm(join(root, 'suite.json')) + await rm(join(root, 'frames.jsonl')) + await rm(join(root, 'references.json')) + const repeated = await runManagedFiles(workspace, join(root, 'descriptor.json')) + expect(repeated).toMatchObject({ state: 'already_completed', trialId: first.trialId, reportHash: first.reportHash, assessmentHash: first.assessmentHash }) + await expect(runManagedFiles(workspace, join(root, 'descriptor.json'), { ...defaultWorkspacePolicy, maximumReportFiles: 1 })).rejects.toThrow('file quota') + descriptor.variants[2].config.offset++ + await writeFile(join(root, 'descriptor.json'), JSON.stringify(descriptor)) + await expect(runManagedFiles(workspace, join(root, 'descriptor.json'))).rejects.toThrow('idempotency conflict') + } + finally { + db.close() + await rm(root, { recursive: true, force: true }) + } + }, 30000) + it('refuses publication after its lease expires while loading retained input', async () => { + const db = new Database(':memory:'), root = await mkdtemp(join(tmpdir(), 'managed-trial-stale-')) + let now = 1000 + try { + const ledger = new PackageExperimentLedger(db, () => now), input = fixture(ledger) + await expect(runManagedTrial({ ...input, ledger, reportDirectory: root, loadNight: async () => { + now += 30000 + return { referenceImport: input.referenceImport, frames: () => input.frames } + } })).rejects.toThrow('Stale trial lease') + expect(ledger.trial(input.trialId).report_hash).toBeNull() + expect(ledger.claim(input.trialId, 30000).unexposedHoldoutAtClaim).toBe(false) + } + finally { + db.close() + await rm(root, { recursive: true, force: true }) + } + }) + it('fails substituted references after recording access and preserves failure evidence', async () => { + const db = new Database(':memory:'), root = await mkdtemp(join(tmpdir(), 'managed-trial-failure-')) + try { + const ledger = new PackageExperimentLedger(db), input = fixture(ledger) + await expect(runManagedTrial({ ...input, ledger, reportDirectory: root, loadNight: async () => ({ + referenceImport: { ...input.referenceImport, source: 'changed' }, frames: () => input.frames, + }) })).rejects.toThrow('reference identity') + const trial = ledger.trial(input.trialId) + expect(trial.state).toBe('failed') + const report = JSON.parse(await readFile(join(root, `${trial.report_hash}.json`), 'utf8')) + expect(dataHash(report)).toBe(trial.report_hash) + expect(ledger.claim(input.trialId, 30000).unexposedHoldoutAtClaim).toBe(false) + } + finally { + db.close() + await rm(root, { recursive: true, force: true }) + } + }) +}) + +// These fail before any container or environment probe and run without Docker. +it.each([false, true])('finalizes failed trials when report publication fails (stale lease: %s)', async (stale) => { + const db = new Database(':memory:'), root = await mkdtemp(join(tmpdir(), 'managed-publication-failure-')) + let now = 1000 + const original = new Error('retained input unavailable') + const publication = new Error('report disk full') + const publish = vi.spyOn(reportFiles, 'publishJson').mockRejectedValue(publication) + try { + const ledger = new PackageExperimentLedger(db, () => now), input = fixture(ledger) + let replacement: ReturnType | undefined + const task = runManagedTrial({ ...input, ledger, reportDirectory: root, loadNight: async () => { + if (stale) { + now += 30000 + replacement = ledger.claim(input.trialId, 30000) + } + throw original + } }) + await expect(task).rejects.toMatchObject({ cause: original, errors: expect.arrayContaining([original, publication]), message: expect.stringContaining('failure evidence incomplete') }) + const trial = ledger.trial(input.trialId) + expect(trial).toMatchObject(stale + ? { state: 'running', attempt_id: replacement?.attemptId, report_hash: null } + : { state: 'failed', report_hash: null, lease_until: null }) + if (!stale) expect(ledger.claim(input.trialId, 30000).unexposedHoldoutAtClaim).toBe(false) + expect(publish).toHaveBeenCalledOnce() + } + finally { + publish.mockRestore() + db.close() + await rm(root, { recursive: true, force: true }) + } +}) diff --git a/scripts/algorithm-experiments/managed.ts b/scripts/algorithm-experiments/managed.ts new file mode 100644 index 00000000..a6fb9cdd --- /dev/null +++ b/scripts/algorithm-experiments/managed.ts @@ -0,0 +1,162 @@ +import { packageFixtureSuiteSchema } from './fixtures' +import { runPackagePreflight } from './preflight' +import { readManagedEvidence } from './managed-evidence' +import { managedProtocolSchema } from './managed-protocol' +export { managedProtocolSchema } from './managed-protocol' +import { createHash } from 'node:crypto' +import { mkdir } from 'node:fs/promises' +import { join } from 'node:path' +import { z } from 'zod' +import { configHash, dataHash } from '../../src/biometrics/algorithms/canonical' +import { createNightUncertainty } from '../../src/biometrics/backtest/uncertainty-core' +import { verifyAlgorithmPackage } from '../../src/biometrics/packages/manifest' +import { signalFrameSchema } from '../../src/biometrics/packages/protocol' +import { referenceImportSchema } from '../../src/biometrics/references/contracts' +import { previewReferenceImport } from '../../src/biometrics/references/normalize' +import { executePackageExperiment, type ExperimentPolicy } from './container' +import type { PackageExperimentLedger } from './ledger' +import { comparePackageOutputs } from './comparison' +import { assessPackageResources } from './resource-assessment' +import { publishJson } from './files' + +interface Variant { + label: 'default' | 'current' | 'candidate' + manifest: unknown + files: ReadonlyMap + config: Record + seed: number + policy: ExperimentPolicy +} +const uncertainty = createNightUncertainty(text => createHash('sha256').update(text).digest(), 'equal_night_mean_common_mae_difference') + +/** The loader is trusted host code, called only after exposure is committed. + * It must read retained inputs; no candidate module is loaded on the host. */ +export async function runManagedTrial(options: { + ledger: PackageExperimentLedger + trialId: string + protocol: unknown + preflightSuite?: unknown + variants: Variant[] + reportDirectory: string + publishReport?: typeof publishJson + loadNight: (id: string) => Promise<{ referenceImport: unknown, frames: () => Iterable }> +}) { + const publishReport = options.publishReport ?? publishJson + const trial = options.ledger.trial(options.trialId) + if (trial.state === 'succeeded') { + const evidence = readManagedEvidence(options.ledger, trial.id, options.reportDirectory) + return { state: 'already_completed' as const, reportHash: evidence.reportHash } + } + const study = options.ledger.study(trial.request.studyId) + const protocol = managedProtocolSchema.parse(structuredClone(options.protocol)) + if (configHash(protocol) !== study.protocolHash) throw new Error('Managed protocol identity mismatch') + if (options.variants.length !== 3 || new Set(options.variants.map(variant => variant.label)).size !== 3) throw new Error('Three distinct variants are required') + const variants = options.variants.map((variant) => { + const verified = verifyAlgorithmPackage(variant.manifest, variant.files) + const expected = trial.request.variants.find(item => item.label === variant.label) + const config = structuredClone(variant.config) + const seed = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER).parse(variant.seed) + if (!expected || verified.packageHash !== expected.packageHash || configHash({ config, seed }) !== expected.configHash || variant.policy.image !== expected.image) throw new Error('Managed variant identity mismatch') + return { ...variant, manifest: verified.manifest, packageHash: verified.packageHash, config, seed, + files: new Map([...variant.files].map(([path, bytes]) => [path, Buffer.from(bytes)])), policy: structuredClone(variant.policy) } + }) + const candidate = variants.find(variant => variant.label === 'candidate') + if (protocol.preflightPlan && (!candidate || protocol.preflightPlan.configHash !== configHash({ config: candidate.config, seed: candidate.seed }) + || protocol.preflightPlan.hardwareId !== protocol.hardwareId || protocol.preflightPlan.sampleRateHz !== protocol.sampleRateHz + || protocol.preflightPlan.side !== study.side)) throw new Error('Managed preflight candidate mismatch') + if (Boolean(protocol.preflightPlan) !== (options.preflightSuite !== undefined)) throw new Error('Managed preflight plan and suite must be supplied together') + const preflightSuite = options.preflightSuite === undefined ? undefined : packageFixtureSuiteSchema.parse(structuredClone(options.preflightSuite)) + if (protocol.preflightPlan && dataHash(preflightSuite) !== protocol.preflightPlan.suiteHash) throw new Error('Managed preflight suite mismatch') + await mkdir(options.reportDirectory, { recursive: true, mode: 0o700 }) + const lease = options.ledger.claim(options.trialId, 30000) + const controller = new AbortController() + const heartbeat = setInterval(() => { + try { + options.ledger.renew(lease, 30000) + } + catch { controller.abort() } + }, 5000) + const nights: { + nightId: string + reportHash: string + comparison: ReturnType + resources: { label: Variant['label'], assessment: ReturnType }[] + }[] = [] + let preflight: { reportHash: string, status: 'failed' | 'incomplete' | 'external_checks_passed' } | undefined + try { + if (protocol.preflightPlan && candidate) { + const report = await runPackagePreflight({ ...candidate, signal: controller.signal }, protocol.preflightPlan, preflightSuite) + const reportHash = dataHash(report) + options.ledger.renew(lease, 30000) + await publishReport(join(options.reportDirectory, `${reportHash}.json`), report) + preflight = { reportHash, status: report.status } + } + for (const night of study.nights.filter(night => night.split === trial.request.split)) { + controller.signal.throwIfAborted() + options.ledger.renew(lease, 30000) + const source = await options.loadNight(night.id) + const referenceImport = referenceImportSchema.parse(structuredClone(source.referenceImport)) + if (referenceImport.side !== study.side || referenceImport.subjectId !== study.subjectId + || previewReferenceImport(referenceImport).summary.contentHash !== night.referenceHash) throw new Error('Managed reference identity mismatch') + const outputs = [] + for (const variant of variants) { + controller.signal.throwIfAborted() + options.ledger.renew(lease, 30000) + const request = { schema: 'biometric-package-run-v1', packageHash: variant.packageHash, inputHash: night.inputHash, + hardwareId: protocol.hardwareId, sampleRateHz: protocol.sampleRateHz, side: study.side, seed: variant.seed, config: variant.config } + const frames = function* () { + for (const input of source.frames()) { + const frame = signalFrameSchema.parse(input) + const length = frame.samples[study.side]?.length ?? 0 + if (frame.timestampMs < night.startMs || frame.timestampMs + length * 1000 / protocol.sampleRateHz > night.endMs) throw new Error('Frame outside frozen night') + yield frame + } + } + const result = await executePackageExperiment({ ...variant, request, frames: frames(), signal: controller.signal }) + outputs.push({ label: variant.label, manifest: variant.manifest, request, result }) + } + const comparison = comparePackageOutputs({ schema: 'biometric-package-comparison-v1', metricId: protocol.metricId, + startMs: night.startMs, endMs: night.endMs, minimumQuality: protocol.minimumQuality, referenceImport, alignment: protocol.alignment, outputs }) + const report = { schema: 'biometric-managed-night-v1', trialId: lease.trialId, attemptId: lease.attemptId, night, referenceImport, outputs, comparison, + resources: outputs.map(output => ({ label: output.label, assessment: assessPackageResources(output.result.execution, protocol.resourcePolicy) })) } + const reportHash = dataHash(report) + options.ledger.renew(lease, 30000) + await publishReport(join(options.reportDirectory, `${reportHash}.json`), report) + nights.push({ nightId: night.id, reportHash, comparison, resources: report.resources }) + } + const uncertaintyReports = (['default', 'current'] as const).map(baseline => ({ baseline, + report: uncertainty(nights.map(night => ({ cohortId: night.nightId, + difference: night.comparison.comparisons.find(comparison => comparison.candidate === 'candidate' && comparison.baseline === baseline)?.differences.commonMae ?? null })), protocol.uncertainty) })) + const report = { schema: 'biometric-managed-trial-v1', trialId: lease.trialId, attemptId: lease.attemptId, + study, protocol, request: trial.request, ...(preflight ? { preflight } : {}), nights, uncertainty: uncertaintyReports, exposureAtClaim: lease, + interpretation: 'requires_current_ledger_exposure_check_and_full_qualification_review' } + const reportHash = dataHash(report) + options.ledger.renew(lease, 30000) + await publishReport(join(options.reportDirectory, `${reportHash}.json`), report) + options.ledger.finish(lease, 'succeeded', reportHash) + return { state: 'succeeded' as const, reportHash, evidence: options.ledger.currentEvidence(lease.trialId) } + } + catch (error) { + controller.abort() + const report = { schema: 'biometric-managed-trial-failure-v1', trialId: lease.trialId, attemptId: lease.attemptId, ...(preflight ? { preflight } : {}), nights, + error: error instanceof Error ? error.message.slice(0, 1000) : 'Managed trial failed' } + const reportHash = dataHash(report) + let published = false + const secondaryErrors: unknown[] = [] + try { + await publishReport(join(options.reportDirectory, `${reportHash}.json`), report) + published = true + } + catch (publicationError) { secondaryErrors.push(publicationError) } + try { + // Publication failure must not leave a writable ledger's attempt running. + // A missing artifact has no report hash. The lease still fences replacement. + options.ledger.finish(lease, 'failed', published ? reportHash : null) + } + catch (ledgerError) { secondaryErrors.push(ledgerError) } + if (secondaryErrors.length) throw new AggregateError([error, ...secondaryErrors], + `${error instanceof Error ? error.message : 'Managed trial failed'}; failure evidence incomplete`, { cause: error }) + throw error + } + finally { clearInterval(heartbeat) } +} diff --git a/scripts/algorithm-experiments/policy.ts b/scripts/algorithm-experiments/policy.ts new file mode 100644 index 00000000..fcb28627 --- /dev/null +++ b/scripts/algorithm-experiments/policy.ts @@ -0,0 +1,5 @@ +import { z } from 'zod' +const positive = z.number().int().positive().max(Number.MAX_SAFE_INTEGER) +export const experimentPolicySchema = z.object({ image: z.string().regex(/^sha256:[a-f0-9]{64}$/), maximumWallMs: positive.max(3600000), + maximumMemoryBytes: positive, maximumProcesses: positive, maximumOutputBytes: positive, maximumInputBytes: positive, +}).strict() diff --git a/scripts/algorithm-experiments/preflight-evidence.ts b/scripts/algorithm-experiments/preflight-evidence.ts new file mode 100644 index 00000000..e6a0e12c --- /dev/null +++ b/scripts/algorithm-experiments/preflight-evidence.ts @@ -0,0 +1,103 @@ +import { z } from 'zod' +import { configHash, dataHash } from '../../src/biometrics/algorithms/canonical' +import { algorithmPackageManifestSchema } from '../../src/biometrics/packages/manifest' +import { PackageFrameValidator } from '../../src/biometrics/packages/protocol' +import { packageComparisonInputSchema } from './comparison' +import { assessFixtureReadings, fixtureCategories, packageFixtureSuiteSchema } from './fixtures' +import { packageInputHash } from './input' +import { packagePreflightPlanSchema } from './preflight' +import { assessPackageResources } from './resource-assessment' + +const hash = z.string().regex(/^[a-f0-9]{64}$/) +const execution = z.object({ image: z.string(), limits: z.unknown(), measurements: z.unknown(), wallMs: z.unknown() }).passthrough() +const result = packageComparisonInputSchema.shape.outputs.element.shape.result.extend({ execution }).passthrough() +const reportSchema = z.object({ schema: z.literal('biometric-package-preflight-report-v1'), packageHash: hash, manifest: algorithmPackageManifestSchema, + image: z.string(), plan: packagePreflightPlanSchema, planHash: hash, + fixtureReport: z.object({ schema: z.literal('biometric-package-fixture-report-v1'), packageHash: hash, suiteHash: hash, + suite: packageFixtureSuiteSchema, status: z.enum(['failed', 'incomplete', 'fixture_checks_passed']), missingCategories: z.array(z.string()), + results: z.array(z.object({ fixtureId: z.string(), category: z.string(), fixtureHash: hash, assessment: z.unknown(), + resourceAssessment: z.unknown(), execution: result.nullable(), error: z.string().nullable() }).strict()).min(1).max(64), + promotion: z.literal('requires_determinism_resource_holdout_compatibility_and_release_review'), + }).strict(), + determinism: z.array(z.object({ fixtureId: z.string(), resources: z.array(z.unknown()), error: z.string().nullable(), + report: z.object({ schema: z.literal('biometric-package-determinism-v1'), deterministic: z.boolean(), + runs: z.array(z.object({ packageHash: hash, runHash: hash, input: result.shape.frozenInput, outputHash: hash, + responseCount: z.number().int().positive(), execution }).strict()).min(2).max(5), + qualification: z.literal('requires_resource_noise_holdout_compatibility_checks_and_review'), + }).strict().nullable(), + }).strict()).min(1).max(64), + status: z.enum(['failed', 'incomplete', 'external_checks_passed']), + interpretation: z.literal('requires_heldout_accuracy_target_pod_compatibility_and_reviewed_release'), +}).strict() +function same(actual: unknown, expected: unknown) { + if (dataHash(actual) !== dataHash(expected)) throw new Error('Retained preflight integrity mismatch') +} +/** Recompute derived preflight status and verify fixture/run identities. Host + * receipts remain trusted execution records, not candidate attestations. */ +export function verifyPreflightEvidence(raw: unknown, expected: { plan: unknown, packageHash: string, image: string }) { + const report = reportSchema.parse(raw), suite = report.fixtureReport.suite + same(report.plan, expected.plan) + if (report.packageHash !== expected.packageHash || configHash(report.manifest) !== expected.packageHash || report.image !== expected.image + || report.planHash !== configHash(report.plan) || dataHash(suite) !== report.plan.suiteHash + || report.fixtureReport.suiteHash !== report.plan.suiteHash || report.fixtureReport.packageHash !== expected.packageHash) throw new Error('Retained preflight identity mismatch') + if (report.fixtureReport.results.length !== suite.fixtures.length || report.determinism.length !== suite.fixtures.length + || new Set(report.fixtureReport.results.map(row => row.fixtureId)).size !== suite.fixtures.length + || new Set(report.determinism.map(row => row.fixtureId)).size !== suite.fixtures.length) throw new Error('Retained preflight fixture set mismatch') + const checkExecution = (value: z.infer) => { + if (value.image !== expected.image) throw new Error('Retained preflight image mismatch') + same(value.limits, report.manifest.resources) + return assessPackageResources(value, suite.resourcePolicy) + } + const fixtures = suite.fixtures.map((fixture) => { + const row = report.fixtureReport.results.find(row => row.fixtureId === fixture.id) + const repeated = report.determinism.find(row => row.fixtureId === fixture.id) + if (!row || !repeated || row.fixtureHash !== dataHash(fixture) || row.category !== fixture.category + || fixture.request.packageHash !== expected.packageHash || fixture.request.inputHash !== packageInputHash(fixture.frames) + || configHash({ config: fixture.request.config, seed: fixture.request.seed }) !== report.plan.configHash + || fixture.request.hardwareId !== report.plan.hardwareId || fixture.request.sampleRateHz !== report.plan.sampleRateHz + || fixture.request.side !== report.plan.side) throw new Error('Retained preflight fixture identity mismatch') + const runHash = configHash(fixture.request) + if (row.execution) { + if (row.error !== null || row.execution.runHash !== runHash || row.execution.packageHash !== expected.packageHash + || row.execution.frozenInput.hash !== fixture.request.inputHash || row.execution.frozenInput.frameCount !== fixture.frames.length + || row.execution.results.length !== fixture.frames.length) throw new Error('Retained preflight run identity mismatch') + const validator = new PackageFrameValidator(report.manifest, fixture.request) + fixture.frames.forEach((frame, index) => validator.validateStep(frame, row.execution?.results[index])) + same(row.assessment, assessFixtureReadings(fixture.expectations, row.execution.results.flatMap(frame => frame.readings))) + same(row.resourceAssessment, checkExecution(row.execution.execution)) + } + else { + if (row.error === null) throw new Error('Missing preflight execution failure') + same(row.assessment, { passed: false, failures: ['execution_failed'], usableReadings: [] }) + same(row.resourceAssessment, null) + } + if (repeated.report) { + if (repeated.error !== null || repeated.report.runs.length !== report.plan.repetitions) throw new Error('Retained repetition count mismatch') + for (const run of repeated.report.runs) { + if (run.runHash !== runHash || run.packageHash !== expected.packageHash || run.input.hash !== fixture.request.inputHash + || run.input.frameCount !== fixture.frames.length || run.responseCount !== fixture.frames.length) throw new Error('Retained determinism identity mismatch') + } + same(repeated.resources, repeated.report.runs.map(run => checkExecution(run.execution))) + same(repeated.report.deterministic, repeated.report.runs.every(run => run.outputHash === repeated.report?.runs[0].outputHash)) + } + else { + if (repeated.error === null) throw new Error('Missing determinism execution failure') + same(repeated.resources, []) + } + return { fixturePassed: row.execution !== null && assessFixtureReadings(fixture.expectations, row.execution.results.flatMap(frame => frame.readings)).passed, + resource: row.execution ? checkExecution(row.execution.execution).status : null, + deterministic: repeated.report?.deterministic ?? false, + repeatResources: repeated.report?.runs.map(run => checkExecution(run.execution).status) ?? [] } + }) + const missingCategories = fixtureCategories.filter(category => !suite.fixtures.some(fixture => fixture.category === category)) + same(report.fixtureReport.missingCategories, missingCategories) + const fixtureStatus = fixtures.some(row => !row.fixturePassed || row.resource === 'failed') + ? 'failed' + : missingCategories.length || fixtures.some(row => row.resource !== 'within_observed_limits') ? 'incomplete' : 'fixture_checks_passed' + same(report.fixtureReport.status, fixtureStatus) + const status = fixtureStatus === 'failed' || fixtures.some(row => !row.deterministic || row.repeatResources.includes('failed')) + ? 'failed' + : fixtureStatus === 'incomplete' || fixtures.some(row => row.repeatResources.some(status => status !== 'within_observed_limits')) ? 'incomplete' : 'external_checks_passed' + same(report.status, status) + return { status, planHash: report.planHash, packageHash: report.packageHash, image: report.image } +} diff --git a/scripts/algorithm-experiments/preflight.ts b/scripts/algorithm-experiments/preflight.ts new file mode 100644 index 00000000..6509b5bf --- /dev/null +++ b/scripts/algorithm-experiments/preflight.ts @@ -0,0 +1,52 @@ +import { z } from 'zod' +import { configHash, dataHash } from '../../src/biometrics/algorithms/canonical' +import { verifyAlgorithmPackage } from '../../src/biometrics/packages/manifest' +import { checkPackageDeterminism } from './determinism' +import { packageFixtureSuiteSchema, runPackageFixtures } from './fixtures' +import { assessPackageResources } from './resource-assessment' +import type { executePackageExperiment } from './container' + +export const packagePreflightPlanSchema = z.object({ schema: z.literal('biometric-package-preflight-plan-v1'), + suiteHash: z.string().regex(/^[a-f0-9]{64}$/), repetitions: z.number().int().min(2).max(5), + configHash: z.string().regex(/^[a-f0-9]{64}$/), hardwareId: z.string().min(1).max(100), + sampleRateHz: z.number().int().positive(), side: z.enum(['left', 'right']), +}).strict() + +/** Execute all evidence in one host-controlled flow. Every fixture is replayed + * independently for determinism, with the same candidate configuration/seed. + * This does not establish pod compatibility, holdout accuracy or installation. */ +export async function runPackagePreflight(options: Pick[0], 'manifest' | 'files' | 'policy' | 'signal'>, + rawPlan: unknown, rawSuite: unknown) { + const plan = packagePreflightPlanSchema.parse(structuredClone(rawPlan)) + const suite = packageFixtureSuiteSchema.parse(structuredClone(rawSuite)) + if (dataHash(suite) !== plan.suiteHash) throw new Error('Preflight suite identity mismatch') + const files = new Map([...options.files].map(([path, bytes]) => [path, Buffer.from(bytes)])) + const verified = verifyAlgorithmPackage(options.manifest, files) + const stable = { manifest: verified.manifest, files, policy: structuredClone(options.policy), signal: options.signal } + for (const fixture of suite.fixtures) { + if (fixture.request.packageHash !== verified.packageHash + || configHash({ config: fixture.request.config, seed: fixture.request.seed }) !== plan.configHash + || fixture.request.hardwareId !== plan.hardwareId || fixture.request.sampleRateHz !== plan.sampleRateHz + || fixture.request.side !== plan.side) throw new Error('Preflight fixture configuration mismatch') + } + options.signal?.throwIfAborted() + const fixtureReport = await runPackageFixtures(stable, suite) + const determinism = [] + for (const fixture of suite.fixtures) { + options.signal?.throwIfAborted() + try { + const report = await checkPackageDeterminism({ ...stable, request: fixture.request, frames: () => fixture.frames }, plan.repetitions) + determinism.push({ fixtureId: fixture.id, report, resources: report.runs.map(run => assessPackageResources(run.execution, suite.resourcePolicy)), error: null }) + } + catch (error) { + options.signal?.throwIfAborted() + determinism.push({ fixtureId: fixture.id, report: null, resources: [], error: error instanceof Error ? error.message.slice(0, 1000) : 'Determinism execution failed' }) + } + } + const failed = fixtureReport.status === 'failed' || determinism.some(row => !row.report?.deterministic || row.resources.some(resource => resource.status === 'failed')) + const incomplete = fixtureReport.status === 'incomplete' || determinism.some(row => row.resources.some(resource => resource.status !== 'within_observed_limits')) + return { schema: 'biometric-package-preflight-report-v1' as const, packageHash: verified.packageHash, manifest: verified.manifest, image: stable.policy.image, + plan, planHash: configHash(plan), fixtureReport, determinism, + status: failed ? 'failed' as const : incomplete ? 'incomplete' as const : 'external_checks_passed' as const, + interpretation: 'requires_heldout_accuracy_target_pod_compatibility_and_reviewed_release' as const } +} diff --git a/scripts/algorithm-experiments/qualification.test.ts b/scripts/algorithm-experiments/qualification.test.ts new file mode 100644 index 00000000..4d758c8d --- /dev/null +++ b/scripts/algorithm-experiments/qualification.test.ts @@ -0,0 +1,71 @@ +// @vitest-environment node +import { describe, expect, it } from 'vitest' +import { configHash } from '../../src/biometrics/algorithms/canonical' +import { combinePackageQualification } from './qualification' +import { fixtureResourceTestPolicy } from './resource-test-policy' +import { managedProtocolSchema } from './managed-protocol' + +type Evidence = Parameters[0] +type Accuracy = Parameters[1] +function fixture() { + const protocol = managedProtocolSchema.parse({ schema: 'biometric-managed-protocol-v1', hardwareId: 'pod4', sampleRateHz: 500, + metricId: 'heart_rate', minimumQuality: 0.8, resourcePolicy: fixtureResourceTestPolicy, + alignment: { version: 'reference-alignment-v1', pointToleranceMs: 0, clockOffsetMs: 0, offsetSource: 'fixed' }, + uncertainty: { version: 'paired-night-percentile-v1', seed: 'test', resamples: 200, minimumNights: 3 } }) + const accuracy: Accuracy = { schema: 'biometric-package-accuracy-assessment-v1', status: 'criteria_met', policyHash: 'a'.repeat(64), failures: [], missing: [], + interpretation: 'reference_agreement_criteria_only_requires_full_package_qualification_and_review' } + // The combiner consumes only the already-verified resource rows for each + // night; comparisons themselves are evaluated by the accuracy assessor. + const resources = (['default', 'current', 'candidate'] as const).map(label => ({ label, + assessment: { status: 'within_observed_limits' as const, policyHash: configHash(protocol.resourcePolicy), failures: [], missing: [] } })) + return { protocol, accuracy, resources } +} +function input(): { evidence: Evidence, accuracy: Accuracy } { + const { protocol, accuracy, resources } = fixture() + // Keep this input's type limited to exactly what combination reads. + const evidence: Evidence = { protocol, preflight: { status: 'external_checks_passed', planHash: 'a'.repeat(64), packageHash: 'b'.repeat(64), image: 'sha256:' + 'c'.repeat(64) }, + nights: [{ nightId: 'night', resources }] } + return { evidence, accuracy } +} +describe('combined external package qualification', () => { + it('combines passing dimensions but never grants installation', () => { + const { evidence, accuracy } = input() + expect(combinePackageQualification(evidence, accuracy)).toMatchObject({ status: 'external_criteria_met', failures: [], missing: [], + installationAuthorized: false, remainingReleaseRequirements: expect.arrayContaining(['target_pod_compatibility_and_resource_qualification', 'reviewed_shipped_registry_release']) }) + }) + it('blocks missing or incomplete preflight even with passing accuracy', () => { + const { evidence, accuracy } = input() + evidence.preflight = null + expect(combinePackageQualification(evidence, accuracy).missing).toContain('no_bound_preflight') + evidence.preflight = { status: 'incomplete', planHash: 'a', packageHash: 'b', image: 'c' } + expect(combinePackageQualification(evidence, accuracy).status).toBe('insufficient_evidence') + }) + it('keeps failures dominant while retaining missing-evidence reasons', () => { + const { evidence, accuracy } = input() + accuracy.status = 'insufficient_evidence' + if (evidence.preflight) evidence.preflight.status = 'failed' + const result = combinePackageQualification(evidence, accuracy) + expect(result).toMatchObject({ status: 'failed', failures: ['preflight_failed'], missing: ['accuracy_evidence_incomplete'] }) + accuracy.status = 'criteria_not_met' + expect(combinePackageQualification(evidence, accuracy).failures).toContain('accuracy_criteria_not_met') + }) + it('requires exact complete resource evidence for every baseline and candidate', () => { + const { evidence, accuracy } = input() + evidence.nights[0].resources.pop() + expect(combinePackageQualification(evidence, accuracy).missing).toContain('night:candidate:resource_evidence_missing_or_ambiguous') + evidence.nights[0].resources.push(evidence.nights[0].resources[0]) + expect(combinePackageQualification(evidence, accuracy).missing).toContain('night:default:resource_evidence_missing_or_ambiguous') + evidence.nights = [] + expect(combinePackageQualification(evidence, accuracy).missing).toContain('no_night_resource_evidence') + }) + it('rejects budget violations, inadequate sampling and substituted policies', () => { + const { evidence, accuracy } = input() + const candidate = evidence.nights[0].resources[2].assessment + candidate.status = 'failed' + candidate.policyHash = 'f'.repeat(64) + const result = combinePackageQualification(evidence, accuracy) + expect(result.failures).toEqual(expect.arrayContaining(['night:candidate:resource_policy_mismatch', 'night:candidate:resource_budget_failed'])) + candidate.status = 'insufficient_evidence' + expect(combinePackageQualification(evidence, accuracy).missing).toContain('night:candidate:resource_evidence_incomplete') + }) +}) diff --git a/scripts/algorithm-experiments/qualification.ts b/scripts/algorithm-experiments/qualification.ts new file mode 100644 index 00000000..4fa69d8f --- /dev/null +++ b/scripts/algorithm-experiments/qualification.ts @@ -0,0 +1,52 @@ +import { configHash, dataHash } from '../../src/biometrics/algorithms/canonical' +import { assessPackageAccuracy } from './accuracy' +import { readManagedEvidence } from './managed-evidence' + +type Evidence = ReturnType +type Accuracy = ReturnType + +/** Combine verified external checks. This is deliberately not a registry entry + * or activation authorization: target-pod and release review are still needed. */ +export function combinePackageQualification(evidence: Pick & { nights: Pick[] }, accuracy: Accuracy) { + const failures: string[] = [], missing: string[] = [] + if (accuracy.status === 'criteria_not_met') failures.push('accuracy_criteria_not_met') + if (accuracy.status === 'insufficient_evidence' || accuracy.missing.length) missing.push('accuracy_evidence_incomplete') + if (!evidence.preflight) missing.push('no_bound_preflight') + else if (evidence.preflight.status === 'failed') failures.push('preflight_failed') + else if (evidence.preflight.status !== 'external_checks_passed') missing.push('preflight_incomplete') + const resourcePolicyHash = configHash(evidence.protocol.resourcePolicy) + if (!evidence.nights.length) missing.push('no_night_resource_evidence') + const resources = evidence.nights.map((night) => { + for (const label of ['default', 'current', 'candidate'] as const) { + const selected = night.resources.filter(row => row.label === label) + if (selected.length !== 1) { + missing.push(`${night.nightId}:${label}:resource_evidence_missing_or_ambiguous`) + continue + } + const assessment = selected[0].assessment + if (assessment.policyHash !== resourcePolicyHash) failures.push(`${night.nightId}:${label}:resource_policy_mismatch`) + if (assessment.status === 'failed') failures.push(`${night.nightId}:${label}:resource_budget_failed`) + if (assessment.status === 'insufficient_evidence' || ('missing' in assessment && assessment.missing?.length)) missing.push(`${night.nightId}:${label}:resource_evidence_incomplete`) + } + return { nightId: night.nightId, assessments: night.resources } + }) + return { schema: 'biometric-package-qualification-assessment-v1' as const, + status: failures.length ? 'failed' as const : missing.length ? 'insufficient_evidence' as const : 'external_criteria_met' as const, + failures, missing, accuracy, preflight: evidence.preflight, resourcePolicyHash, resources, + installationAuthorized: false as const, + remainingReleaseRequirements: ['representative_fixture_review', 'target_pod_compatibility_and_resource_qualification', + 'prospective_health_and_reference_comparison', 'reviewed_shipped_registry_release'] as const, + interpretation: 'external_evidence_snapshot_not_installation_or_activation_authorization' as const } +} + +/** The public orchestration entry point accepts ledger identities, never a + * client-authored success report. Re-read for every review: later overlapping + * holdout access changes the evidence and therefore the assessment hash. */ +export function assessManagedQualification(...args: Parameters) { + const evidence = readManagedEvidence(...args) + const accuracy = assessPackageAccuracy(evidence) + const qualification = combinePackageQualification(evidence, accuracy) + const snapshot = { trialId: evidence.trialId, reportHash: evidence.reportHash, protocolHash: configHash(evidence.protocol), + request: evidence.request, currentEvidence: evidence.currentEvidence, qualification } + return { ...snapshot, assessmentHash: dataHash(snapshot) } +} diff --git a/scripts/algorithm-experiments/recovery.test.ts b/scripts/algorithm-experiments/recovery.test.ts new file mode 100644 index 00000000..fecb9f17 --- /dev/null +++ b/scripts/algorithm-experiments/recovery.test.ts @@ -0,0 +1,70 @@ +// @vitest-environment node +import { describe, expect, it } from 'vitest' +import { expiredOwnedContainer, recoveryLabels } from './recovery' + +const now = 1000000 +const id = 'a'.repeat(64) +const valid = (kind: 'experiment' | 'environment' = 'experiment') => ({ Id: id, + Name: `/sleepypod-${kind}-12345678-1234-1234-1234-123456789abc`, + Config: { Labels: recoveryLabels(kind, 1000, now) } }) + +describe('expired experiment recovery ownership', () => { + it('preserves live controllers through their deadline and cleanup grace', () => { + expect(expiredOwnedContainer(valid(), now + 120999)).toBeNull() + expect(expiredOwnedContainer(valid(), now + 121000)).toBe(id) + expect(expiredOwnedContainer(valid('environment'), now + 121000)).toBe(id) + }) + it('requires exact version, host owner, kind and immutable full ID', () => { + const container = valid() + for (const [field, value] of [['version', '2'], ['owner', 'foreign'], ['kind', 'environment'], ['expires', 'NaN'], ['expires', '0'], ['expires', '9999999999999999']]) { + expect(expiredOwnedContainer({ ...container, Config: { Labels: { ...container.Config.Labels, + [`org.sleepypod.experiment.${field}`]: value } } }, now + 121000)).toBeNull() + } + for (const value of [null, {}, { ...container, Id: 'short' }, { ...container, Name: '/unrelated' }, + { ...container, Config: {} }, { ...container, Name: '/sleepypod-experiment-not-a-uuid' }]) { + expect(expiredOwnedContainer(value, now + 121000)).toBeNull() + } + }) + it('rejects invalid clock and budget inputs', () => { + for (const value of [-1, 0, NaN, Infinity, 3600001]) expect(() => recoveryLabels('experiment', value)).toThrow() + expect(() => recoveryLabels('experiment', 1000, -1)).toThrow() + expect(expiredOwnedContainer(valid(), NaN)).toBeNull() + }) +}) + +// Actual daemon test: an abandoned process remains alive until recovery, while +// an unexpired controller and an unlabelled lookalike survive the same sweep. +it.skipIf(!process.env.BIOMETRIC_EXPERIMENT_TEST_IMAGE)('removes an overdue running container and preserves other containers', async () => { + const { execFile } = await import('node:child_process') + const { promisify } = await import('node:util') + const { randomUUID } = await import('node:crypto') + const { recoverExpiredContainers } = await import('./recovery') + const exec = promisify(execFile) + const containers: string[] = [] + const clock = Date.now() + const image = process.env.BIOMETRIC_EXPERIMENT_TEST_IMAGE + if (!image) throw new Error('Test image required') + try { + for (const labels of [recoveryLabels('experiment', 1000, clock - 122000), recoveryLabels('experiment', 1000, clock), {}]) { + const result = await exec('docker', ['run', '-d', '--name', `sleepypod-experiment-${randomUUID()}`, + ...Object.entries(labels).flatMap(([key, value]) => ['--label', `${key}=${value}`]), + '--network', 'none', '--read-only', '--cap-drop', 'ALL', '--security-opt', 'no-new-privileges=true', + '--user', '65534:65534', '--pids-limit', '16', '--memory', '134217728', '--cpus', '1', + '--entrypoint', 'python', image, '-I', '-B', '-c', 'import time; time.sleep(120)'], + { timeout: 10000 }) + containers.push(result.stdout.trim()) + } + const report = await recoverExpiredContainers(clock) + expect(report.removed).toContain(containers[0]) + expect(report.removed).not.toContain(containers[1]) + expect(report.removed).not.toContain(containers[2]) + await expect(exec('docker', ['inspect', containers[0]])).rejects.toThrow() + for (const container of containers.slice(1)) { + expect((await exec('docker', ['inspect', '--format', '{{.State.Running}}', container])).stdout.trim()).toBe('true') + } + expect((await recoverExpiredContainers(clock)).removed).not.toContain(containers[0]) + } + finally { + for (const container of containers) await exec('docker', ['rm', '-f', container]).catch(() => undefined) + } +}, 60000) diff --git a/scripts/algorithm-experiments/recovery.ts b/scripts/algorithm-experiments/recovery.ts new file mode 100644 index 00000000..f1758ef3 --- /dev/null +++ b/scripts/algorithm-experiments/recovery.ts @@ -0,0 +1,68 @@ +import { execFile } from 'node:child_process' +import { promisify } from 'node:util' + +const exec = promisify(execFile) +const prefix = 'org.sleepypod.experiment' +const owner = `${process.getuid?.() ?? 'unknown'}` +const uuid = '[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}' +const namePattern = new RegExp(`^/sleepypod-(experiment|environment)-${uuid}$`) + +/** Immutable Docker labels survive a terminated controller. Leave two minutes + * beyond the hard deadline for startup, measurement and ordinary cleanup. */ +export function recoveryLabels(kind: 'experiment' | 'environment', maximumWallMs: number, now = Date.now()) { + if (!Number.isSafeInteger(maximumWallMs) || maximumWallMs <= 0 || maximumWallMs > 3600000 + || !Number.isSafeInteger(now) || now < 0 + || !Number.isSafeInteger(now + maximumWallMs + 120000)) throw new Error('Invalid recovery deadline') + return { [`${prefix}.version`]: '1', [`${prefix}.owner`]: owner, + [`${prefix}.kind`]: kind, [`${prefix}.expires`]: String(now + maximumWallMs + 120000) } +} +export function recoveryLabelArguments(kind: 'experiment' | 'environment', maximumWallMs: number) { + return Object.entries(recoveryLabels(kind, maximumWallMs)).flatMap(([key, value]) => ['--label', `${key}=${value}`]) +} + +/** Fail closed on malformed metadata; names alone never establish ownership. */ +export function expiredOwnedContainer(raw: unknown, now: number): string | null { + if (!Number.isSafeInteger(now) || now < 0 || !raw || typeof raw !== 'object') return null + const value = raw as { Id?: unknown, Name?: unknown, Config?: { Labels?: Record } } + if (typeof value.Id !== 'string' || !/^[a-f0-9]{64}$/.test(value.Id) + || typeof value.Name !== 'string' || !namePattern.test(value.Name)) return null + const labels = value.Config?.Labels + if (!labels || labels[`${prefix}.version`] !== '1' || labels[`${prefix}.owner`] !== owner) return null + const kind = value.Name.startsWith('/sleepypod-experiment-') ? 'experiment' : 'environment' + if (labels[`${prefix}.kind`] !== kind) return null + const expires = labels[`${prefix}.expires`] + if (typeof expires !== 'string' || !/^[1-9][0-9]{0,15}$/.test(expires) + || !Number.isSafeInteger(Number(expires)) || Number(expires) > now) return null + return value.Id +} + +/** Operates only on the selected Docker daemon and the invoking host UID. + * Remove by inspected immutable ID, never by a reusable container name. */ +export async function recoverExpiredContainers(now = Date.now()) { + if (!Number.isSafeInteger(now) || now < 0) throw new Error('Invalid recovery time') + const { stdout } = await exec('docker', ['ps', '-aq', '--no-trunc', '--filter', `label=${prefix}.version=1`, + '--filter', `label=${prefix}.owner=${owner}`], { timeout: 10000, maxBuffer: 1024 * 1024 }) + const ids = stdout.trim() ? stdout.trim().split(/\s+/) : [] + if (ids.length > 4096 || ids.some(id => !/^[a-f0-9]{64}$/.test(id))) throw new Error('Invalid recovery inventory') + const removed: string[] = [] + for (const id of ids) { + let inspection + try { + inspection = await exec('docker', ['inspect', '--type', 'container', id], { timeout: 10000, maxBuffer: 1024 * 1024 }) + } + catch (error) { + if (error instanceof Error && /No such (object|container):/.test(error.message)) continue + throw error + } + const values: unknown = JSON.parse(inspection.stdout) + if (!Array.isArray(values) || values.length !== 1 || expiredOwnedContainer(values[0], now) !== id) continue + try { + await exec('docker', ['rm', '-f', id], { timeout: 10000, maxBuffer: 65536 }) + removed.push(id) + } + catch (error) { + if (!(error instanceof Error) || !error.message.includes('No such container:')) throw error + } + } + return { schema: 'biometric-container-recovery-v1' as const, checkedAt: now, inspected: ids.length, removed } +} diff --git a/scripts/algorithm-experiments/resource-assessment.test.ts b/scripts/algorithm-experiments/resource-assessment.test.ts new file mode 100644 index 00000000..4d29364e --- /dev/null +++ b/scripts/algorithm-experiments/resource-assessment.test.ts @@ -0,0 +1,39 @@ +import { fixtureResourceTestPolicy } from './resource-test-policy' +import { describe, expect, it } from 'vitest' +import { assessPackageResources } from './resource-assessment' +import { ResourceObservations } from './resources' +function fixture() { + const observations = new ResourceObservations(0) + observations.record({ sourceTimestampMs: 1000, cpuMs: 10, memoryBytes: 1000000, processes: 1 }, 100) + return { measurements: observations.snapshot(200), limits: { cpuMs: 1000, wallMs: 1000, memoryBytes: 2000000, processes: 1, outputBytes: 10000 }, wallMs: 200 } +} +describe('package fixture resource assessment', () => { + it('binds explicit policy and accepts complete measurements within enforced limits', () => { + const result = assessPackageResources(fixture(), fixtureResourceTestPolicy) + expect(result.status).toBe('within_observed_limits') + expect(assessPackageResources(fixture(), { ...fixtureResourceTestPolicy, maximumCpuMs: 6000 }).policyHash).not.toBe(result.policyHash) + }) + it('does not turn missing, stale or inconsistent evidence into passing evidence', () => { + const input = fixture() + for (const measurements of [null, { ...input.measurements, attempts: 2, allAttemptsMeasured: false }, + { ...input.measurements, samples: 2 }, { ...input.measurements, source: 'candidate_claim' }]) { + expect(assessPackageResources({ ...input, measurements }, fixtureResourceTestPolicy).status).toBe('insufficient_evidence') + } + expect(assessPackageResources(input, { ...fixtureResourceTestPolicy, minimumSamples: 2 }).status).toBe('insufficient_evidence') + expect(assessPackageResources(input, { ...fixtureResourceTestPolicy, maximumObservationGapMs: 50 }).status).toBe('insufficient_evidence') + }) + it('fails measured overruns and limits broader than the reviewed policy', () => { + const input = fixture() + for (const changed of [{ ...input, wallMs: 6000 }, { ...input, measurements: { ...input.measurements, maximumObservedCpuMs: 6000 } }, + { ...input, measurements: { ...input.measurements, peakObservedMemoryBytes: 100000000 } }, + { ...input, measurements: { ...input.measurements, maximumObservedProcesses: 2 } }, + { ...input, limits: { ...input.limits, memoryBytes: 100000000 } }]) { + expect(assessPackageResources(changed, fixtureResourceTestPolicy).status).toBe('failed') + } + }) + it('retains a known budget failure even if observation coverage is incomplete', () => { + const input = fixture() + const result = assessPackageResources({ ...input, wallMs: 6000, measurements: { ...input.measurements, attempts: 2, allAttemptsMeasured: false } }, fixtureResourceTestPolicy) + expect(result).toMatchObject({ status: 'failed', failures: ['wall_budget_exceeded'], missing: ['unmeasured_attempts'] }) + }) +}) diff --git a/scripts/algorithm-experiments/resource-assessment.ts b/scripts/algorithm-experiments/resource-assessment.ts new file mode 100644 index 00000000..49d2c67e --- /dev/null +++ b/scripts/algorithm-experiments/resource-assessment.ts @@ -0,0 +1,48 @@ +import { z } from 'zod' +import { configHash } from '../../src/biometrics/algorithms/canonical' + +const positive = z.number().finite().positive() +const count = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) +const resourceLimitsSchema = z.object({ cpuMs: positive, wallMs: positive, memoryBytes: positive.int(), processes: positive.int(), outputBytes: positive.int() }).strict() +export const packageResourcePolicySchema = z.object({ + schema: z.literal('biometric-package-resource-policy-v1'), + maximumWallMs: positive, maximumCpuMs: positive, maximumMemoryBytes: positive.int(), maximumProcesses: positive.int(), + minimumSamples: positive.int(), maximumObservationGapMs: positive, + evidenceMode: z.literal('sampled_counters_and_enforced_limits'), +}).strict() +const measurementSchema = z.object({ + schema: z.literal('biometric-package-resources-v1'), source: z.literal('host_docker_engine_stats'), + attempts: count, samples: count, allAttemptsMeasured: z.boolean(), + maximumObservedCpuMs: z.number().finite().nonnegative().nullable(), peakObservedMemoryBytes: positive.nullable(), maximumObservedProcesses: positive.int().nullable(), + maximumObservationGapMs: z.number().finite().nonnegative(), elapsedMs: z.number().finite().nonnegative(), + exactLifetimeCpu: z.literal(false), exactPeakMemory: z.literal(false), memoryIncludesCache: z.literal(true), +}).strict().refine(value => value.samples <= value.attempts + && value.allAttemptsMeasured === (value.attempts > 0 && value.samples === value.attempts) + && value.maximumObservationGapMs <= value.elapsedMs + && (value.samples === 0 + ? value.maximumObservedCpuMs === null && value.peakObservedMemoryBytes === null && value.maximumObservedProcesses === null + : value.maximumObservedCpuMs !== null && value.peakObservedMemoryBytes !== null && value.maximumObservedProcesses !== null), 'Inconsistent resource observations') + +/** Assesses actual host evidence against explicit reviewed fixture criteria. + * No default thresholds or implicit target-pod fitness claim. */ +export function assessPackageResources(input: { measurements: unknown, limits: unknown, wallMs: unknown }, policyInput: unknown) { + const policy = packageResourcePolicySchema.parse(policyInput) + const policyHash = configHash(policy) + const measurements = measurementSchema.safeParse(input.measurements) + const limits = resourceLimitsSchema.safeParse(input.limits) + const wall = z.number().finite().nonnegative().safeParse(input.wallMs) + if (!measurements.success || !limits.success || !wall.success) return { status: 'insufficient_evidence' as const, policyHash, failures: ['invalid_or_missing_resource_evidence'] } + const evidence = measurements.data + const failures: string[] = [], missing: string[] = [] + if (!evidence.allAttemptsMeasured) missing.push('unmeasured_attempts') + if (evidence.samples < policy.minimumSamples) missing.push('insufficient_samples') + if (evidence.maximumObservationGapMs > policy.maximumObservationGapMs) missing.push('observation_gap') + if (wall.data > policy.maximumWallMs) failures.push('wall_budget_exceeded') + if (evidence.maximumObservedCpuMs !== null && evidence.maximumObservedCpuMs > policy.maximumCpuMs) failures.push('cpu_budget_exceeded') + if (evidence.peakObservedMemoryBytes !== null && evidence.peakObservedMemoryBytes > policy.maximumMemoryBytes) failures.push('memory_budget_exceeded') + if (evidence.maximumObservedProcesses !== null && evidence.maximumObservedProcesses > policy.maximumProcesses) failures.push('process_budget_exceeded') + if (limits.data.cpuMs > policy.maximumCpuMs || limits.data.wallMs > policy.maximumWallMs + || limits.data.memoryBytes > policy.maximumMemoryBytes || limits.data.processes > policy.maximumProcesses) failures.push('enforced_limits_exceed_policy') + return { status: failures.length ? 'failed' as const : missing.length ? 'insufficient_evidence' as const : 'within_observed_limits' as const, + policyHash, failures, missing } +} diff --git a/scripts/algorithm-experiments/resource-test-policy.ts b/scripts/algorithm-experiments/resource-test-policy.ts new file mode 100644 index 00000000..de77cec4 --- /dev/null +++ b/scripts/algorithm-experiments/resource-test-policy.ts @@ -0,0 +1,3 @@ +// Synthetic test criteria only; not target-pod qualification. +export const fixtureResourceTestPolicy = { schema: 'biometric-package-resource-policy-v1', maximumWallMs: 5000, maximumCpuMs: 5000, + maximumMemoryBytes: 64 * 1024 * 1024, maximumProcesses: 1, minimumSamples: 1, maximumObservationGapMs: 5000, evidenceMode: 'sampled_counters_and_enforced_limits' } diff --git a/scripts/algorithm-experiments/resources.test.ts b/scripts/algorithm-experiments/resources.test.ts new file mode 100644 index 00000000..1266363e --- /dev/null +++ b/scripts/algorithm-experiments/resources.test.ts @@ -0,0 +1,46 @@ +// @vitest-environment node +import { describe, expect, it } from 'vitest' +import { ResourceObservations, startResourceMonitor } from './resources' +const sample = { sourceTimestampMs: 1000, cpuMs: 10, memoryBytes: 1000000, processes: 1 } +describe('host experiment resource evidence', () => { + it('retains maxima and includes initial, interior and final observation gaps', () => { + const readings = new ResourceObservations(0) + readings.record(sample, 100) + readings.record({ ...sample, sourceTimestampMs: 2000, cpuMs: 20, memoryBytes: 900000, processes: 2 }, 400) + expect(readings.snapshot(1000)).toMatchObject({ attempts: 2, samples: 2, allAttemptsMeasured: true, + maximumObservedCpuMs: 20, peakObservedMemoryBytes: 1000000, maximumObservedProcesses: 2, + maximumObservationGapMs: 600, exactLifetimeCpu: false, exactPeakMemory: false }) + }) + it('marks failed, stale, regressing and invalid observations incomplete', () => { + const readings = new ResourceObservations(0) + readings.record(sample, 100) + readings.record(null, 200) + readings.record(sample, 300) + readings.record({ ...sample, sourceTimestampMs: 2000, cpuMs: 1 }, 400) + readings.record({ ...sample, sourceTimestampMs: 2000, memoryBytes: NaN }, 500) + expect(readings.snapshot(1000)).toMatchObject({ attempts: 5, samples: 1, allAttemptsMeasured: false, maximumObservationGapMs: 900 }) + expect(new ResourceObservations(0).snapshot(1000)).toMatchObject({ samples: 0, allAttemptsMeasured: false, + maximumObservedCpuMs: null, peakObservedMemoryBytes: null }) + }) + it('coalesces concurrent reads and waits for a pending observation at stop', async () => { + let resolve: (value: typeof sample) => void = () => undefined + let calls = 0 + const monitor = startResourceMonitor(() => { + calls++ + return new Promise((done) => { + resolve = done + }) + }, 0) + const first = monitor.sample(), second = monitor.sample() + expect(first).toBe(second) + const stopping = monitor.stop() + resolve(sample) + expect(await stopping).toMatchObject({ attempts: 1, samples: 1 }) + expect(calls).toBe(1) + }) + it('reports an unavailable transport rather than fabricated zero usage', async () => { + const monitor = startResourceMonitor(null, 0) + await monitor.sample() + expect(await monitor.stop()).toMatchObject({ attempts: 1, samples: 0, allAttemptsMeasured: false, maximumObservedCpuMs: null }) + }) +}) diff --git a/scripts/algorithm-experiments/resources.ts b/scripts/algorithm-experiments/resources.ts new file mode 100644 index 00000000..9413fc19 --- /dev/null +++ b/scripts/algorithm-experiments/resources.ts @@ -0,0 +1,121 @@ +import { execFile } from 'node:child_process' +import { get } from 'node:http' +import { promisify } from 'node:util' +import { performance } from 'node:perf_hooks' +import { z } from 'zod' + +const exec = promisify(execFile) +const counter = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) +const statsSchema = z.object({ read: z.string(), id: z.string().regex(/^[a-f0-9]{64}$/), + cpu_stats: z.object({ cpu_usage: z.object({ total_usage: counter }) }), + memory_stats: z.object({ usage: counter.positive() }), pids_stats: z.object({ current: counter.positive() }), +}) + +function dockerJson(socketPath: string, path: string): Promise { + return new Promise((resolve, reject) => { + const request = get({ socketPath, path }, (response) => { + if (response.statusCode !== 200) { + response.resume() + reject(new Error('Docker measurement unavailable')) + return + } + const chunks: Buffer[] = [] + let length = 0 + response.on('data', (chunk: Buffer) => { + length += chunk.length + if (length > 128 * 1024) request.destroy(new Error('Docker measurement byte limit')) + else chunks.push(chunk) + }) + response.on('error', reject) + response.on('end', () => { + try { + resolve(JSON.parse(Buffer.concat(chunks).toString('utf8'))) + } + catch (error) { reject(error) } + }) + }) + const timeout = setTimeout(() => request.destroy(new Error('Docker measurement timeout')), 1500) + request.on('error', reject) + request.on('close', () => clearTimeout(timeout)) + }) +} + +/** Independent host evidence. Never accepts counters emitted by candidate code. + * Unsupported remote Docker transports remain explicitly unmeasured. */ +export async function dockerResourceReader(containerId: string) { + if (!/^[a-f0-9]{64}$/.test(containerId)) throw new Error('Invalid measured container identity') + const host = !process.env.DOCKER_CONTEXT && process.env.DOCKER_HOST + ? process.env.DOCKER_HOST + : JSON.parse((await exec('docker', ['context', 'inspect', '--format', '{{json .Endpoints.docker.Host}}'], { timeout: 5000, maxBuffer: 16384 })).stdout) as string + if (!host.startsWith('unix:///')) throw new Error('Resource measurement requires a local Unix Docker endpoint') + const socket = host.slice('unix://'.length) + const version = z.object({ ApiVersion: z.string().regex(/^1\.\d+$/) }).parse(await dockerJson(socket, '/version')).ApiVersion + if (Number(version.split('.')[1]) < 41) throw new Error('Docker API does not support one-shot resource measurements') + return async () => { + const value = statsSchema.parse(await dockerJson(socket, `/v${version}/containers/${containerId}/stats?stream=false&one-shot=true`)) + if (value.id !== containerId) throw new Error('Measured container identity mismatch') + const sourceTimestampMs = Date.parse(value.read) + if (!Number.isFinite(sourceTimestampMs) || sourceTimestampMs <= 0) throw new Error('Invalid resource observation timestamp') + return { sourceTimestampMs, cpuMs: value.cpu_stats.cpu_usage.total_usage / 1e6, memoryBytes: value.memory_stats.usage, processes: value.pids_stats.current } + } +} + +interface Observation { sourceTimestampMs: number, cpuMs: number, memoryBytes: number, processes: number } +export class ResourceObservations { + private attempts = 0 + private samples = 0 + private peakMemoryBytes: number | null = null + private maximumCpuMs: number | null = null + private maximumProcesses: number | null = null + private sourceTimestampMs = 0 + private lastAt: number + private maximumGapMs = 0 + constructor(private readonly startedAt: number) { this.lastAt = startedAt } + record(value: Observation | null, at: number) { + this.attempts++ + if (!value || ![value.cpuMs, value.memoryBytes, value.processes].every(number => Number.isFinite(number) && number >= 0)) return + if (!Number.isFinite(value.sourceTimestampMs) || value.sourceTimestampMs <= this.sourceTimestampMs || !Number.isFinite(at) || at < this.lastAt) return + if (this.maximumCpuMs !== null && value.cpuMs < this.maximumCpuMs) return + this.sourceTimestampMs = value.sourceTimestampMs + this.samples++ + this.maximumGapMs = Math.max(this.maximumGapMs, at - this.lastAt) + this.lastAt = at + this.maximumCpuMs = Math.max(this.maximumCpuMs ?? 0, value.cpuMs) + this.peakMemoryBytes = Math.max(this.peakMemoryBytes ?? 0, value.memoryBytes) + this.maximumProcesses = Math.max(this.maximumProcesses ?? 0, value.processes) + } + + snapshot(endedAt: number) { + return { schema: 'biometric-package-resources-v1' as const, source: 'host_docker_engine_stats' as const, + attempts: this.attempts, samples: this.samples, allAttemptsMeasured: this.attempts > 0 && this.samples === this.attempts, + maximumObservedCpuMs: this.maximumCpuMs, peakObservedMemoryBytes: this.peakMemoryBytes, maximumObservedProcesses: this.maximumProcesses, + maximumObservationGapMs: Math.max(this.maximumGapMs, endedAt - this.lastAt), elapsedMs: endedAt - this.startedAt, + exactLifetimeCpu: false, exactPeakMemory: false, memoryIncludesCache: true } + } +} + +export function startResourceMonitor(read: (() => Promise) | null, startedAt: number) { + const observations = new ResourceObservations(startedAt) + let pending: Promise | null = null + let stopped = false + const sample = () => { + if (stopped) return Promise.resolve() + if (pending) return pending + pending = (async () => { + try { + observations.record(read ? await read() : null, performance.now()) + } + catch { observations.record(null, performance.now()) } + })().finally(() => { pending = null }) + return pending + } + const timer = setInterval(() => { + if (!stopped) void sample() + }, 250) + return { sample, stop: async () => { + stopped = true + clearInterval(timer) + if (pending) await pending + return observations.snapshot(performance.now()) + } } +} diff --git a/scripts/algorithm-experiments/staging-recovery.test.ts b/scripts/algorithm-experiments/staging-recovery.test.ts new file mode 100644 index 00000000..a2a6ef11 --- /dev/null +++ b/scripts/algorithm-experiments/staging-recovery.test.ts @@ -0,0 +1,66 @@ +// @vitest-environment node +import { mkdtemp, readFile, writeFile, rm, stat, symlink } from 'node:fs/promises' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import { recordExperimentStaging, recoverExperimentStaging } from './staging-recovery' +const docker = vi.hoisted(() => ({ mode: 'absent', calls: 0 })) +vi.mock('node:child_process', () => ({ execFile: (_file: string, _args: string[], _options: unknown, callback: (error: Error | null, stdout: string, stderr: string) => void) => { + docker.calls++ + callback(docker.mode === 'absent' ? new Error('No such container: test') : docker.mode === 'offline' ? new Error('Docker unavailable') : null, '[]', '') +} })) +let base: string +const deadPid = 2147483647 +beforeEach(async () => { + base = await mkdtemp(join(tmpdir(), 'staging-recovery-test-')) + docker.mode = 'absent' + docker.calls = 0 + vi.spyOn(process, 'kill').mockImplementation((pid) => { + if (pid === deadPid) throw Object.assign(new Error('gone'), { code: 'ESRCH' }) + return true + }) +}) +afterEach(async () => { + vi.restoreAllMocks() + await rm(base, { recursive: true, force: true }) +}) +async function fixture(changes: Record = {}) { + const root = await mkdtemp(join(base, 'sleepypod-package-')) + await recordExperimentStaging(root, 'sleepypod-experiment-12345678-1234-1234-1234-123456789abc', 1000, 1000) + const path = join(root, '.sleepypod-staging.json') + const receipt = JSON.parse(await readFile(path, 'utf8')) + await writeFile(path, JSON.stringify({ ...receipt, controllerPid: deadPid, ...changes })) + await writeFile(join(root, 'input.jsonl'), 'private input', { mode: 0o600 }) + return root +} +it('removes only an expired, owned staging directory after its controller and container disappear', async () => { + const root = await fixture() + expect((await recoverExperimentStaging(121999, base)).removed).toEqual([]) + expect(docker.calls).toBe(0) + expect((await recoverExperimentStaging(122000, base)).removed).toHaveLength(1) + await expect(stat(root)).rejects.toMatchObject({ code: 'ENOENT' }) + expect((await recoverExperimentStaging(122000, base)).removed).toEqual([]) +}) +it.each([{ controllerPid: process.pid }, { inode: 1 }, { owner: 999999 }, { schema: 'unknown' }])('preserves unproven ownership or a live controller: %j', async (changes) => { + const root = await fixture(changes) + expect((await recoverExperimentStaging(122000, base)).removed).toEqual([]) + expect((await stat(root)).isDirectory()).toBe(true) + expect(docker.calls).toBe(0) +}) +it('retains files when Docker still has the container or is unavailable', async () => { + const root = await fixture() + docker.mode = 'present' + expect((await recoverExperimentStaging(122000, base)).removed).toEqual([]) + docker.mode = 'offline' + await expect(recoverExperimentStaging(122000, base)).rejects.toThrow('Docker unavailable') + expect((await stat(root)).isDirectory()).toBe(true) +}) +it('does not follow symlinked ownership markers or infer legacy directory ownership', async () => { + const root = await fixture() + await rm(join(root, '.sleepypod-staging.json')) + const legacy = await mkdtemp(join(base, 'sleepypod-package-')) + await symlink('/etc/passwd', join(root, '.sleepypod-staging.json')) + expect((await recoverExperimentStaging(122000, base)).removed).toEqual([]) + expect((await stat(legacy)).isDirectory()).toBe(true) + expect(docker.calls).toBe(0) +}) diff --git a/scripts/algorithm-experiments/staging-recovery.ts b/scripts/algorithm-experiments/staging-recovery.ts new file mode 100644 index 00000000..bd16d7e9 --- /dev/null +++ b/scripts/algorithm-experiments/staging-recovery.ts @@ -0,0 +1,106 @@ +import { execFile } from 'node:child_process' +import { constants } from 'node:fs' +import { lstat, open, opendir, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { basename, join } from 'node:path' +import { promisify } from 'node:util' +import { z } from 'zod' + +const exec = promisify(execFile) +const marker = '.sleepypod-staging.json' +const directoryName = /^sleepypod-package-[a-zA-Z0-9]{6}$/ +const integer = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) +const receiptSchema = z.object({ schema: z.literal('biometric-private-staging-v1'), + directory: z.string().regex(directoryName), device: integer, inode: integer, owner: integer, + controllerPid: integer.positive().max(2147483647), expiresAtMs: integer, + containerName: z.string().regex(/^sleepypod-experiment-[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/), +}).strict() + +/** Written before input/package bytes. Legacy unmarked directories are never + * inferred to be owned from a matching name alone. */ +export async function recordExperimentStaging(root: string, containerName: string, maximumWallMs: number, now = Date.now()) { + z.number().int().min(1).max(3600000).parse(maximumWallMs) + integer.parse(now) + const info = await lstat(root) + if (!info.isDirectory() || info.isSymbolicLink() || info.mode & 0o077 || info.uid !== process.getuid?.()) throw new Error('Staging directory must be private and owned') + const receipt = receiptSchema.parse({ schema: 'biometric-private-staging-v1', directory: basename(root), device: info.dev, inode: info.ino, + owner: info.uid, controllerPid: process.pid, expiresAtMs: now + maximumWallMs + 120000, containerName }) + const file = await open(join(root, marker), constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600) + try { + await file.writeFile(JSON.stringify(receipt)) + await file.sync() + } + finally { await file.close() } +} +function processAbsent(pid: number) { + try { + process.kill(pid, 0) + } + catch (error) { return (error as NodeJS.ErrnoException).code === 'ESRCH' } + return false +} +async function containerAbsent(name: string) { + try { + await exec('docker', ['inspect', '--type', 'container', name], { timeout: 10000, maxBuffer: 1024 * 1024 }) + return false + } + catch (error) { + if (error instanceof Error && /No such (object|container):/.test(error.message)) return true + throw error // An unavailable daemon is not proof of absence. + } +} +/** Trusted host entry point; base is injectable for isolated tests. Container + * recovery must run first. Only an expired receipt, dead controller and absent + * named container permit deletion. PID reuse conservatively retains a directory. */ +export async function recoverExperimentStaging(now = Date.now(), base = tmpdir()) { + integer.parse(now) + const removed: string[] = [] + let scanned = 0, candidates = 0 + const entries = await opendir(base) + for await (const entry of entries) { + if (++scanned > 16384) throw new Error('Staging recovery inventory exceeds limit') + if (!directoryName.test(entry.name)) continue + if (++candidates > 128) throw new Error('Staging recovery candidate limit exceeded') + const root = join(base, entry.name) + let before, receipt, markerIdentity + try { + before = await lstat(root) + if (!before.isDirectory() || before.isSymbolicLink() || before.mode & 0o077 || before.uid !== process.getuid?.()) continue + const file = await open(join(root, marker), constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK) + try { + markerIdentity = await file.stat() + if (!markerIdentity.isFile() || markerIdentity.nlink !== 1 || markerIdentity.mode & 0o077 || markerIdentity.uid !== before.uid || markerIdentity.size > 4096) continue + const bytes = Buffer.alloc(4097) + const { bytesRead } = await file.read(bytes, 0, bytes.length, 0) + if (bytesRead !== markerIdentity.size) continue + receipt = receiptSchema.parse(JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes.subarray(0, bytesRead)))) + } + finally { await file.close() } + } + catch { continue } // Unknown or damaged ownership never authorizes deletion. + if (receipt.directory !== entry.name || receipt.device !== before.dev || receipt.inode !== before.ino || receipt.owner !== before.uid + || receipt.expiresAtMs > now || !processAbsent(receipt.controllerPid)) continue + if (!await containerAbsent(receipt.containerName)) continue + let current, currentMarker + try { + current = await lstat(root) + currentMarker = await lstat(join(root, marker)) + } + catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') continue + throw error + } + if (current.dev !== before.dev || current.ino !== before.ino || current.mode !== before.mode || current.uid !== before.uid + || currentMarker.dev !== markerIdentity.dev || currentMarker.ino !== markerIdentity.ino || currentMarker.size !== markerIdentity.size + || currentMarker.mtimeMs !== markerIdentity.mtimeMs || !processAbsent(receipt.controllerPid)) continue + try { + await rm(root, { recursive: true }) + } + catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') continue + throw error + } + removed.push(entry.name) + } + return { schema: 'biometric-staging-recovery-v1' as const, checkedAt: now, scanned, candidates, removed } +} diff --git a/scripts/algorithm-experiments/workspace.test.ts b/scripts/algorithm-experiments/workspace.test.ts new file mode 100644 index 00000000..75e5f7aa --- /dev/null +++ b/scripts/algorithm-experiments/workspace.test.ts @@ -0,0 +1,104 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import * as filesystem from 'node:fs/promises' +import { spawn } from 'node:child_process' +import { once } from 'node:events' +import { mkdtemp, readdir, rm, symlink, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import { withExperimentWorkspace } from './workspace' + +vi.mock('node:fs/promises', async (original) => { + const actual = await original() + return { ...actual, statfs: vi.fn(actual.statfs) } +}) + +let root: string +const policy = { maximumReportBytes: 80, maximumReportFiles: 2, maximumSingleReportBytes: 60, + maximumDatabaseBytes: 65536, minimumFreeBytes: 0 } +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'experiment-workspace-test-')) +}) +afterEach(async () => { + vi.restoreAllMocks() + await rm(root, { recursive: true, force: true }) +}) +it('rejects concurrent workspace owners and releases on ordinary failure', async () => { + await withExperimentWorkspace(root, policy, async () => { + await expect(withExperimentWorkspace(root, policy, async () => null)).rejects.toThrow('locked') + }) + await expect(withExperimentWorkspace(root, policy, async () => { + throw new Error('failed') + })).rejects.toThrow('failed') + expect(await withExperimentWorkspace(root, policy, async () => 'available')).toBe('available') +}) +it('bounds report bytes/counts before publication without deleting prior evidence', async () => { + await withExperimentWorkspace(root, policy, async (workspace) => { + await workspace.publishReport(join(workspace.reports, 'first.json'), { value: 'a'.repeat(30) }) + await expect(workspace.publishReport(join(workspace.reports, 'second.json'), { value: 'b'.repeat(30) })).rejects.toThrow('byte quota') + await expect(workspace.publishReport(join(workspace.reports, 'oversize.json'), { value: 'c'.repeat(60) })).rejects.toThrow('single-report') + await workspace.publishReport(join(workspace.reports, 'small.json'), {}) + await expect(workspace.publishReport(join(workspace.reports, 'third.json'), {})).rejects.toThrow('file quota') + expect((await readdir(workspace.reports)).sort()).toEqual(['first.json', 'small.json']) + await expect(workspace.publishReport(join(root, 'outside.json'), {})).rejects.toThrow('outside') + }) +}) +it('refuses unsafe existing report entries and a symlinked lock', async () => { + await withExperimentWorkspace(root, policy, async (workspace) => { + await symlink('/tmp', join(workspace.reports, 'link')) + await expect(workspace.checkCapacity()).rejects.toThrow('Unsafe') + }) + await rm(join(root, '.writer.sqlite')) + await symlink('/tmp', join(root, '.writer.sqlite')) + await expect(withExperimentWorkspace(root, policy, async () => null)).rejects.toThrow('Unsafe') +}) +it('enforces the ledger page ceiling and refuses a reduced limit below existing size', async () => { + const db = new Database(join(root, 'experiments.sqlite')) + try { + await withExperimentWorkspace(root, policy, async (workspace) => { + workspace.configureDatabase(db) + db.exec('CREATE TABLE payload(bytes BLOB)') + expect(() => db.prepare('INSERT INTO payload VALUES (?)').run(Buffer.alloc(100000))).toThrow(/full/) + expect(db.prepare('SELECT COUNT(*) AS count FROM payload').get()).toEqual({ count: 0 }) + }) + db.pragma('max_page_count = 1000') + db.prepare('INSERT INTO payload VALUES (?)').run(Buffer.alloc(100000)) + await expect(withExperimentWorkspace(root, policy, async workspace => workspace.configureDatabase(db))).rejects.toThrow('database quota') + } + finally { db.close() } +}) +it('fails admission before starting work when the free-space reserve is unavailable', async () => { + vi.mocked(filesystem.statfs).mockResolvedValueOnce({ type: 0n, bsize: 4096n, blocks: 1n, bfree: 0n, bavail: 0n, files: 0n, ffree: 0n }) + let entered = false + await expect(withExperimentWorkspace(root, { ...policy, minimumFreeBytes: 1024 ** 4 }, async () => { + entered = true + })).rejects.toThrow('free-space') + expect(entered).toBe(false) +}) +it('releases the OS writer lock after the owning controller is killed', async () => { + await writeFile(join(root, '.writer.sqlite'), '', { mode: 0o600 }) + const child = spawn(process.execPath, ['-e', `const Database=require('better-sqlite3');const db=new Database(process.argv[1]);db.exec('BEGIN IMMEDIATE');process.send('locked');setInterval(()=>{},1000)`, join(root, '.writer.sqlite')], { stdio: ['ignore', 'ignore', 'pipe', 'ipc'] }) + const closed = once(child, 'close') + try { + await once(child, 'message') + await expect(withExperimentWorkspace(root, policy, async () => null)).rejects.toThrow('locked') + child.kill('SIGKILL') + await closed + expect(await withExperimentWorkspace(root, policy, async () => 'recovered')).toBe('recovered') + } + finally { + child.kill('SIGKILL') + await closed + } +}) + +it('serializes concurrent publications under the same quota snapshot', async () => { + await withExperimentWorkspace(root, policy, async (workspace) => { + const first = workspace.publishReport(join(workspace.reports, 'first.json'), { value: 'a'.repeat(30) }) + const second = workspace.publishReport(join(workspace.reports, 'second.json'), { value: 'b'.repeat(30) }) + const results = await Promise.allSettled([first, second]) + expect(results.map(result => result.status)).toEqual(['fulfilled', 'rejected']) + expect(await readdir(workspace.reports)).toEqual(['first.json']) + }) +}) diff --git a/scripts/algorithm-experiments/workspace.ts b/scripts/algorithm-experiments/workspace.ts new file mode 100644 index 00000000..2597e865 --- /dev/null +++ b/scripts/algorithm-experiments/workspace.ts @@ -0,0 +1,100 @@ +import Database from 'better-sqlite3' +import { constants } from 'node:fs' +import { lstat, mkdir, open, opendir, statfs } from 'node:fs/promises' +import { dirname, join, resolve } from 'node:path' +import { z } from 'zod' +import { publishJson } from './files' + +const integer = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) +export const workspacePolicySchema = z.object({ + maximumReportBytes: integer.positive().max(64 * 1024 ** 3), + maximumReportFiles: integer.positive().max(100000), + maximumSingleReportBytes: integer.positive().max(32 * 1024 ** 2), + maximumDatabaseBytes: integer.min(65536).max(1024 ** 3), + minimumFreeBytes: integer.max(1024 ** 4), +}).strict() +/** Off-pod workstation limits, not pod production budgets. */ +export const defaultWorkspacePolicy = Object.freeze({ maximumReportBytes: 1024 ** 3, maximumReportFiles: 2048, + maximumSingleReportBytes: 32 * 1024 ** 2, maximumDatabaseBytes: 64 * 1024 ** 2, minimumFreeBytes: 1024 ** 3 }) + +async function privateDirectory(path: string) { + await mkdir(path, { recursive: true, mode: 0o700 }) + const info = await lstat(path) + if (!info.isDirectory() || info.isSymbolicLink() || (info.mode & 0o077) || (process.getuid && info.uid !== process.getuid())) throw new Error('Experiment workspace must be private and owned') +} +async function privateFile(path: string, missing = false) { + try { + const info = await lstat(path) + if (!info.isFile() || info.isSymbolicLink() || info.nlink !== 1 || (info.mode & 0o077) + || (process.getuid && info.uid !== process.getuid())) throw new Error('Unsafe experiment workspace file') + return info + } + catch (error) { + if (missing && (error as NodeJS.ErrnoException).code === 'ENOENT') return null + throw error + } +} + +/** A separate SQLite write lock survives async work and is released by the OS + * on controller death. Never remove its inode or steal ownership by timeout. */ +export async function withExperimentWorkspace(directory: string, rawPolicy: unknown, + action: (workspace: { reports: string, configureDatabase: (db: Database.Database) => void, publishReport: typeof publishJson, checkCapacity: () => Promise }) => Promise) { + const policy = workspacePolicySchema.parse(rawPolicy), root = resolve(directory) + await privateDirectory(root) + const lockPath = join(root, '.writer.sqlite') + try { + const file = await open(lockPath, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY | constants.O_NOFOLLOW, 0o600) + await file.close() + } + catch (error) { if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error } + for (const suffix of ['', '-wal', '-shm', '-journal']) { + const info = await privateFile(lockPath + suffix, suffix !== '') + if (info && info.size > 65536) throw new Error('Workspace lock exceeds its size bound') + } + const lock = new Database(lockPath, { timeout: 0 }) + let publication: Promise = Promise.resolve() + try { + lock.exec('BEGIN IMMEDIATE') + if ((lock.prepare('SELECT COUNT(*) AS count FROM sqlite_master').get() as { count: number }).count !== 0) throw new Error('Unexpected workspace lock database') + const reports = join(root, 'reports') + await privateDirectory(reports) + const capacity = async (additionalBytes = 0, additionalFiles = 0) => { + let count = additionalFiles, bytes = additionalBytes + const entries = await opendir(reports) + for await (const entry of entries) { + if (++count > policy.maximumReportFiles) throw new Error('Experiment report file quota exceeded') + const info = await privateFile(join(reports, entry.name)) + bytes += info?.size ?? 0 + if (bytes > policy.maximumReportBytes) throw new Error('Experiment report byte quota exceeded') + } + if (count > policy.maximumReportFiles || bytes > policy.maximumReportBytes) throw new Error('Experiment report quota exceeded') + const disk = await statfs(root, { bigint: true }) + // Reserve for a complete rollback journal and growth of the bounded ledger. + const required = BigInt(policy.minimumFreeBytes) + BigInt(2 * policy.maximumDatabaseBytes) + BigInt(additionalBytes) + if (disk.bavail < 0n || disk.bsize <= 0n || disk.bavail * disk.bsize < required) throw new Error('Experiment workspace free-space reserve unavailable') + } + await capacity() + return await action({ reports, checkCapacity: () => capacity(), configureDatabase: (db) => { + const pageSize = Number(db.pragma('page_size', { simple: true })) + if (!Number.isSafeInteger(pageSize) || pageSize < 512) throw new Error('Invalid experiment database page size') + const pages = Math.floor(policy.maximumDatabaseBytes / pageSize) + if (Number(db.pragma(`max_page_count = ${pages}`, { simple: true })) > pages) throw new Error('Experiment database quota exceeded') + if (db.pragma('journal_mode = DELETE', { simple: true }) !== 'delete') throw new Error('Bounded experiment rollback journal required') + }, publishReport: async (path, value) => { + if (dirname(resolve(path)) !== reports) throw new Error('Report path outside managed workspace') + const snapshot = structuredClone(value) + const bytes = Buffer.byteLength(JSON.stringify(snapshot, null, 2) + '\n') + if (bytes > policy.maximumSingleReportBytes) throw new Error('Experiment single-report quota exceeded') + const result = publication.then(async () => { + await capacity(bytes, 1) + await publishJson(path, snapshot) + }) + publication = result.catch(() => {}) + return result + } }) + } + finally { + await publication + lock.close() + } +} diff --git a/src/biometrics/backtest/admission-service.ts b/src/biometrics/backtest/admission-service.ts new file mode 100644 index 00000000..6c19726a --- /dev/null +++ b/src/biometrics/backtest/admission-service.ts @@ -0,0 +1,59 @@ +import type { HealthAdmission } from './job-operations' +import { dirname } from 'node:path' +import { sql } from 'drizzle-orm' +import { z } from 'zod' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import type * as schema from '@/src/db/biometrics-schema' +import { admissionPolicySchema, collectAdmissionSample, evaluateAdmission } from './admission' + +export const replayAdmissionSchema = z.object({ + configured: z.boolean(), qualificationId: z.string().nullable(), admitted: z.boolean(), + reasons: z.array(z.enum(['unqualified', 'health_unavailable', 'stale_sample', 'clock_skew', + 'memory_pressure', 'disk_pressure', 'cpu_pressure', 'event_loop_delay', 'piezo_stale', 'temperature'])), +}).strict() + +/** Server-owned policy only. Missing or malformed configuration never enables + * background I/O. The qualification ID is operator-supplied evidence attribution, + * not an assertion that supplying JSON has qualified the hardware. + */ +async function measuredReplayAdmission(db: BetterSQLite3Database): Promise & { receipt?: Awaited> }> { + const policy = configuredReplayPolicy() + if (!policy) return { configured: false, qualificationId: null, admitted: false, reasons: ['unqualified'] } + try { + const path = db.all<{ name: string, file: string }>(sql`PRAGMA database_list`).find(row => row.name === 'main')?.file + if (!path) throw new Error('Missing persistent database') + const { getSensorFrameTimes } = await import('@/src/streaming/piezoStream') + const sample = await collectAdmissionSample(dirname(path), () => getSensorFrameTimes()['piezo-dual']) + const checkedAtMs = Date.now() + const decision = evaluateAdmission(sample, policy, checkedAtMs) + return { configured: true, qualificationId: policy.qualificationId, ...decision, + receipt: { allowed: decision.admitted, sample: decision.admitted && sample + ? { qualificationId: policy.qualificationId, checkedAtMs, ingestionAgeMs: checkedAtMs - sample.piezoReceivedAtMs } + : null } } + } + catch { return { configured: true, qualificationId: policy.qualificationId, admitted: false, reasons: ['health_unavailable'] } } +} + +export async function replayAdmission(db: BetterSQLite3Database) { + const { configured, qualificationId, admitted, reasons } = await measuredReplayAdmission(db) + return replayAdmissionSchema.parse({ configured, qualificationId, admitted, reasons }) +} + +export function replayHealthAdmission(db: BetterSQLite3Database): HealthAdmission { + return async () => { + const measured = await measuredReplayAdmission(db) + return measured.receipt ?? { allowed: false, sample: null } + } +} + +export function replayAdmitter(db: BetterSQLite3Database) { + return async () => (await replayAdmission(db)).admitted +} + +/** Reading configuration has no database, streaming or health side effects. */ +export function configuredReplayPolicy() { + try { + return admissionPolicySchema.parse(JSON.parse(process.env.SLEEPYPOD_BIOMETRICS_ADMISSION_POLICY ?? 'null')) + } + catch { return null } +} diff --git a/src/biometrics/backtest/admission.ts b/src/biometrics/backtest/admission.ts new file mode 100644 index 00000000..fdcaec50 --- /dev/null +++ b/src/biometrics/backtest/admission.ts @@ -0,0 +1,82 @@ +import { readFile, statfs } from 'node:fs/promises' +import { availableParallelism, loadavg } from 'node:os' +import { performance } from 'node:perf_hooks' +import { z } from 'zod' + +/** Limits must come from an explicitly qualified deployment policy. There is + * no fallback that silently enables replay on an unmeasured device. */ +export const admissionPolicySchema = z.object({ + qualificationId: z.string().min(1).max(200), + minimumAvailableMemoryBytes: z.number().int().positive(), + minimumFreeDiskBytes: z.number().int().positive(), + maximumLoadPerCpu: z.number().positive(), + maximumEventLoopDelayMs: z.number().positive(), + maximumPiezoAgeMs: z.number().int().positive(), + maximumTemperatureC: z.number().positive(), + maximumSampleAgeMs: z.number().int().positive(), +}).strict() +export type AdmissionPolicy = z.infer +const sampleSchema = z.object({ + measuredAtMs: z.number().int().nonnegative(), + availableMemoryBytes: z.number().int().nonnegative(), + freeDiskBytes: z.number().int().nonnegative(), + loadPerCpu: z.number().nonnegative(), + eventLoopDelayMs: z.number().nonnegative(), + piezoReceivedAtMs: z.number().int().nonnegative(), + temperatureC: z.number().min(-50).max(150), +}).strict() +export type AdmissionSample = z.infer +export type AdmissionReason = 'unqualified' | 'health_unavailable' | 'stale_sample' | 'clock_skew' + | 'memory_pressure' | 'disk_pressure' | 'cpu_pressure' | 'event_loop_delay' | 'piezo_stale' | 'temperature' + +export function evaluateAdmission(rawSample: unknown, rawPolicy: unknown, now = Date.now()): { admitted: boolean, reasons: AdmissionReason[] } { + const policy = admissionPolicySchema.safeParse(rawPolicy) + if (!policy.success) return { admitted: false, reasons: ['unqualified'] } + const sample = sampleSchema.safeParse(rawSample) + if (!sample.success || !Number.isSafeInteger(now) || now < 0) return { admitted: false, reasons: ['health_unavailable'] } + const s = sample.data, p = policy.data + const reasons: AdmissionReason[] = [] + if (s.measuredAtMs > now || s.piezoReceivedAtMs > now) reasons.push('clock_skew') + if (now - s.measuredAtMs > p.maximumSampleAgeMs) reasons.push('stale_sample') + if (s.availableMemoryBytes < p.minimumAvailableMemoryBytes) reasons.push('memory_pressure') + if (s.freeDiskBytes < p.minimumFreeDiskBytes) reasons.push('disk_pressure') + if (s.loadPerCpu > p.maximumLoadPerCpu) reasons.push('cpu_pressure') + if (s.eventLoopDelayMs > p.maximumEventLoopDelayMs) reasons.push('event_loop_delay') + if (now - s.piezoReceivedAtMs > p.maximumPiezoAgeMs) reasons.push('piezo_stale') + if (s.temperatureC > p.maximumTemperatureC) reasons.push('temperature') + return { admitted: reasons.length === 0, reasons } +} + +export function parseAvailableMemory(text: string) { + const match = /^MemAvailable:\s+(\d+)\s+kB\s*$/m.exec(text) + if (!match) throw new Error('Available memory is unavailable') + const bytes = Number(match[1]) * 1024 + if (!Number.isSafeInteger(bytes) || bytes < 0) throw new Error('Invalid available memory') + return bytes +} + +/** No hardware commands, database queries, or lifetime histogram. Measure the + * scheduling delay of this process and only the live piezo frame timestamp. */ +export async function collectAdmissionSample(databaseDirectory: string, piezoReceivedAt: () => number | undefined): Promise { + const measuredAtMs = Date.now() + try { + // Probe Linux health files directly. Turbopack folds both process.platform + // and os.platform() to the build host; missing runtime files fail closed. + const probeStarted = performance.now() + const [memory, disk, thermal, delay] = await Promise.all([ + readFile('/proc/meminfo', 'utf8'), + statfs(databaseDirectory), + readFile('/sys/class/thermal/thermal_zone0/temp', 'utf8'), + new Promise(resolve => setTimeout(() => resolve(Math.max(0, performance.now() - probeStarted - 20)), 20)), + ]) + const rawTemperature = thermal.trim() + if (!/^-?\d+$/.test(rawTemperature)) return null + const parsed = sampleSchema.safeParse({ + measuredAtMs, availableMemoryBytes: parseAvailableMemory(memory), freeDiskBytes: disk.bavail * disk.bsize, + loadPerCpu: loadavg()[0] / availableParallelism(), eventLoopDelayMs: delay, + piezoReceivedAtMs: piezoReceivedAt(), temperatureC: Number(rawTemperature) / 1000, + }) + return parsed.success ? parsed.data : null + } + catch { return null } +} diff --git a/src/biometrics/backtest/discovery.ts b/src/biometrics/backtest/discovery.ts new file mode 100644 index 00000000..7f4f1ffd --- /dev/null +++ b/src/biometrics/backtest/discovery.ts @@ -0,0 +1,60 @@ +import { and, desc, eq, gt, gte, lt, or, sql } from 'drizzle-orm' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import { z } from 'zod' +import * as schema from '@/src/db/biometrics-schema' +import { ReferenceInputError } from '../references/time' +import { coverageSchema } from './contracts' +import { describeCoverage } from './coverage' + +const side = z.enum(['left', 'right']) +const sessionSchema = z.object({ id: z.number(), side, startMs: z.number(), endMs: z.number(), sleepDurationSeconds: z.number(), timesExitedBed: z.number() }) +export const sessionPageSchema = z.object({ items: z.array(sessionSchema), nextCursor: z.object({ startMs: z.number(), id: z.number() }).nullable() }) +export const sessionCapabilitiesSchema = z.object({ + session: sessionSchema, + processed: z.object({ vitals: coverageSchema, movement: coverageSchema, legacyUnknownProvenanceCount: z.number() }), + referenceMetrics: z.array(z.object({ metric: z.string(), count: z.number() })), + comparison: z.boolean(), stageReplay: z.boolean(), signalReplay: z.literal(false), rawWaveforms: z.literal('not_inventoried'), + limitations: z.array(z.string()), +}) +const describeSession = (row: typeof schema.sleepRecords.$inferSelect) => ({ id: row.id, side: row.side, startMs: row.enteredBedAt.getTime(), + endMs: row.leftBedAt.getTime(), sleepDurationSeconds: row.sleepDurationSeconds, timesExitedBed: row.timesExitedBed }) + +export function listReplaySessions(db: BetterSQLite3Database, selectedSide: 'left' | 'right', cursor?: { startMs: number, id: number }) { + const records = db.select().from(schema.sleepRecords).where(and(eq(schema.sleepRecords.side, selectedSide), cursor + ? or(lt(schema.sleepRecords.enteredBedAt, new Date(cursor.startMs)), and(eq(schema.sleepRecords.enteredBedAt, new Date(cursor.startMs)), lt(schema.sleepRecords.id, cursor.id))) + : undefined)) + .orderBy(desc(schema.sleepRecords.enteredBedAt), desc(schema.sleepRecords.id)).limit(21).all() + const items = records.slice(0, 20).map(describeSession) + const last = items.at(-1) + return { items, nextCursor: records.length > 20 && last ? { startMs: last.startMs, id: last.id } : null } +} + +export function sessionCapabilities(db: BetterSQLite3Database, selectedSide: 'left' | 'right', sessionId: number) { + return db.transaction(() => { + const record = db.select().from(schema.sleepRecords).where(and(eq(schema.sleepRecords.side, selectedSide), eq(schema.sleepRecords.id, sessionId))).get() + if (!record) throw new ReferenceInputError('NOT_FOUND', 'Session not found for this side') + const session = describeSession(record) + if (session.endMs <= session.startMs || session.endMs - session.startMs > 36 * 3600000) throw new ReferenceInputError('RESOURCE_LIMIT', 'Session boundaries exceed the inspection budget') + const vitals = db.select({ timestamp: schema.vitals.timestamp, run: schema.vitals.processingRunId }).from(schema.vitals).where(and(eq(schema.vitals.side, selectedSide), + gte(schema.vitals.timestamp, record.enteredBedAt), lt(schema.vitals.timestamp, record.leftBedAt))).orderBy(schema.vitals.timestamp).limit(20001).all() + const movement = db.select({ timestamp: schema.movement.timestamp }).from(schema.movement).where(and(eq(schema.movement.side, selectedSide), + gte(schema.movement.timestamp, record.enteredBedAt), lt(schema.movement.timestamp, record.leftBedAt))).orderBy(schema.movement.timestamp).limit(20001).all() + if (vitals.length + movement.length > 20000) throw new ReferenceInputError('RESOURCE_LIMIT', 'Session exceeds the retained processed-data row budget') + const referenceMetrics = db.select({ metric: schema.referenceMeasurements.metric, count: sql`count(*)` }).from(schema.referenceMeasurements) + .where(and(eq(schema.referenceMeasurements.side, selectedSide), lt(schema.referenceMeasurements.startMs, session.endMs), + or(gt(schema.referenceMeasurements.endMs, session.startMs), and(eq(schema.referenceMeasurements.temporalKind, 'point'), gte(schema.referenceMeasurements.startMs, session.startMs))))) + .groupBy(schema.referenceMeasurements.metric).all() + return { session, + processed: { vitals: describeCoverage(vitals.map(row => row.timestamp.getTime()), session.startMs, session.endMs, 60000), + movement: describeCoverage(movement.map(row => row.timestamp.getTime()), session.startMs, session.endMs, 60000), + legacyUnknownProvenanceCount: vitals.filter(row => row.run === null).length }, + referenceMetrics, comparison: vitals.length > 0, stageReplay: vitals.length > 0 && session.endMs - session.startMs <= 86400000, + signalReplay: false as const, rawWaveforms: 'not_inventoried' as const, + limitations: ['Reference counts include available revisions; select one revision per source family when freezing inputs.', + 'Raw archives and pump/occupancy context have not been retained by this processed-data service.', + 'Movement and vitals availability use nominal one-minute cadence, not inferred continuous signal support.', + ...(session.endMs - session.startMs > 86400000 ? ['Stage replay currently requires a merged window of at most one day.'] : []), + ...(!vitals.length ? ['No stored vitals are available for comparison or stage replay.'] : [])], + } + }) +} diff --git a/src/biometrics/backtest/execution-budget.ts b/src/biometrics/backtest/execution-budget.ts new file mode 100644 index 00000000..539bcdf4 --- /dev/null +++ b/src/biometrics/backtest/execution-budget.ts @@ -0,0 +1,14 @@ +import { z } from 'zod' + +/** Trusted deployment settings, never an experiment/configuration or RPC field. + * These ceilings permit qualification of whole nights; they do not enable a + * worker host or replace ongoing resource admission and lease fencing. + */ +export const replayBudgetSchema = z.object({ + jobWallMs: z.number().int().min(1000).max(48 * 60 * 60 * 1000), + waveformWallMs: z.number().int().min(1000).max(60 * 60 * 1000), + waveformCpuSeconds: z.number().int().min(1).max(3600), +}).strict().refine(budget => budget.waveformWallMs <= budget.jobWallMs + && budget.waveformCpuSeconds * 1000 <= budget.waveformWallMs, +'Waveform CPU and wall budgets must fit within the job budget') +export type ReplayBudget = z.infer diff --git a/src/biometrics/backtest/night-summary.ts b/src/biometrics/backtest/night-summary.ts new file mode 100644 index 00000000..108d2f57 --- /dev/null +++ b/src/biometrics/backtest/night-summary.ts @@ -0,0 +1,3 @@ +import { nightUncertainty } from './uncertainty' +import { createNightSummary } from './night-summary-core' +export const summarizeStageNights = createNightSummary(nightUncertainty) diff --git a/src/biometrics/backtest/numeric-trace-page.ts b/src/biometrics/backtest/numeric-trace-page.ts new file mode 100644 index 00000000..7530a7a2 --- /dev/null +++ b/src/biometrics/backtest/numeric-trace-page.ts @@ -0,0 +1,57 @@ +import { z } from 'zod' +import { ReferenceInputError } from '../references/time' +import { traceRequestSchema } from './trace-contracts' +import { waveformReadingSchema } from './waveform-output' + +const reading = waveformReadingSchema.pick({ timestampMs: true, heartRate: true, breathingRate: true, hrvIndex: true, + hrvMetric: true, qualityScore: true, flags: true, sourceSide: true, configHash: true, side: true, limitations: true }).strip() +export const numericTracePageSchema = z.object({ jobId: z.string().uuid(), contentHash: z.string(), + configuration: z.enum(['default', 'current', 'candidate']), side: z.enum(['left', 'right']), + provenance: z.literal('retained_waveform_recomputation'), timestampBasis: z.literal('original_trace_before_reference_clock_offset'), clockOffsetMs: z.number(), + totalItems: z.number().int(), offset: z.number().int(), limit: z.number().int(), downsampled: z.literal(false), + items: z.array(reading.extend({ cohortId: z.string(), revisionId: z.string().uuid().nullable() })).max(500), + nextCursor: z.object({ offset: z.number().int(), contentHash: z.string() }).nullable(), +}) +const reportSchema = z.object({ version: z.literal('waveform-backtest-report-v1'), side: z.enum(['left', 'right']), + protocol: z.object({ mode: z.literal('signal_replay'), clockOffsetMs: z.number().int() }), + waveform: z.object({ outputs: z.array(z.object({ cohortId: z.string(), configHash: z.string(), readings: z.array(reading).max(20000) })).max(180) }), + nights: z.array(z.object({ cohortId: z.string(), evaluations: z.array(z.object({ label: z.enum(['default', 'current', 'candidate']), + configHash: z.string(), revisionId: z.string().uuid().nullable(), + windows: z.array(z.object({ startMs: z.number().int(), endMs: z.number().int() })).max(60), + })).max(3) })).max(60), +}).refine(report => report.waveform.outputs.reduce((sum, output) => sum + output.readings.length, 0) <= 20000 + && new Set(report.nights.map(night => night.cohortId)).size === report.nights.length) + +/** Exact retained numeric rows for the dataset side only. Cross-side routing + * provenance is retained, but opposite-side observations are never returned. */ +export function pageNumericTraces(report: unknown, raw: z.input) { + const input = traceRequestSchema.parse(raw) + const parsed = reportSchema.safeParse(report) + if (!parsed.success) throw new ReferenceInputError('RESULT_CORRUPT', 'Numeric trace format or side provenance is unavailable') + const data = parsed.data + const nights = data.nights.filter(night => input.cohortId === undefined || night.cohortId === input.cohortId) + const items: z.infer['items'] = [] + let totalItems = 0, selections = 0 + for (const night of nights) { + const evaluations = night.evaluations.filter(row => row.label === input.configuration) + if (evaluations.length > 1) throw new ReferenceInputError('RESULT_CORRUPT', 'Duplicate numeric configuration') + for (const evaluation of evaluations) { + selections++ + const outputs = data.waveform.outputs.filter(output => output.cohortId === night.cohortId && output.configHash === evaluation.configHash) + if (outputs.length !== 1) throw new ReferenceInputError('RESULT_CORRUPT', 'Numeric trace output identity is ambiguous or missing') + for (const row of outputs[0].readings) { + if (row.configHash !== evaluation.configHash && row.side === data.side) throw new ReferenceInputError('RESULT_CORRUPT', 'Numeric reading configuration mismatch') + if (row.side !== data.side || !evaluation.windows.some(window => row.timestampMs >= window.startMs && row.timestampMs < window.endMs)) continue + if (totalItems >= input.offset && items.length < input.limit) items.push({ ...row, cohortId: night.cohortId, revisionId: evaluation.revisionId }) + totalItems++ + } + } + } + if (!selections) throw new ReferenceInputError('NOT_FOUND', 'Requested cohort or configuration was not evaluated') + if (input.offset > totalItems) throw new ReferenceInputError('INVALID_CURSOR', 'Trace cursor exceeds the selected report') + const next = input.offset + items.length + return numericTracePageSchema.parse({ jobId: input.id, contentHash: input.expectedContentHash, configuration: input.configuration, side: data.side, + provenance: 'retained_waveform_recomputation', timestampBasis: 'original_trace_before_reference_clock_offset', clockOffsetMs: data.protocol.clockOffsetMs, + totalItems, offset: input.offset, limit: input.limit, items, downsampled: false, + nextCursor: next < totalItems ? { offset: next, contentHash: input.expectedContentHash } : null }) +} diff --git a/src/biometrics/backtest/process-group-memory.ts b/src/biometrics/backtest/process-group-memory.ts new file mode 100644 index 00000000..71de1b61 --- /dev/null +++ b/src/biometrics/backtest/process-group-memory.ts @@ -0,0 +1,82 @@ +import { readdir, readFile } from 'node:fs/promises' +import { join } from 'node:path' +import type { ProcessGroupMemory } from './process-group-memory-contract' + +/** Fixed procfs reads only; never inspect argv, environment, or other exports. + * Shared pages may be counted in more than one process's RSS. */ +export async function sampleProcessGroupRss(group: number, root = '/proc'): Promise { + if (!Number.isSafeInteger(group) || group < 1) return null + try { + const pids = (await readdir(/* turbopackIgnore: true */ root)).filter(name => /^[1-9]\d*$/.test(name)) + if (pids.length > 4096) return null + let total = 0, members = 0 + for (let i = 0; i < pids.length; i += 16) { + const values = await Promise.all(pids.slice(i, i + 16).map(async (pid) => { + try { + const stat = await readFile(join(root, pid, 'stat'), 'utf8') + if (stat.length > 16384) throw new Error('Invalid proc stat') + const end = stat.lastIndexOf(')') + if (end < 0) throw new Error('Invalid proc stat') + const fields = stat.slice(end + 1).trim().split(/\s+/) + if (!/^\d+$/.test(fields[2] ?? '')) throw new Error('Missing process group') + if (Number(fields[2]) !== group) return 0 + const status = await readFile(join(root, pid, 'status'), 'utf8') + if (status.length > 16384) throw new Error('Invalid proc status') + const rss = /^VmRSS:\s+(\d+)\s+kB\s*$/m.exec(status) + // Zombies have no address space and no VmRSS entry. + if (!rss && fields[0] === 'Z') return 0 + if (!rss) throw new Error('Missing process RSS') + members++ + const bytes = Number(rss[1]) * 1024 + if (!Number.isSafeInteger(bytes) || bytes < 0) throw new Error('Invalid process RSS') + return bytes + } + catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT' || (error as NodeJS.ErrnoException).code === 'ESRCH') return 0 + throw error + } + })) + total += values.reduce((sum, value) => sum + value, 0) + if (members > 64 || !Number.isSafeInteger(total)) return null + } + return members > 0 && total > 0 ? total : null + } + catch { return null } +} + +export function monitorProcessGroupMemory(group: number, options: { sample?: () => Promise, now?: () => number } = {}) { + const now = options.now ?? (() => performance.now()) + const read = options.sample ?? (() => process.platform === 'linux' ? sampleProcessGroupRss(group) : Promise.resolve(null)) + const started = now() + let last = started, gap = 0, attempts = 0, samples = 0, peak = 0, stopped = false + let active: Promise | null = null + const sample = () => { + if (stopped) return Promise.resolve() + active ??= Promise.resolve().then(read).catch(() => null).then((rss) => { + const at = now() + gap = Math.max(gap, at - last) + last = at + attempts++ + if (rss !== null && Number.isSafeInteger(rss) && rss > 0) { + samples++ + peak = Math.max(peak, rss) + } + }).finally(() => { active = null }) + return active + } + const timer = setInterval(() => { + void sample() + }, 1000) + timer.unref() + void sample() + return { sample, + snapshot: (): ProcessGroupMemory => ({ version: 'process-group-rss-observations-v1', scope: 'sum_rss_of_replay_process_group', + attempts, samples, allSamplesAvailable: samples > 0 && samples === attempts, peakObservedRssBytes: samples ? peak : null, + maximumObservationGapMs: Math.max(gap, now() - last), intervalMs: 1000, exactKernelPeak: false }), + async stop() { + stopped = true + clearInterval(timer) + await active + }, + } +} diff --git a/src/biometrics/backtest/raw-collection-child.ts b/src/biometrics/backtest/raw-collection-child.ts new file mode 100644 index 00000000..4941a17f --- /dev/null +++ b/src/biometrics/backtest/raw-collection-child.ts @@ -0,0 +1,61 @@ +import { spawn } from 'node:child_process' +import { tmpdir } from 'node:os' +import { z } from 'zod' + +const resultSchema = z.object({ version: z.literal('raw-collection-result-v1'), released: z.number().int().min(0).max(10000), orphans: z.number().int().min(0).max(10000) }).strict() +interface RawCollectionLocation { pythonPath: string, modulesPath: string, databasePath: string, retainedDirectory: string } + +/** Trusted deployment paths only; no RPC arguments select files or alter retention grace. */ +export function runRawCollectionChild(location: RawCollectionLocation, options: { + signal?: AbortSignal + timeoutMs?: number +} = {}): Promise> { + const timeoutMs = options.timeoutMs ?? 120000 + if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 10 || timeoutMs > 120000) throw new Error('Invalid collection time budget') + if (options.signal?.aborted) return Promise.reject(new Error('RAW collection cancelled')) + return new Promise((resolve, reject) => { + const child = spawn(location.pythonPath, ['-B', '-m', 'common.biometrics.collection_worker', '--database-path', location.databasePath, '--retained-directory', location.retainedDirectory], { + cwd: tmpdir(), env: { NODE_ENV: 'production', PYTHONPATH: location.modulesPath, LANG: 'C.UTF-8', TZ: 'UTC', + OPENBLAS_NUM_THREADS: '1', OMP_NUM_THREADS: '1', MKL_NUM_THREADS: '1', NUMEXPR_NUM_THREADS: '1' }, + stdio: ['pipe', 'pipe', 'pipe'], + }) + const chunks: Buffer[] = [] + let outputBytes = 0, diagnosticBytes = 0 + let failure: Error | undefined, killTimer: ReturnType | undefined + const stop = (error: Error) => { + failure ??= error + child.kill('SIGTERM') + killTimer ??= setTimeout(() => child.kill('SIGKILL'), 500) + } + const timeout = setTimeout(() => stop(new Error('RAW collection exceeded its time budget')), timeoutMs) + const abort = () => stop(new Error('RAW collection cancelled')) + options.signal?.addEventListener('abort', abort, { once: true }) + if (options.signal?.aborted) abort() + child.on('error', () => stop(new Error('RAW collection worker could not start'))) + child.stdin.on('error', () => stop(new Error('RAW collection input pipe closed'))) + child.stderr.on('data', (chunk: Buffer) => { + diagnosticBytes += chunk.length + if (diagnosticBytes > 16384) stop(new Error('RAW collection diagnostics exceeded their bound')) + }) + child.stdout.on('data', (chunk: Buffer) => { + if (failure) return + outputBytes += chunk.length + if (outputBytes > 1024) return stop(new Error('RAW collection output exceeded its bound')) + chunks.push(chunk) + }) + child.on('close', (code) => { + clearTimeout(timeout) + if (killTimer) clearTimeout(killTimer) + options.signal?.removeEventListener('abort', abort) + if (failure) return reject(failure) + try { + if (code !== 0) throw new Error('Worker failed') + const text = new TextDecoder('utf-8', { fatal: true }).decode(Buffer.concat(chunks)) + const result = resultSchema.parse(JSON.parse(text)) + resolve(result) + } + catch { reject(new Error('RAW collection worker returned an invalid or incomplete result')) } + }) + child.stdin.end(JSON.stringify({ version: 'raw-collection-work-v1' })) + }) +} diff --git a/src/biometrics/backtest/raw-publication-child.ts b/src/biometrics/backtest/raw-publication-child.ts new file mode 100644 index 00000000..0f6b6f64 --- /dev/null +++ b/src/biometrics/backtest/raw-publication-child.ts @@ -0,0 +1,70 @@ +import { spawn } from 'node:child_process' +import { tmpdir } from 'node:os' +import { z } from 'zod' +import type { RawInspectionLocation } from './raw-inspection-child' +import type { prepareSelectedWaveformDataset } from './waveform-selection' + +const publicationResultSchema = z.object({ type: z.literal('result'), id: z.string().uuid(), + contentHash: z.string().regex(/^[a-f0-9]{64}$/), available: z.boolean() }).strict() +export interface RawPublicationLocation extends RawInspectionLocation { + databasePath: string + retainedDirectory: string +} + +/** Only coordinator-prepared rows enter this fixed shipped module. No RPC paths or code. */ +export function runRawPublicationChild(location: RawPublicationLocation, work: ReturnType, options: { + signal?: AbortSignal + timeoutMs?: number +} = {}): Promise> { + const serialized = JSON.stringify({ version: 'waveform-publication-work-v1', ...work }) + if (Buffer.byteLength(serialized) > 10 * 1024 * 1024) return Promise.reject(new Error('Waveform publication input exceeds 10 MiB')) + const timeoutMs = options.timeoutMs ?? 120000 + if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 10 || timeoutMs > 120000) throw new Error('Invalid publication time budget') + if (options.signal?.aborted) return Promise.reject(new Error('Waveform publication cancelled')) + return new Promise((resolve, reject) => { + const child = spawn(location.pythonPath, ['-B', '-m', 'common.biometrics.publication_worker', '--database', location.databasePath, '--archive-directory', location.archiveDirectory, '--retained-directory', location.retainedDirectory], { + cwd: tmpdir(), env: { NODE_ENV: 'production', PYTHONPATH: location.modulesPath, LANG: 'C.UTF-8', TZ: 'UTC', + OPENBLAS_NUM_THREADS: '1', OMP_NUM_THREADS: '1', MKL_NUM_THREADS: '1', NUMEXPR_NUM_THREADS: '1' }, + stdio: ['pipe', 'pipe', 'pipe'], + }) + const chunks: Buffer[] = [] + let outputBytes = 0, diagnosticBytes = 0 + let failure: Error | undefined, killTimer: ReturnType | undefined + const stop = (error: Error) => { + failure ??= error + child.kill('SIGTERM') + killTimer ??= setTimeout(() => child.kill('SIGKILL'), 500) + } + const timeout = setTimeout(() => stop(new Error('Waveform publication exceeded its time budget')), timeoutMs) + const abort = () => stop(new Error('Waveform publication cancelled')) + options.signal?.addEventListener('abort', abort, { once: true }) + if (options.signal?.aborted) abort() + child.on('error', () => stop(new Error('Waveform publication worker could not start'))) + child.stdin.on('error', () => stop(new Error('Waveform publication input pipe closed'))) + child.stderr.on('data', (chunk: Buffer) => { + diagnosticBytes += chunk.length + if (diagnosticBytes > 16384) stop(new Error('Waveform publication diagnostics exceeded their bound')) + }) + child.stdout.on('data', (chunk: Buffer) => { + if (failure) return + outputBytes += chunk.length + if (outputBytes > 65536) return stop(new Error('Waveform publication output exceeded its bound')) + chunks.push(chunk) + }) + child.on('close', (code) => { + clearTimeout(timeout) + if (killTimer) clearTimeout(killTimer) + options.signal?.removeEventListener('abort', abort) + if (failure) return reject(failure) + try { + if (code !== 0) throw new Error('Worker failed') + const text = new TextDecoder('utf-8', { fatal: true }).decode(Buffer.concat(chunks)) + const result = publicationResultSchema.parse(JSON.parse(text)) + if (result.contentHash !== work.prepared.contentHash) throw new Error('Mismatched published dataset') + resolve(result) + } + catch { reject(new Error('Waveform publication worker returned an invalid or incomplete result')) } + }) + child.stdin.end(serialized) + }) +} diff --git a/src/biometrics/backtest/reference-trace-page.ts b/src/biometrics/backtest/reference-trace-page.ts new file mode 100644 index 00000000..9987a848 --- /dev/null +++ b/src/biometrics/backtest/reference-trace-page.ts @@ -0,0 +1,66 @@ +import { z } from 'zod' +import { traceRequestSchema } from './trace-contracts' +import { normalizedMeasurementSchema } from '../references/contracts' +import { referenceCompatibility } from '../references/alignment' +import { ReferenceInputError } from '../references/time' +import type { DatasetSnapshot } from './contracts' + +export const referenceTraceRequestSchema = traceRequestSchema.omit({ configuration: true }).extend({ + cohortId: z.string().min(1).max(200), metric: z.enum(['heart_rate', 'breathing_rate', 'window_ibi_variability_v1', 'sleep_stages']), +}) +const projected = normalizedMeasurementSchema.omit({ original: true }).strip().extend({ + importId: z.string(), importRevision: z.number().int(), importContentHash: z.string(), source: z.string(), + compatibility: z.string().nullable(), +}) +export const referenceTracePageSchema = z.object({ jobId: z.string().uuid(), contentHash: z.string(), datasetHash: z.string(), + cohortId: z.string(), metric: referenceTraceRequestSchema.shape.metric, side: z.enum(['left', 'right']), + provenance: z.literal('frozen_dataset_reference'), timestampBasis: z.literal('reference_source_time'), + clockOffsetMs: z.number().int(), pointToleranceMs: z.number().int(), + totalItems: z.number().int(), offset: z.number().int(), limit: z.number().int(), downsampled: z.literal(false), + items: z.array(projected).max(500), nextCursor: z.object({ offset: z.number().int(), contentHash: z.string() }).nullable(), +}) +const reportSchema = z.object({ version: z.enum(['processed-backtest-report-v1', 'waveform-backtest-report-v1']), + datasetId: z.string().uuid(), datasetHash: z.string(), + protocol: z.object({ clockOffsetMs: z.number().int(), pointToleranceMs: z.number().int() }), + nights: z.array(z.object({ cohortId: z.string(), evaluations: z.array(z.object({ + windows: z.array(z.object({ startMs: z.number().int(), endMs: z.number().int() })).max(60), + })).max(3) })).max(60), +}) +export function referenceTraceIdentity(report: unknown) { + const parsed = reportSchema.safeParse(report) + if (!parsed.success) throw new ReferenceInputError('RESULT_CORRUPT', 'Reference trace report identity is unavailable') + return parsed.data +} + +/** Read only frozen imports, never the current reference library. Projection + * excludes submitted payloads, annotations and subject identity. No score is + * recomputed, and display records are not claims of independently paired data. */ +export function pageReferenceTraces(report: unknown, dataset: { contentHash: string, snapshot: DatasetSnapshot }, raw: z.input) { + const input = referenceTraceRequestSchema.parse(raw), identity = referenceTraceIdentity(report) + if (identity.datasetHash !== dataset.contentHash) throw new ReferenceInputError('REVISION_CONFLICT', 'Reference dataset hash changed') + const nights = identity.nights.filter(night => night.cohortId === input.cohortId) + if (nights.length !== 1) throw new ReferenceInputError('NOT_FOUND', 'Requested reference cohort was not evaluated') + const windows = nights[0].evaluations[0]?.windows + if (!windows) throw new ReferenceInputError('RESULT_CORRUPT', 'Reference scoring windows unavailable') + const items: z.infer['items'] = [] + let totalItems = 0 + for (const reference of dataset.snapshot.references) { + for (const row of reference.measurements) { + const sameFamily = row.metric === input.metric || (input.metric === 'window_ibi_variability_v1' && row.metric.startsWith('hrv_')) + if (!sameFamily || !windows.some(window => row.startMs >= window.startMs && (row.temporalKind === 'point' ? row.startMs < window.endMs : row.endMs <= window.endMs))) continue + if (totalItems >= input.offset && items.length < input.limit) { + items.push(projected.parse({ ...row, importId: reference.importId, importRevision: reference.revision, importContentHash: reference.contentHash, source: reference.source, + compatibility: input.metric === 'sleep_stages' + ? (row.scorable ? null : row.exclusions[0] ?? 'unscorable') + : referenceCompatibility(input.metric, input.metric === 'window_ibi_variability_v1' ? 300000 : null, row) })) + } + totalItems++ + } + } + if (input.offset > totalItems) throw new ReferenceInputError('INVALID_CURSOR', 'Reference cursor exceeds the selected records') + const next = input.offset + items.length + return referenceTracePageSchema.parse({ jobId: input.id, contentHash: input.expectedContentHash, datasetHash: identity.datasetHash, + cohortId: input.cohortId, metric: input.metric, side: dataset.snapshot.side, provenance: 'frozen_dataset_reference', timestampBasis: 'reference_source_time', + ...identity.protocol, totalItems, offset: input.offset, limit: input.limit, items, downsampled: false, + nextCursor: next < totalItems ? { offset: next, contentHash: input.expectedContentHash } : null }) +} diff --git a/src/biometrics/backtest/report-summary.ts b/src/biometrics/backtest/report-summary.ts new file mode 100644 index 00000000..55f78d1c --- /dev/null +++ b/src/biometrics/backtest/report-summary.ts @@ -0,0 +1,40 @@ +import { z } from 'zod' +import { ReferenceInputError } from '../references/time' + +const report = z.looseObject({ waveform: z.looseObject({ outputs: z.array(z.looseObject({ readings: z.unknown() })).max(180) }).optional(), version: z.enum(['processed-backtest-report-v1', 'waveform-backtest-report-v1']), + nights: z.array(z.looseObject({ evaluations: z.array(z.looseObject({ + windows: z.array(z.looseObject({ blocks: z.unknown() })).max(60), + })).max(3) })).max(60), +}) + +/** Project only after verifying the complete retained report. The content hash + * still identifies that complete artifact; exact traces remain separately paged. */ +export function summarizeBacktestResult(result: T | null) { + if (result === null) return null + let summary: unknown = null + let waveformReport = false + if (result.available) { + const parsed = report.safeParse(result.report) + if (!parsed.success) throw new ReferenceInputError('RESULT_CORRUPT', 'Report summary format is unavailable') + waveformReport = parsed.data.version === 'waveform-backtest-report-v1' + summary = { ...parsed.data, ...(parsed.data.waveform + ? { waveform: { ...parsed.data.waveform, outputs: parsed.data.waveform.outputs.map((output) => { + const metadata = { ...output } + delete metadata.readings + return metadata + }) } } + : {}), nights: parsed.data.nights.map(night => ({ ...night, + evaluations: night.evaluations.map(evaluation => ({ ...evaluation, windows: evaluation.windows.map((window) => { + const scores = { ...window } + delete scores.blocks + return scores + }) })), + })) } + // Bound the response without truncating scores or silently dropping nights. + if (new TextEncoder().encode(JSON.stringify(summary)).byteLength > 2 * 1024 * 1024) { + throw new ReferenceInputError('RESOURCE_LIMIT', 'Report summary exceeds its 2 MiB transport limit') + } + } + return { ...result, report: summary, projection: waveformReport ? 'scores_without_replay_trace_rows_v1' as const : 'scores_without_stage_trace_rows_v1' as const, + contentHashScope: 'complete_retained_report' as const } +} diff --git a/src/biometrics/backtest/runner.ts b/src/biometrics/backtest/runner.ts new file mode 100644 index 00000000..de48c293 --- /dev/null +++ b/src/biometrics/backtest/runner.ts @@ -0,0 +1,157 @@ +import { assessEvaluation } from '../profiles/evaluation-assessment' +import { dataHash } from '../algorithms/canonical' +import { verifyReplayContext } from '../profiles/replay-context' +import { datasetManifestSchema } from './contracts' +import { installedPromotionPolicies, unqualifiedPolicies } from '../profiles/qualification-registry' +import { numericObjectivePolicies } from './contracts' +import { finalizeObjectiveReport, finalizeStageObjectiveReport } from './objective-finalization' +import { createJobOperations, type AdmissionReceipt } from './job-operations' +import type { ReplayBudget } from './execution-budget' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import { z } from 'zod' +import type * as schema from '@/src/db/biometrics-schema' +import { DatasetStore } from './datasets' +import { BacktestJobStore } from './jobs' +import { backtestWorkSchema } from './evaluator' +import { executeWaveformInputs, verifyWaveformJobBuild } from './waveform-execution' +import type { RawInspectionLocation } from './raw-inspection-child' +import { workerBuildHash, runBacktestChild } from './child' + +/** One leased attempt. Call only after the host's resource/ingest gate admits work. */ +export async function executeNextBacktest(db: BetterSQLite3Database, options: { + owner: string + workerPath: string + rawLocation?: RawInspectionLocation + replayBudget?: ReplayBudget + signal?: AbortSignal + timeoutMs?: number + admit: () => Promise + admissionReceipt?: () => AdmissionReceipt | null +}) { + const operations = createJobOperations() + const admit = async () => { + const allowed = await options.admit() + operations.record(options.admissionReceipt?.() ?? null) + return allowed + } + if (options.signal?.aborted || !await admit() || options.signal?.aborted) return { state: 'deferred' as const } + const jobs = new BacktestJobStore(db) + const claimed = jobs.claim(options.owner) + if (!claimed) return { state: 'idle' as const } + const lease = { id: claimed.id, owner: options.owner, token: claimed.fencingToken } + const controller = new AbortController() + const abort = () => controller.abort() + options.signal?.addEventListener('abort', abort, { once: true }) + if (options.signal?.aborted) controller.abort() + let progress = z.object({ completedNights: z.number(), totalNights: z.number() }).parse(claimed.progress) + let heartbeatFailure: unknown + let resourceFailure = false + let resourceCheck: Promise | null = null + const heartbeat = () => { + try { + const result = jobs.heartbeat(lease, { phase: 'evaluating', ...progress }) + if (result.cancelRequested) controller.abort() + } + catch (error) { + heartbeatFailure = error + controller.abort() + } + } + const checkResources = async () => { + try { + if (await admit()) return + } + catch { /* Missing health evidence must stop background work. */ } + resourceFailure = true + controller.abort() + } + const timer = setInterval(() => { + heartbeat() + if (!resourceCheck && !controller.signal.aborted) { + resourceCheck = checkResources().finally(() => { + resourceCheck = null + }) + } + }, 5000) + try { + const input = z.object({ datasetId: z.string() }).parse(claimed.input) + const retained = new DatasetStore(db).read(input.datasetId) + const work = backtestWorkSchema.parse({ job: claimed.input, dataset: { manifest: retained.manifest, snapshot: retained.snapshot } }) + if (work.job.protocol.mode === 'signal_replay') { + work.waveform = await executeWaveformInputs(db, work, { ...options, signal: controller.signal }) + } + else if (await workerBuildHash(options.workerPath) !== work.job.evaluatorBuildHash) throw new Error('Evaluator bundle changed since enqueue') + heartbeat() + const report = await runBacktestChild(options.workerPath, work, { signal: controller.signal, timeoutMs: options.timeoutMs, + onProgress: (completedNights, totalNights) => { + if (totalNights !== progress.totalNights || completedNights < progress.completedNights || completedNights > totalNights) throw new Error('Invalid worker progress') + progress = { completedNights, totalNights } + heartbeat() + }, + }) + if (work.job.protocol.mode === 'signal_replay') await verifyWaveformJobBuild(work, { ...options, signal: controller.signal }) + let policies = await installedPromotionPolicies(options.workerPath, { evaluatorBuildHash: work.job.evaluatorBuildHash, + pythonBuildHash: work.job.waveform?.pythonBuildHash, protocolHash: work.job.protocolHash }) + // Quiesce the periodic check before the final fresh check and publication. + clearInterval(timer) + await resourceCheck + await checkResources() + if (heartbeatFailure) throw heartbeatFailure + const identity = z.object({ datasetHash: z.string(), protocolHash: z.string(), evaluatorBuildHash: z.string(), split: z.string(), nights: z.array(z.unknown()) }).parse(report) + if (identity.evaluatorBuildHash !== work.job.evaluatorBuildHash || identity.datasetHash !== work.job.datasetHash || identity.protocolHash !== work.job.protocolHash || identity.split !== work.job.split + || identity.nights.length !== progress.totalNights || progress.completedNights !== progress.totalNights) throw new Error('Worker report identity/progress mismatch') + if (controller.signal.aborted) throw new Error('Replay cancelled') + const jobOperations = operations.finish() + let qualificationWarning: string | null = null + if (policies.protocolHash && work.job.waveform) { + try { + verifyReplayContext(z.object({ waveform: z.unknown() }).parse(report).waveform, { + cohortIds: datasetManifestSchema.parse(retained.manifest).splitProtocol.assignments.filter(row => row.split === work.job.split).map(row => row.cohortId), + configHashes: work.job.configs.map(config => config.resolvedHash), side: retained.side, + pythonBuildHash: work.job.waveform.pythonBuildHash, otherSideConfigHash: work.job.waveform.otherSide.resolvedHash, + }, policies.waveformContext) + } + catch { + policies = unqualifiedPolicies() + qualificationWarning = 'Replay context is outside the installed qualification; this report is descriptive.' + } + } + const finalized = finalizeObjectiveReport(report, numericObjectivePolicies(work.job.protocol), jobOperations, policies.numeric) + const measured = { ...finalizeStageObjectiveReport(finalized, work.job.protocol.stageObjectivePolicy, jobOperations, policies.stages), jobOperations, qualificationWarning } + let status: ReturnType['status'] = 'insufficient_evidence' + const candidate = work.job.configs.find(config => config.label === 'candidate') + const current = work.job.configs.find(config => config.label === 'current') + if (candidate?.revisionId && current) { + try { + status = assessEvaluation(work.job, { report: measured, contentHash: dataHash(measured), available: true, + currentHoldoutUseCount: jobs.holdoutUseCount(work.job.datasetId) }, { + side: retained.side, revisionId: candidate.revisionId, resolvedHash: candidate.resolvedHash, currentResolvedHash: current.resolvedHash, + evaluatorBuildHash: work.job.evaluatorBuildHash, dataset: { id: retained.id, contentHash: retained.contentHash, side: retained.side, + available: retained.available, holdoutCohortIds: datasetManifestSchema.parse(retained.manifest).splitProtocol.assignments.filter(row => row.split === 'holdout').map(row => row.cohortId) }, + ...(work.job.waveform ? { waveform: { pythonBuildHash: work.job.waveform.pythonBuildHash, otherSideConfigHash: work.job.waveform.otherSide.resolvedHash } } : {}), + }, policies).status + } + catch { /* Exploratory, incomplete or incompatible evidence cannot claim improvement. */ } + } + if (candidate && current && candidate.resolvedHash === current.resolvedHash) status = 'no_material_change' + jobs.complete(lease, { ...measured, status, evidence: status === 'improved' ? 'qualified_reference_agreement' : work.job.evidence }) + return { state: 'succeeded' as const, jobId: claimed.id } + } + catch { + // Only the authoritative lease may update state. Expired work is recovered + // by the next claimant; it must not overwrite a replacement's result. + try { + const latest = jobs.get(claimed.id) + if (latest.state === 'cancel_requested') jobs.acknowledgeCancellation(lease) + else if (options.signal?.aborted || resourceFailure) jobs.interrupt(lease, resourceFailure ? 'Resource admission withdrawn' : 'Worker host stopped') + else jobs.fail(lease, 'Replay failed validation, resources, or execution') + } + catch { /* Lost lease: recovery owns the state transition. */ } + return { state: 'stopped' as const, jobId: claimed.id } + } + finally { + clearInterval(timer) + await resourceCheck + options.signal?.removeEventListener('abort', abort) + } +} diff --git a/src/biometrics/backtest/submission.ts b/src/biometrics/backtest/submission.ts index ef9d3cfd..3d1c2d05 100644 --- a/src/biometrics/backtest/submission.ts +++ b/src/biometrics/backtest/submission.ts @@ -6,7 +6,8 @@ import * as bioSchema from '@/src/db/biometrics-schema' import { resolveReplayBaseline } from '../profiles/replay-baseline' import { experimentProtocolSchema } from './contracts' import { workerBuildHash } from './child' -import { packagedCoordinatorPath, packagedWorkerPath } from './location' +import { dirname, join } from 'node:path' +import { packagedWorkerPath } from './location' import { waveformEvaluatorBuildIdentity } from './waveform-build' import { podRawInspectionLocation } from './raw-inspection-child' import { configHash } from '../algorithms/canonical' @@ -35,7 +36,8 @@ const buildProbes = globalThis as typeof globalThis & { __waveformSubmissionBuild?: { key: string, promise: ReturnType } } function sharedWaveformBuild(workerPath: string) { - const coordinatorPath = packagedCoordinatorPath() + // Pin the coordinator that ships beside this worker; execution hashes the same sibling. + const coordinatorPath = join(dirname(workerPath), 'coordinator.cjs') const key = JSON.stringify([coordinatorPath, workerPath, podRawInspectionLocation.pythonPath, podRawInspectionLocation.modulesPath]) if (buildProbes.__waveformSubmissionBuild?.key === key) return buildProbes.__waveformSubmissionBuild.promise const entry = { key, promise: waveformEvaluatorBuildIdentity(coordinatorPath, workerPath, podRawInspectionLocation) } diff --git a/src/biometrics/backtest/tests/admission-build.test.ts b/src/biometrics/backtest/tests/admission-build.test.ts new file mode 100644 index 00000000..0331ecfa --- /dev/null +++ b/src/biometrics/backtest/tests/admission-build.test.ts @@ -0,0 +1,32 @@ +// @vitest-environment node +import { build } from 'esbuild' +import { createRequire } from 'node:module' +import { runInNewContext } from 'node:vm' +import { expect, it } from 'vitest' + +it('retains Linux health collection when a bundler substitutes a macOS build platform', async () => { + const compiled = await build({ entryPoints: ['src/biometrics/backtest/admission.ts'], bundle: true, + platform: 'node', format: 'cjs', write: false, minify: true, + define: { 'process.platform': JSON.stringify('darwin') } }) + const nativeRequire = createRequire(import.meta.url) + const exported: { exports: { collectAdmissionSample?: (path: string, piezo: () => number) => Promise } } = { exports: {} } + const accesses: string[] = [] + const at = Date.now() + runInNewContext(compiled.outputFiles[0].text, { module: exported, exports: exported.exports, + process, setTimeout, clearTimeout, Buffer, + require: (name: string) => { + if (name === 'node:os') return { platform: () => 'darwin', availableParallelism: () => 4, loadavg: () => [1, 1, 1] } + if (name === 'node:fs/promises') return { + readFile: async (path: string) => { + accesses.push(path) + return path === '/proc/meminfo' ? 'MemAvailable: 1024 kB\n' : '35000\n' + }, + statfs: async () => ({ bavail: 100, bsize: 4096 }), + } + return nativeRequire(name) + } }) + expect(await exported.exports.collectAdmissionSample?.('/fixture', () => at)).toMatchObject({ + availableMemoryBytes: 1048576, freeDiskBytes: 409600, temperatureC: 35, loadPerCpu: 0.25, piezoReceivedAtMs: at, + }) + expect(accesses).toEqual(expect.arrayContaining(['/proc/meminfo', '/sys/class/thermal/thermal_zone0/temp'])) +}) diff --git a/src/biometrics/backtest/tests/admission-collector.test.ts b/src/biometrics/backtest/tests/admission-collector.test.ts new file mode 100644 index 00000000..ba8c9908 --- /dev/null +++ b/src/biometrics/backtest/tests/admission-collector.test.ts @@ -0,0 +1,39 @@ +// @vitest-environment node +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +const mocks = vi.hoisted(() => ({ readFile: vi.fn(), statfs: vi.fn(), loadavg: vi.fn(), availableParallelism: vi.fn() })) +vi.mock('node:fs/promises', () => ({ readFile: mocks.readFile, statfs: mocks.statfs })) +vi.mock('node:os', () => ({ loadavg: mocks.loadavg, availableParallelism: mocks.availableParallelism })) +import { collectAdmissionSample } from '../admission' +beforeEach(() => { + mocks.readFile.mockImplementation(async (path: string) => path === '/proc/meminfo' ? 'MemAvailable: 1024 kB\n' : '35000\n') + mocks.statfs.mockResolvedValue({ bavail: 100, bsize: 4096 }) + mocks.loadavg.mockReturnValue([2, 1, 1]) + mocks.availableParallelism.mockReturnValue(4) +}) +afterEach(() => { + vi.resetAllMocks() +}) +it('collects available disk/memory and a current scheduling probe with a piezo timestamp', async () => { + const before = Date.now() + const sample = await collectAdmissionSample('/fixture/data', () => before - 100) + expect(sample).toMatchObject({ availableMemoryBytes: 1048576, freeDiskBytes: 409600, loadPerCpu: 0.5, + temperatureC: 35, piezoReceivedAtMs: before - 100 }) + expect(sample?.measuredAtMs).toBeGreaterThanOrEqual(before) + expect(sample?.measuredAtMs).toBeLessThanOrEqual(Date.now()) + expect(sample?.eventLoopDelayMs).toBeGreaterThanOrEqual(0) + expect(mocks.statfs).toHaveBeenCalledWith('/fixture/data') +}) +it('returns unavailable when piezo has never been observed', async () => { + expect(await collectAdmissionSample('/fixture/data', () => undefined)).toBeNull() +}) +it('does not convert empty thermal files into a zero-degree reading', async () => { + mocks.readFile.mockImplementation(async (path: string) => path === '/proc/meminfo' ? 'MemAvailable: 1024 kB\n' : '\n') + expect(await collectAdmissionSample('/fixture/data', Date.now)).toBeNull() +}) +it('returns unavailable on collector failures and missing Linux health files', async () => { + mocks.statfs.mockRejectedValue(new Error('I/O error')) + expect(await collectAdmissionSample('/fixture/data', Date.now)).toBeNull() + mocks.statfs.mockResolvedValue({ bavail: 100, bsize: 4096 }) + mocks.readFile.mockRejectedValue(Object.assign(new Error('Missing Linux health file'), { code: 'ENOENT' })) + expect(await collectAdmissionSample('/fixture/data', Date.now)).toBeNull() +}) diff --git a/src/biometrics/backtest/tests/admission-service.test.ts b/src/biometrics/backtest/tests/admission-service.test.ts new file mode 100644 index 00000000..3b95a596 --- /dev/null +++ b/src/biometrics/backtest/tests/admission-service.test.ts @@ -0,0 +1,65 @@ +// @vitest-environment node +import { afterEach, expect, it, vi } from 'vitest' +import type * as Admission from '../admission' +import { collectAdmissionSample } from '../admission' +import { replayAdmission, replayAdmitter } from '../admission-service' + +const frames = vi.hoisted(() => ({ 'piezo-dual': 1000 })) +vi.mock('@/src/streaming/piezoStream', () => ({ getSensorFrameTimes: () => frames })) +vi.mock('../admission', async original => ({ ...await original(), collectAdmissionSample: vi.fn() })) +const db = { all: vi.fn(() => [{ name: 'main', file: '/private/biometrics.db' }]) } as unknown as Parameters[0] +const policy = { qualificationId: 'test-evidence', minimumAvailableMemoryBytes: 100, minimumFreeDiskBytes: 100, + maximumLoadPerCpu: 1, maximumEventLoopDelayMs: 100, maximumPiezoAgeMs: 10000, maximumTemperatureC: 65, maximumSampleAgeMs: 1000 } +afterEach(() => { + vi.unstubAllEnvs() + vi.clearAllMocks() +}) + +it.each([undefined, '', '{', '{}', JSON.stringify({ ...policy, maximumPiezoAgeMs: 0 })])('denies missing or invalid server policy without collecting health: %s', async (value) => { + vi.stubEnv('SLEEPYPOD_BIOMETRICS_ADMISSION_POLICY', value) + expect(await replayAdmission(db)).toEqual({ configured: false, qualificationId: null, admitted: false, reasons: ['unqualified'] }) + expect(collectAdmissionSample).not.toHaveBeenCalled() +}) + +it('uses fresh piezo receipt times and reevaluates health for every admission', async () => { + vi.stubEnv('SLEEPYPOD_BIOMETRICS_ADMISSION_POLICY', JSON.stringify(policy)) + vi.mocked(collectAdmissionSample).mockImplementation(async (directory, piezo) => { + expect(directory).toBe('/private') + expect(piezo()).toBe(frames['piezo-dual']) + return { measuredAtMs: Date.now(), piezoReceivedAtMs: Date.now(), availableMemoryBytes: 200, freeDiskBytes: 200, + loadPerCpu: 0.2, eventLoopDelayMs: 2, temperatureC: 30 } + }) + expect(await replayAdmission(db)).toEqual({ configured: true, qualificationId: 'test-evidence', admitted: true, reasons: [] }) + vi.mocked(collectAdmissionSample).mockResolvedValueOnce(null) + expect(await replayAdmitter(db)()).toBe(false) + expect(collectAdmissionSample).toHaveBeenCalledTimes(2) +}) + +it('fails closed when collecting health throws', async () => { + vi.stubEnv('SLEEPYPOD_BIOMETRICS_ADMISSION_POLICY', JSON.stringify(policy)) + vi.mocked(collectAdmissionSample).mockRejectedValueOnce(new Error('unavailable')) + expect(await replayAdmission(db)).toEqual({ configured: true, qualificationId: 'test-evidence', admitted: false, reasons: ['health_unavailable'] }) +}) + +it('fails closed without a persistent database path or a configured policy', async () => { + const { replayHealthAdmission } = await import('../admission-service') + vi.stubEnv('SLEEPYPOD_BIOMETRICS_ADMISSION_POLICY', JSON.stringify(policy)) + const memory = { all: vi.fn(() => [{ name: 'main', file: '' }]) } as unknown as Parameters[0] + expect(await replayAdmission(memory)).toEqual({ configured: true, qualificationId: 'test-evidence', admitted: false, reasons: ['health_unavailable'] }) + expect(collectAdmissionSample).not.toHaveBeenCalled() + vi.stubEnv('SLEEPYPOD_BIOMETRICS_ADMISSION_POLICY', '') + expect(await replayHealthAdmission(db)()).toEqual({ allowed: false, sample: null }) +}) + +it('returns the admitted sample only through the internal health callback', async () => { + const { replayHealthAdmission } = await import('../admission-service') + vi.stubEnv('SLEEPYPOD_BIOMETRICS_ADMISSION_POLICY', JSON.stringify(policy)) + vi.mocked(collectAdmissionSample).mockImplementation(async () => ({ measuredAtMs: Date.now(), piezoReceivedAtMs: Date.now() - 250, + availableMemoryBytes: 200, freeDiskBytes: 200, loadPerCpu: 0.2, eventLoopDelayMs: 2, temperatureC: 30 })) + const measured = await replayHealthAdmission(db)() + expect(measured).toMatchObject({ allowed: true, sample: { qualificationId: 'test-evidence', ingestionAgeMs: expect.any(Number) } }) + expect(measured.sample?.ingestionAgeMs).toBeGreaterThanOrEqual(250) + expect(await replayAdmission(db)).not.toHaveProperty('receipt') + vi.mocked(collectAdmissionSample).mockResolvedValueOnce(null) + expect(await replayHealthAdmission(db)()).toEqual({ allowed: false, sample: null }) +}) diff --git a/src/biometrics/backtest/tests/admission.test.ts b/src/biometrics/backtest/tests/admission.test.ts new file mode 100644 index 00000000..a15e1465 --- /dev/null +++ b/src/biometrics/backtest/tests/admission.test.ts @@ -0,0 +1,39 @@ +// @vitest-environment node +import { describe, expect, it } from 'vitest' +import { evaluateAdmission, parseAvailableMemory, type AdmissionPolicy, type AdmissionSample } from '../admission' +// Test values deliberately are not deployment defaults. +const policy: AdmissionPolicy = { qualificationId: 'fixture', minimumAvailableMemoryBytes: 1000, minimumFreeDiskBytes: 2000, + maximumLoadPerCpu: 0.8, maximumEventLoopDelayMs: 50, maximumPiezoAgeMs: 1000, maximumTemperatureC: 60, maximumSampleAgeMs: 500 } +const sample: AdmissionSample = { measuredAtMs: 10000, availableMemoryBytes: 1000, freeDiskBytes: 2000, + loadPerCpu: 0.8, eventLoopDelayMs: 50, piezoReceivedAtMs: 9500, temperatureC: 60 } +describe('replay resource admission', () => { + it('admits fresh evidence at exact qualified boundaries', () => { + expect(evaluateAdmission(sample, policy, 10500)).toEqual({ admitted: true, reasons: [] }) + }) + it.each([ + ['availableMemoryBytes', 999, 'memory_pressure'], ['freeDiskBytes', 1999, 'disk_pressure'], + ['loadPerCpu', 0.81, 'cpu_pressure'], ['eventLoopDelayMs', 51, 'event_loop_delay'], + ['piezoReceivedAtMs', 8999, 'piezo_stale'], ['temperatureC', 61, 'temperature'], + ['measuredAtMs', 9499, 'stale_sample'], ['piezoReceivedAtMs', 10001, 'clock_skew'], + ['measuredAtMs', 10001, 'clock_skew'], + ])('defers for %s outside the qualified range', (key, value, reason) => { + expect(evaluateAdmission({ ...sample, [key]: value }, policy, 10000)).toEqual({ admitted: false, reasons: [reason] }) + }) + it('does not turn absent, malformed, or nonfinite health evidence into zeros', () => { + for (const invalid of [null, {}, { ...sample, availableMemoryBytes: undefined }, { ...sample, temperatureC: NaN }, { ...sample, freeDiskBytes: Infinity }]) { + expect(evaluateAdmission(invalid, policy, 10000)).toEqual({ admitted: false, reasons: ['health_unavailable'] }) + } + expect(evaluateAdmission(sample, null, 10000)).toEqual({ admitted: false, reasons: ['unqualified'] }) + expect(evaluateAdmission(sample, { ...policy, maximumLoadPerCpu: 0 }, 10000).admitted).toBe(false) + }) + it('reports simultaneous pressure without hiding subsequent reasons', () => { + expect(evaluateAdmission({ ...sample, availableMemoryBytes: 0, freeDiskBytes: 0, temperatureC: 70 }, policy, 10000).reasons) + .toEqual(['memory_pressure', 'disk_pressure', 'temperature']) + }) + it('reads reclaimable available Linux memory, not only unused pages', () => { + expect(parseAvailableMemory('MemTotal: 2000 kB\nMemFree: 100 kB\nMemAvailable: 1200 kB\n')).toBe(1228800) + for (const invalid of ['MemFree: 100 kB', 'MemAvailable: -1 kB', 'MemAvailable: 2 MB', 'MemAvailable: 99999999999999999 kB']) { + expect(() => parseAvailableMemory(invalid)).toThrow() + } + }) +}) diff --git a/src/biometrics/backtest/tests/discovery.test.ts b/src/biometrics/backtest/tests/discovery.test.ts new file mode 100644 index 00000000..2242bcfd --- /dev/null +++ b/src/biometrics/backtest/tests/discovery.test.ts @@ -0,0 +1,62 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { afterEach, beforeEach, expect, it } from 'vitest' +import * as schema from '@/src/db/biometrics-schema' +import { listReplaySessions, sessionCapabilities } from '../discovery' +let sqlite: Database.Database +let db: ReturnType> +const start = 1791673200000 +function session(id = 1, hours = 1, side: 'left' | 'right' = 'left') { + db.insert(schema.sleepRecords).values({ id, side, enteredBedAt: new Date(start), leftBedAt: new Date(start + hours * 3600000), sleepDurationSeconds: Math.max(0, hours * 3600), timesExitedBed: 0 }).run() +} +beforeEach(() => { + sqlite = new Database(':memory:') + db = drizzle(sqlite, { schema }) + migrate(db, { migrationsFolder: 'src/db/biometrics-migrations' }) +}) +afterEach(() => sqlite.close()) +it('pages equal start times without duplicates and keeps sides separate', () => { + for (let id = 1; id <= 22; id++) session(id) + session(23, 1, 'right') + const first = listReplaySessions(db, 'left') + expect(first.items.map(row => row.id)).toEqual(Array.from({ length: 20 }, (_, i) => 22 - i)) + expect(first.nextCursor).toEqual({ startMs: start, id: 3 }) + if (!first.nextCursor) throw new Error('Expected another page') + const next = listReplaySessions(db, 'left', first.nextCursor) + expect(next.items.map(row => row.id)).toEqual([2, 1]) + expect(next.nextCursor).toBeNull() + expect(listReplaySessions(db, 'right').items.map(row => row.id)).toEqual([23]) +}) +it('reports an empty session without implying retained waveforms or stage support', () => { + session() + expect(sessionCapabilities(db, 'left', 1)).toMatchObject({ comparison: false, stageReplay: false, signalReplay: false, + processed: { legacyUnknownProvenanceCount: 0, vitals: { count: 0 }, movement: { count: 0 } }, referenceMetrics: [], + limitations: expect.arrayContaining(['No stored vitals are available for comparison or stage replay.']) }) + expect(() => sessionCapabilities(db, 'right', 1)).toThrow('not found for this side') +}) +it.each([0, -1, 37])('rejects invalid or over-budget session duration %s hours', (hours) => { + session(1, hours) + expect(() => sessionCapabilities(db, 'left', 1)).toThrow('boundaries exceed') +}) +it.each([24, 25])('distinguishes comparison from the one-day stage replay limit at %s hours', (hours) => { + session(1, hours) + db.insert(schema.biometricProcessingRuns).values({ id: 'known', side: 'left', kind: 'live', bundle: {}, configHash: 'fixture', inputIdentity: {}, startedAtMs: start }).run() + db.insert(schema.vitals).values([ + { side: 'left', timestamp: new Date(start), heartRate: 60, processingRunId: 'known' }, + { side: 'left', timestamp: new Date(start + 60000), heartRate: 61 }, + { side: 'right', timestamp: new Date(start), heartRate: 70 }, + { side: 'left', timestamp: new Date(start + hours * 3600000), heartRate: 80 }, + ]).run() + db.insert(schema.movement).values([{ side: 'left', timestamp: new Date(start + 120000), totalMovement: 3 }, { side: 'right', timestamp: new Date(start + 120000), totalMovement: 9 }]).run() + const result = sessionCapabilities(db, 'left', 1) + expect(result).toMatchObject({ comparison: true, stageReplay: hours === 24, processed: { legacyUnknownProvenanceCount: 1, vitals: { count: 2 }, movement: { count: 1 } } }) + expect(result.limitations.includes('Stage replay currently requires a merged window of at most one day.')).toBe(hours > 24) +}) +it('bounds aggregate processed rows before returning an incomplete inventory', () => { + session(1, 6) + sqlite.exec(`WITH RECURSIVE rows(n) AS (SELECT 1 UNION ALL SELECT n+1 FROM rows WHERE n < 20001) + INSERT INTO vitals(side,timestamp,heart_rate) SELECT 'left', ${start / 1000} + n, 60 FROM rows`) + expect(() => sessionCapabilities(db, 'left', 1)).toThrow('processed-data row budget') +}) diff --git a/src/biometrics/backtest/tests/execution-budget.test.ts b/src/biometrics/backtest/tests/execution-budget.test.ts new file mode 100644 index 00000000..cd43e128 --- /dev/null +++ b/src/biometrics/backtest/tests/execution-budget.test.ts @@ -0,0 +1,12 @@ +// @vitest-environment node +import { expect, it } from 'vitest' +import { replayBudgetSchema } from '../execution-budget' + +const budget = { jobWallMs: 3600000, waveformWallMs: 600000, waveformCpuSeconds: 300 } +it('accepts nested ceilings and rejects budgets that cannot fit inside the job', () => { + expect(replayBudgetSchema.parse(budget)).toEqual(budget) + expect(() => replayBudgetSchema.parse({ ...budget, waveformWallMs: budget.jobWallMs + 1000 })).toThrow('fit within the job budget') + expect(() => replayBudgetSchema.parse({ ...budget, waveformCpuSeconds: 601 })).toThrow('fit within the job budget') + expect(() => replayBudgetSchema.parse({ ...budget, jobWallMs: 999 })).toThrow() + expect(() => replayBudgetSchema.parse({ ...budget, extra: true })).toThrow() +}) diff --git a/src/biometrics/backtest/tests/night-summary.test.ts b/src/biometrics/backtest/tests/night-summary.test.ts new file mode 100644 index 00000000..9de65405 --- /dev/null +++ b/src/biometrics/backtest/tests/night-summary.test.ts @@ -0,0 +1,56 @@ +// @vitest-environment node +import { expect, it } from 'vitest' +import { compareStageOutputs } from '../stage-comparison' +import { summarizeStageNights } from '../night-summary' +import { previewReferenceImport } from '../../references/normalize' +const startMs = 1791673200000 +function night(cohortId: string, duration: number, improved: boolean, missing = false) { + const endMs = startMs + duration + const references = missing + ? [] + : previewReferenceImport({ side: 'left', subjectId: 'fixture', source: 'test', rows: [ + { recordKey: 'stage', metric: 'sleep_stages', value: 'light', unit: 'stage', startMs, endMs }, + ] }).measurements + const score = compareStageOutputs([{ start: startMs, end: endMs, stage: improved ? 'deep' : 'light' }], + [{ start: startMs, end: endMs, stage: improved ? 'light' : 'deep' }], references, startMs, endMs) + return { cohortId, comparisons: [{ baseline: 'default', candidate: 'candidate', baselineHash: 'a', candidateHash: 'b', windows: [{ ...score, startMs, endMs }] }] } +} +it('keeps equal-night effects distinct from pooled duration effects and includes unscored nights', () => { + const result = summarizeStageNights([night('long', 900000, true), night('short', 100000, false), night('unknown', 500000, true, true)])[0] + expect(result).toMatchObject({ totalNights: 3, scoredNights: 2, unscoredNights: 1, improvedAgreementNights: 1, + regressedAgreementNights: 1, equalNightMeanAgreementDifference: 0, observedNightRange: { minimum: -1, maximum: 1 } }) + expect(result.pooledCommonSupport.candidate.agreement).toBe(0.9) + expect(result.pooledCommonSupport.baseline.agreement).toBe(0.1) + expect(result.nights[2].commonAgreementDifference).toBeNull() +}) +it('combines window confusion durations before computing nightly F1', () => { + const first = night('one', 900000, true), second = night('two', 100000, false) + second.comparisons[0].windows[0].startMs += 900000 + second.comparisons[0].windows[0].endMs += 900000 + first.comparisons[0].windows.push(second.comparisons[0].windows[0]) + const result = summarizeStageNights([first])[0] + expect(result.nights[0].commonSupport.candidate.agreement).toBe(0.9) + expect(result.nights[0].commonSupport.candidate.macroF1).toBeCloseTo(0.9 / 1.9) + expect(result.nights[0].commonSupport.candidate.pairedDurationMs).toBe(1000000) +}) +it('never merges different hashes or counts duplicated cohort/comparison evidence twice', () => { + const first = night('one', 100000, true), second = night('two', 100000, true) + second.comparisons[0].candidateHash = 'different' + expect(summarizeStageNights([first, second])).toHaveLength(2) + expect(() => summarizeStageNights([first, first])).toThrow('whole-night') + first.comparisons.push(first.comparisons[0]) + expect(() => summarizeStageNights([first])).toThrow('Duplicate comparison') +}) +it('keeps all-missing evidence unknown', () => { + const result = summarizeStageNights([night('missing', 100000, true, true)])[0] + expect(result.equalNightMeanAgreementDifference).toBeNull() + expect(result.observedNightRange).toBeNull() + expect(result.pooledCommonSupport.candidate.agreement).toBeNull() + expect(summarizeStageNights([])).toEqual([]) +}) + +it('rejects overlapping windows instead of double-counting their support', () => { + const input = night('one', 100000, true) + input.comparisons[0].windows.push(input.comparisons[0].windows[0]) + expect(() => summarizeStageNights([input])).toThrow('nonoverlapping') +}) diff --git a/src/biometrics/backtest/tests/numeric-traces.test.ts b/src/biometrics/backtest/tests/numeric-traces.test.ts new file mode 100644 index 00000000..423e0940 --- /dev/null +++ b/src/biometrics/backtest/tests/numeric-traces.test.ts @@ -0,0 +1,46 @@ +// @vitest-environment node +import { expect, it } from 'vitest' +import { pageNumericTraces } from '../numeric-trace-page' +const hash = 'b'.repeat(64) +const request = { id: '11111111-1111-4111-8111-111111111111', expectedContentHash: 'a'.repeat(64), configuration: 'candidate' as const, limit: 2 } +function fixture() { + const reading = { side: 'left', sourceSide: 'right', configHash: hash, timestampMs: 1000, heartRate: 60, breathingRate: null, hrvIndex: null, + hrvMetric: 'window_ibi_variability_v1', qualityScore: 0, flags: null, limitations: ['unknown_context'], hrRaw: 65 } + return { version: 'waveform-backtest-report-v1', side: 'left', protocol: { mode: 'signal_replay', clockOffsetMs: 60000 }, + nights: [{ cohortId: 'one', evaluations: ['default', 'candidate'].map(label => ({ label, configHash: hash, revisionId: null, windows: [{ startMs: 1000, endMs: 5000 }] })) }], + waveform: { outputs: [{ cohortId: 'one', configHash: hash, readings: [ + { ...reading, timestampMs: 999 }, reading, { ...reading, side: 'right', heartRate: 99 }, + { ...reading, timestampMs: 2000, heartRate: null }, { ...reading, timestampMs: 4000 }, { ...reading, timestampMs: 5000 }, + ] }] } } +} +it('pages exact dataset-side readings without offset, downsampling, opposite-side values or private extra fields', () => { + const report = fixture(), before = structuredClone(report) + const first = pageNumericTraces(report, request) + expect(first).toMatchObject({ totalItems: 3, side: 'left', downsampled: false, clockOffsetMs: 60000, + provenance: 'retained_waveform_recomputation', nextCursor: { offset: 2, contentHash: request.expectedContentHash } }) + expect(first.items.map(row => row.timestampMs)).toEqual([1000, 2000]) + expect(first.items[1].heartRate).toBeNull() + expect(first.items[0]).not.toHaveProperty('hrRaw') + expect(first.items[0].sourceSide).toBe('right') + const last = pageNumericTraces(report, { ...request, offset: first.nextCursor?.offset }) + expect(last.items.map(row => row.timestampMs)).toEqual([4000]) + expect(last.nextCursor).toBeNull() + expect(pageNumericTraces(report, { ...request, configuration: 'default' }).items).toEqual(first.items) + expect(report).toEqual(before) +}) +it('rejects missing provenance, ambiguous output identity and invalid selections', () => { + expect(() => pageNumericTraces({ ...fixture(), side: undefined }, request)).toThrow('provenance') + const duplicate = fixture() + duplicate.waveform.outputs.push(duplicate.waveform.outputs[0]) + expect(() => pageNumericTraces(duplicate, request)).toThrow('ambiguous') + const mismatch = fixture() + mismatch.waveform.outputs[0].readings[1].configHash = 'c'.repeat(64) + expect(() => pageNumericTraces(mismatch, request)).toThrow('configuration mismatch') + const repeated = fixture() + repeated.nights[0].evaluations.push(repeated.nights[0].evaluations[1]) + expect(() => pageNumericTraces(repeated, request)).toThrow('Duplicate numeric configuration') + expect(() => pageNumericTraces(fixture(), { ...request, cohortId: 'absent' })).toThrow('not evaluated') + expect(() => pageNumericTraces(fixture(), { ...request, configuration: 'current' })).toThrow('not evaluated') + expect(() => pageNumericTraces(fixture(), { ...request, offset: 4 })).toThrow('cursor') + expect(() => pageNumericTraces(fixture(), { ...request, limit: 501 })).toThrow() +}) diff --git a/src/biometrics/backtest/tests/process-group-memory.test.ts b/src/biometrics/backtest/tests/process-group-memory.test.ts new file mode 100644 index 00000000..06bffa92 --- /dev/null +++ b/src/biometrics/backtest/tests/process-group-memory.test.ts @@ -0,0 +1,107 @@ +// @vitest-environment node +import { mkdtemp, mkdir, writeFile, rm } from 'node:fs/promises' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { afterEach, expect, it, vi } from 'vitest' +import { monitorProcessGroupMemory, sampleProcessGroupRss } from '../process-group-memory' +let root: string | undefined +afterEach(async () => { + vi.useRealTimers() + if (root) await rm(root, { recursive: true, force: true }) + root = undefined +}) +async function processFixture(pid: number, group: number, rss: string | null, state = 'S') { + if (!root) root = await mkdtemp(join(tmpdir(), 'replay-proc-')) + const directory = join(root, String(pid)) + await mkdir(directory, { recursive: true }) + await writeFile(join(directory, 'stat'), `${pid} (name with ) parentheses) ${state} 1 ${group} 0 0\n`) + if (rss !== null) await writeFile(join(directory, 'status'), rss) + return directory +} +it('sums only the owned process group and treats disappearing members as an observation race', async () => { + await processFixture(10, 10, 'VmRSS:\t100 kB\n') + await processFixture(11, 10, 'VmRSS:\t200 kB\n') + await processFixture(12, 12, 'VmRSS:\t999 kB\n') + await processFixture(13, 10, null) + await processFixture(14, 10, 'Name: zombie\n', 'Z') + expect(await sampleProcessGroupRss(10, root)).toBe(300 * 1024) + expect(await sampleProcessGroupRss(99, root)).toBeNull() +}) +it('reports unavailable memory for malformed owned RSS and excessive group size', async () => { + const directory = await processFixture(10, 10, 'VmRSS: unknown\n') + expect(await sampleProcessGroupRss(10, root)).toBeNull() + await writeFile(join(directory, 'status'), 'VmRSS: 99999999999999999999999 kB\n') + expect(await sampleProcessGroupRss(10, root)).toBeNull() + for (let pid = 10; pid < 75; pid++) await processFixture(pid, 10, 'VmRSS: 1 kB\n') + expect(await sampleProcessGroupRss(10, root)).toBeNull() +}) +it('reports unavailable memory for invalid groups, unreadable tables and malformed procfs records', async () => { + expect(await sampleProcessGroupRss(0, tmpdir())).toBeNull() + expect(await sampleProcessGroupRss(1.5, tmpdir())).toBeNull() + expect(await sampleProcessGroupRss(10, join(tmpdir(), 'replay-proc-missing'))).toBeNull() + const directory = await processFixture(10, 10, 'VmRSS:\t100 kB\n') + for (const stat of ['x'.repeat(16385), '10 (name S 1 10 0 0\n', '10 (name) S 1\n']) { + await writeFile(join(directory, 'stat'), stat) + expect(await sampleProcessGroupRss(10, root)).toBeNull() + } + await writeFile(join(directory, 'stat'), '10 (name) S 1 10 0 0\n') + await writeFile(join(directory, 'status'), `VmRSS:\t100 kB\n${'x'.repeat(16385)}`) + expect(await sampleProcessGroupRss(10, root)).toBeNull() + await writeFile(join(directory, 'status'), 'VmRSS:\t100 kB\n') + expect(await sampleProcessGroupRss(10, root)).toBe(100 * 1024) + if (!root) throw new Error('Missing fixture root') + await Promise.all(Array.from({ length: 4100 }, (_, index) => mkdir(join(root as string, String(index + 100)), { recursive: true }))) + expect(await sampleProcessGroupRss(10, root)).toBeNull() +}) +it('falls back to the platform sampler, records failed reads and samples on its interval', async () => { + vi.useFakeTimers() + const platform = monitorProcessGroupMemory(1) + await platform.sample() + expect(platform.snapshot()).toMatchObject({ attempts: 1, intervalMs: 1000, exactKernelPeak: false }) + await platform.stop() + const read = vi.fn().mockRejectedValueOnce(new Error('procfs unavailable')).mockResolvedValue(700) + const monitor = monitorProcessGroupMemory(1, { sample: read }) + await monitor.sample() + expect(monitor.snapshot()).toMatchObject({ attempts: 1, samples: 0, allSamplesAvailable: false, peakObservedRssBytes: null }) + await vi.advanceTimersByTimeAsync(2000) + await monitor.stop() + expect(read).toHaveBeenCalledTimes(3) + expect(monitor.snapshot()).toMatchObject({ attempts: 3, samples: 2, allSamplesAvailable: false, peakObservedRssBytes: 700 }) +}) +it('retains sampled peaks, missing samples and gaps and stops its periodic work', async () => { + vi.useFakeTimers() + let time = 0 + const read = vi.fn().mockResolvedValueOnce(100).mockResolvedValueOnce(null).mockResolvedValue(300) + const monitor = monitorProcessGroupMemory(1, { sample: read, now: () => time }) + await monitor.sample() + time = 2000 + await monitor.sample() + time = 2200 + await monitor.sample() + expect(monitor.snapshot()).toMatchObject({ attempts: 3, samples: 2, allSamplesAvailable: false, + peakObservedRssBytes: 300, maximumObservationGapMs: 2000, exactKernelPeak: false }) + await monitor.stop() + await vi.advanceTimersByTimeAsync(5000) + await monitor.sample() + expect(read).toHaveBeenCalledTimes(3) +}) +it('serializes slow reads and waits for them when stopped', async () => { + let resolve: (value: number) => void = () => {} + const read = vi.fn(() => new Promise((done) => { + resolve = done + })) + const monitor = monitorProcessGroupMemory(1, { sample: read }) + const first = monitor.sample(), second = monitor.sample() + await Promise.resolve() + expect(read).toHaveBeenCalledTimes(1) + let stopped = false + const stopping = monitor.stop().then(() => { + stopped = true + }) + await Promise.resolve() + expect(stopped).toBe(false) + resolve(500) + await Promise.all([first, second, stopping]) + expect(stopped).toBe(true) + expect(monitor.snapshot().peakObservedRssBytes).toBe(500) +}) diff --git a/src/biometrics/backtest/tests/raw-collection-child.test.ts b/src/biometrics/backtest/tests/raw-collection-child.test.ts new file mode 100644 index 00000000..b93a8eb7 --- /dev/null +++ b/src/biometrics/backtest/tests/raw-collection-child.test.ts @@ -0,0 +1,57 @@ +// @vitest-environment node +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, expect, it } from 'vitest' +import { runRawCollectionChild } from '../raw-collection-child' +let directory: string, executable: string +const location = () => ({ pythonPath: executable, modulesPath: directory, databasePath: join(directory, 'db'), retainedDirectory: join(directory, 'raw') }) +beforeEach(async () => { + directory = await mkdtemp(join(tmpdir(), 'collection-child-')) + executable = join(directory, 'worker') +}) +afterEach(() => rm(directory, { recursive: true, force: true })) +async function worker(body: string) { + await writeFile(executable, `#!${process.execPath}\nlet input='';process.stdin.on('data',c=>input+=c);process.stdin.on('end',()=>{${body}});`, { mode: 0o700 }) +} +const result = { version: 'raw-collection-result-v1', released: 2, orphans: 1 } +const emit = `process.stdout.write(${JSON.stringify(JSON.stringify(result))});` +it('uses fixed module, trusted paths and an input with no caller-controlled retention settings', async () => { + const args = ['-B', '-m', 'common.biometrics.collection_worker', '--database-path', location().databasePath, '--retained-directory', location().retainedDirectory] + await worker(`if(JSON.stringify(process.argv.slice(2))!==${JSON.stringify(JSON.stringify(args))} || input!==${JSON.stringify(JSON.stringify({ version: 'raw-collection-work-v1' }))})process.exit(1);${emit}`) + expect(await runRawCollectionChild(location())).toEqual(result) +}) +it('rejects malformed output and unsuccessful exit', async () => { + await worker('process.stdout.write("{}");') + await expect(runRawCollectionChild(location())).rejects.toThrow('invalid or incomplete') + await worker(`${emit}process.exitCode=1;`) + await expect(runRawCollectionChild(location())).rejects.toThrow('invalid or incomplete') +}) +it('bounds output, diagnostics and time, and cancels a running child', async () => { + await worker('process.stdout.write("x".repeat(1025));') + await expect(runRawCollectionChild(location())).rejects.toThrow('output exceeded') + await worker('process.stderr.write("x".repeat(16385));') + await expect(runRawCollectionChild(location())).rejects.toThrow('diagnostics exceeded') + await worker('setInterval(()=>{},1000);') + await expect(runRawCollectionChild(location(), { timeoutMs: 50 })).rejects.toThrow('time budget') + const controller = new AbortController() + const promise = runRawCollectionChild(location(), { signal: controller.signal }) + setTimeout(() => controller.abort(), 30) + await expect(promise).rejects.toThrow('cancelled') +}) +it('rejects invalid budgets, pre-cancelled work and unstartable workers, and escalates to SIGKILL', async () => { + expect(() => runRawCollectionChild(location(), { timeoutMs: 120001 })).toThrow('time budget') + expect(() => runRawCollectionChild(location(), { timeoutMs: 9 })).toThrow('time budget') + const controller = new AbortController() + controller.abort() + await expect(runRawCollectionChild(location(), { signal: controller.signal })).rejects.toThrow('cancelled') + await expect(runRawCollectionChild({ ...location(), pythonPath: join(directory, 'missing') })).rejects.toThrow('could not start') + await worker('process.on("SIGTERM",()=>{});setInterval(()=>{},1000);') + await expect(runRawCollectionChild(location(), { timeoutMs: 1500 })).rejects.toThrow('time budget') +}) +it('ignores output after a failure and tolerates bounded diagnostics on success', async () => { + await worker('process.stderr.write("x".repeat(16385));setTimeout(()=>{process.stderr.write("more");process.stdout.write("late")},150);process.on("SIGTERM",()=>setTimeout(()=>process.exit(0),200));') + await expect(runRawCollectionChild(location())).rejects.toThrow('diagnostics exceeded') + await worker(`process.stderr.write("note");${emit}`) + expect(await runRawCollectionChild(location())).toEqual(result) +}) diff --git a/src/biometrics/backtest/tests/raw-publication-child.test.ts b/src/biometrics/backtest/tests/raw-publication-child.test.ts new file mode 100644 index 00000000..63d0756f --- /dev/null +++ b/src/biometrics/backtest/tests/raw-publication-child.test.ts @@ -0,0 +1,67 @@ +// @vitest-environment node +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, expect, it } from 'vitest' +import { runRawPublicationChild } from '../raw-publication-child' +let directory: string, executable: string +const hash = 'a'.repeat(64) +// This protocol fixture never runs Python or publishes rows. +const work = { prepared: { contentHash: hash }, selections: [] } as unknown as Parameters[1] +const result = { type: 'result', id: '11111111-1111-4111-8111-111111111111', contentHash: hash, available: true } +const location = () => ({ pythonPath: executable, modulesPath: directory, archiveDirectory: directory, retainedDirectory: join(directory, 'raw'), databasePath: join(directory, 'test.db') }) +beforeEach(async () => { + directory = await mkdtemp(join(tmpdir(), 'publication-child-')) + executable = join(directory, 'worker') +}) +afterEach(() => rm(directory, { recursive: true, force: true })) +async function worker(body: string) { + await writeFile(executable, `#!${process.execPath}\nsetTimeout(() => process.exit(91), 3000).unref(); let input='';process.stdin.on('data',c=>input+=c);process.stdin.on('end',()=>{${body}});`, { mode: 0o700 }) +} +const emit = (value: unknown) => `process.stdout.write(${JSON.stringify(JSON.stringify(value))});` +it('uses fixed publication arguments, a clean environment and the versioned envelope', async () => { + await worker(`if(JSON.stringify(process.argv.slice(2))!==${JSON.stringify(JSON.stringify(['-B', '-m', 'common.biometrics.publication_worker', '--database', location().databasePath, '--archive-directory', directory, '--retained-directory', location().retainedDirectory]))} || process.env.RAW_TEST_SECRET || process.env.PYTHONPATH!==${JSON.stringify(directory)} || input!==${JSON.stringify(JSON.stringify({ version: 'waveform-publication-work-v1', ...work }))})process.exit(1);${emit(result)}`) + process.env.RAW_TEST_SECRET = 'private' + try { + expect(await runRawPublicationChild(location(), work)).toEqual(result) + } + finally { delete process.env.RAW_TEST_SECRET } + expect(() => runRawPublicationChild(location(), work, { timeoutMs: 120001 })).toThrow('time budget') +}) +it.each([ + {}, { ...result, contentHash: 'b'.repeat(64) }, { ...result, id: 'bad' }, { ...result, extra: true }, +])('rejects invalid identities or unexpected fields %#', async (output) => { + await worker(emit(output)) + await expect(runRawPublicationChild(location(), work)).rejects.toThrow('invalid or incomplete') +}) +it('rejects extra output, invalid UTF-8 and unsuccessful exit after a response', async () => { + for (const suffix of ['process.stdout.write("{}");', 'process.stdout.write(Buffer.from([0xc3]));', 'process.exitCode=1;']) { + await worker(`${emit(result)}${suffix}`) + await expect(runRawPublicationChild(location(), work)).rejects.toThrow('invalid or incomplete') + } +}) +it('bounds input, output, diagnostics and execution, and cancels work', async () => { + await expect(runRawPublicationChild(location(), { ...work, padding: 'x'.repeat(10 * 1024 * 1024) } as typeof work)).rejects.toThrow('input exceeds') + await worker('process.stdout.write("x".repeat(65537));') + await expect(runRawPublicationChild(location(), work)).rejects.toThrow('output exceeded') + await worker('process.stderr.write("x".repeat(16385));') + await expect(runRawPublicationChild(location(), work)).rejects.toThrow('diagnostics exceeded') + await worker('setInterval(()=>{},1000);') + await expect(runRawPublicationChild(location(), work, { timeoutMs: 30 })).rejects.toThrow('time budget') + const controller = new AbortController() + const running = runRawPublicationChild(location(), work, { signal: controller.signal }) + controller.abort() + await expect(running).rejects.toThrow('cancelled') + await expect(runRawPublicationChild(location(), work, { signal: controller.signal })).rejects.toThrow('cancelled') + await expect(runRawPublicationChild({ ...location(), pythonPath: join(directory, 'missing') }, work)).rejects.toThrow('could not start') +}) +it('escalates to SIGKILL, ignores output after a failure and tolerates bounded diagnostics', async () => { + await worker('process.on("SIGTERM",()=>{});setInterval(()=>{},1000);') + await expect(runRawPublicationChild(location(), work, { timeoutMs: 1500 })).rejects.toThrow('time budget') + await worker('process.stderr.write("x".repeat(16385));setTimeout(()=>{process.stderr.write("more");process.stdout.write("late")},150);process.on("SIGTERM",()=>setTimeout(()=>process.exit(0),200));') + await expect(runRawPublicationChild(location(), work)).rejects.toThrow('diagnostics exceeded') + await writeFile(executable, `#!${process.execPath}\nprocess.exit(3)`, { mode: 0o700 }) + await expect(runRawPublicationChild(location(), { ...work, padding: 'x'.repeat(4 * 1024 * 1024) } as typeof work)).rejects.toThrow('input pipe closed') + await worker(`process.stderr.write("note");${emit(result)}`) + expect(await runRawPublicationChild(location(), work)).toEqual(result) +}) diff --git a/src/biometrics/backtest/tests/reference-traces.test.ts b/src/biometrics/backtest/tests/reference-traces.test.ts new file mode 100644 index 00000000..8b8d7ed0 --- /dev/null +++ b/src/biometrics/backtest/tests/reference-traces.test.ts @@ -0,0 +1,50 @@ +// @vitest-environment node +import { expect, it } from 'vitest' +import { pageReferenceTraces } from '../reference-trace-page' +import { evaluateBacktest } from '../evaluator' +import { backtestFixture } from './fixtures/backtest' +import { previewReferenceImport } from '../../references/normalize' +const request = { id: '11111111-1111-4111-8111-111111111111', expectedContentHash: 'a'.repeat(64), cohortId: 'night', metric: 'heart_rate' as const, limit: 1 } +function fixture() { + const work = backtestFixture(), report = evaluateBacktest(work) + return { report, dataset: { ...work.dataset, contentHash: report.datasetHash } } +} +it('pages frozen source revisions without annotations or submitted identity and preserves point times', () => { + const { report, dataset } = fixture() + const row = dataset.snapshot.references[0].measurements[1] + row.original.annotation = 'private note' + row.original.subjectId = 'private subject' + dataset.snapshot.references.push({ ...dataset.snapshot.references[0], importId: 'second', revision: 2, source: 'another source' }) + const first = pageReferenceTraces(report, dataset, request) + expect(first).toMatchObject({ totalItems: 2, side: 'left', provenance: 'frozen_dataset_reference', timestampBasis: 'reference_source_time', downsampled: false, + nextCursor: { offset: 1, contentHash: request.expectedContentHash }, items: [{ source: 'fixture', importRevision: 1, temporalKind: 'point', value: 62, startMs: row.startMs }] }) + expect(JSON.stringify(first)).not.toContain('private') + expect(first.items[0]).not.toHaveProperty('original') + const second = pageReferenceTraces(report, dataset, { ...request, offset: first.nextCursor?.offset }) + expect(second.items[0].importId).toBe('second') + expect(second.nextCursor).toBeNull() +}) +it('keeps interval semantics and incompatible HRV visible without claiming a paired score', () => { + const { report, dataset } = fixture() + const start = dataset.snapshot.sessions[0].startMs + dataset.snapshot.references[0].measurements.push(...previewReferenceImport({ side: 'left', source: 'watch', subjectId: 'A', rows: [ + { recordKey: 'hrv', metric: 'hrv_rmssd', value: 30, unit: 'ms', startMs: start, endMs: start + 300000, windowMs: 300000 }, + ] }).measurements) + const page = pageReferenceTraces(report, dataset, { ...request, metric: 'window_ibi_variability_v1' }) + expect(page.items[0]).toMatchObject({ temporalKind: 'interval', metric: 'hrv_rmssd', compatibility: 'incompatible_metric_definition', endMs: start + 300000 }) + expect(page.items[0]).not.toHaveProperty('paired') +}) +it('filters scoring windows and fails on stale identities or invalid pages', () => { + const { report, dataset } = fixture() + const row = dataset.snapshot.references[0].measurements[1] + dataset.snapshot.references[0].measurements.push({ ...row, startMs: dataset.snapshot.sessions[0].endMs, endMs: dataset.snapshot.sessions[0].endMs }) + expect(pageReferenceTraces(report, dataset, request).totalItems).toBe(1) + expect(() => pageReferenceTraces(report, { ...dataset, contentHash: 'other' }, request)).toThrow('hash changed') + expect(() => pageReferenceTraces(report, dataset, { ...request, cohortId: 'absent' })).toThrow('not evaluated') + expect(() => pageReferenceTraces(report, dataset, { ...request, offset: 2 })).toThrow('cursor') +}) +it('refuses reports without a readable identity or scoring windows', () => { + const { report, dataset } = fixture() + expect(() => pageReferenceTraces({ ...report, datasetId: 'not-a-uuid' }, dataset, request)).toThrow('identity is unavailable') + expect(() => pageReferenceTraces({ ...report, nights: [{ ...report.nights[0], evaluations: [] }] }, dataset, request)).toThrow('windows unavailable') +}) diff --git a/src/biometrics/backtest/tests/report-summary.test.ts b/src/biometrics/backtest/tests/report-summary.test.ts new file mode 100644 index 00000000..8b19c486 --- /dev/null +++ b/src/biometrics/backtest/tests/report-summary.test.ts @@ -0,0 +1,34 @@ +// @vitest-environment node +import { expect, it } from 'vitest' +import { evaluateBacktest } from '../evaluator' +import { dataHash } from '../../algorithms/canonical' +import { backtestFixture } from './fixtures/backtest' +import { summarizeBacktestResult } from '../report-summary' +import { pageStageTraces } from '../trace-page' + +it('preserves all scores and the retained artifact hash while omitting trace rows', () => { + const report = evaluateBacktest(backtestFixture()) + const contentHash = dataHash(report) + const descriptor = { report, available: true, contentHash } + const summary = summarizeBacktestResult(descriptor) + if (!summary) throw new Error('Expected a summary') + expect(summary).toMatchObject({ contentHash, contentHashScope: 'complete_retained_report', projection: 'scores_without_stage_trace_rows_v1' }) + const expected = structuredClone(report) as unknown as { nights: Array<{ evaluations: Array<{ windows: Array<{ blocks?: unknown }> }> }> } + for (const night of expected.nights) { + for (const evaluation of night.evaluations) { + for (const window of evaluation.windows) delete window.blocks + } + } + expect(summary.report).toEqual(expected) + expect(dataHash(report)).toBe(contentHash) + expect(pageStageTraces(report, { id: '11111111-1111-4111-8111-111111111111', expectedContentHash: contentHash, configuration: 'default' }).totalItems).toBeGreaterThan(0) + expect(JSON.stringify(summary.report).length).toBeLessThan(JSON.stringify(report).length) +}) +it('distinguishes missing and released reports without manufacturing content', () => { + expect(summarizeBacktestResult(null)).toBeNull() + expect(summarizeBacktestResult({ available: false, report: null, contentHash: 'retained' })).toMatchObject({ report: null, available: false, contentHash: 'retained' }) +}) +it('rejects unknown versions and oversize scores instead of silently truncating them', () => { + expect(() => summarizeBacktestResult({ available: true, report: { version: 'future' } })).toThrow('summary format') + expect(() => summarizeBacktestResult({ available: true, report: { version: 'processed-backtest-report-v1', nights: [], extra: 'x'.repeat(2 * 1024 * 1024) } })).toThrow('transport limit') +}) diff --git a/src/biometrics/backtest/tests/runner-waveform.test.ts b/src/biometrics/backtest/tests/runner-waveform.test.ts new file mode 100644 index 00000000..e0ee6b17 --- /dev/null +++ b/src/biometrics/backtest/tests/runner-waveform.test.ts @@ -0,0 +1,120 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { execFile } from 'node:child_process' +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { promisify } from 'node:util' +import { afterAll, afterEach, beforeAll, beforeEach, expect, it, vi } from 'vitest' +import * as schema from '@/src/db/biometrics-schema' +import { configHash, dataHash } from '../../algorithms/canonical' +import { defaultConfig, manifestHash } from '../../algorithms/registry' +import { workerBuildHash } from '../child' +import { datasetManifestSchema } from '../contracts' +import { DatasetStore } from '../datasets' +import { BacktestJobStore } from '../jobs' +import { executeNextBacktest } from '../runner' +import { prepareWaveformDataset } from '../waveform-dataset' +import { executeWaveformInputs, verifyWaveformJobBuild } from '../waveform-execution' +import { stageObjectiveTestPolicy } from './fixtures/stage-objective' + +// The Python replay child is exercised by waveform-execution.test.ts; here the +// runner's signal-replay orchestration is driven through the real scoring worker. +vi.mock('../waveform-execution', () => ({ executeWaveformInputs: vi.fn(), verifyWaveformJobBuild: vi.fn() })) + +type Work = Parameters[1] +const python = 'c'.repeat(64), digest = 'a'.repeat(64), start = 1791673200000 +const resolvedHash = configHash(defaultConfig) +const protocol = { version: 'descriptive-evaluation-v1' as const, mode: 'signal_replay' as const, assumedStageCalibrationQuality: 0, pointToleranceMs: 30000, clockOffsetMs: 0 } +let sqlite: Database.Database +let db: ReturnType> +let jobs: BacktestJobStore +let directory: string +let evaluatorBuildHash: string +let datasetId: string +beforeAll(async () => { + directory = await mkdtemp(join(tmpdir(), 'biometric-runner-waveform-')) + await promisify(execFile)(process.execPath, ['scripts/build/biometrics-worker.mjs', directory]) + evaluatorBuildHash = await workerBuildHash(join(directory, 'worker.cjs')) +}) +afterAll(async () => rm(directory, { recursive: true, force: true })) +beforeEach(() => { + sqlite = new Database(':memory:') + db = drizzle(sqlite, { schema }) + migrate(db, { migrationsFolder: 'src/db/biometrics-migrations' }) + db.insert(schema.sleepRecords).values({ id: 1, side: 'left', enteredBedAt: new Date(start), leftBedAt: new Date(start + 600000), sleepDurationSeconds: 600, timesExitedBed: 0 }).run() + for (let minute = 0; minute < 10; minute++) db.insert(schema.vitals).values({ side: 'left', timestamp: new Date(start + minute * 60000), heartRate: 60, hrv: 30, breathingRate: 14 }).run() + const datasets = new DatasetStore(db) + const source = datasets.read(datasets.create({ side: 'left', subjectId: 'A', sessionIds: [1], referenceImportIds: [], timezone: 'UTC', idempotencyKey: 'source' }).id) + const cohort = datasetManifestSchema.parse(source.manifest).splitProtocol.assignments[0] + const prepared = prepareWaveformDataset({ ...source, manifest: datasetManifestSchema.parse(source.manifest) }, [{ cohortId: cohort.cohortId, + input: { version: 'retained-waveform-input-v1', retained_start_ms: cohort.startMs, score_start_ms: cohort.startMs, score_end_ms: cohort.endMs, contexts: [], + artifacts: [{ artifact: { sha256: digest, name: `${digest}.RAW.gz`, compressed_bytes: 1, expanded_bytes: 2 }, first_event_ms: cohort.startMs, last_event_ms: cohort.endMs - 1 }] } }], 'raw') + db.insert(schema.biometricDatasets).values({ ...prepared, byteLength: Buffer.byteLength(JSON.stringify(prepared.manifest)) + Buffer.byteLength(JSON.stringify(prepared.snapshot)) }).run() + datasetId = prepared.id + jobs = new BacktestJobStore(db) + vi.mocked(verifyWaveformJobBuild).mockResolvedValue(undefined) + vi.mocked(executeWaveformInputs).mockImplementation(async (_db, work: Work) => { + if (work.dataset.snapshot.version !== 'waveform-dataset-v1' || !work.job.waveform) throw new Error('Processed work reached the waveform executor') + const assignments = datasetManifestSchema.parse(work.dataset.manifest).splitProtocol.assignments.filter(row => row.split === work.job.split) + const configs = [...new Map(work.job.configs.map(config => [config.resolvedHash, config])).values()] + const outputs = assignments.flatMap(cohort => configs.map((config) => { + const input = work.dataset.snapshot.version === 'waveform-dataset-v1' ? work.dataset.snapshot.waveformInputs.find(row => row.cohortId === cohort.cohortId)?.input : undefined + if (!input) throw new Error('Missing retained cohort') + const readings = Array.from({ length: 10 }, (_, minute) => ({ side: 'left' as const, timestampMs: cohort.startMs + minute * 60000, heartRate: 60, hrvIndex: 30, + hrvMetric: 'window_ibi_variability_v1' as const, breathingRate: 14, qualityScore: 1, flags: [], hrRaw: 60, sourceSide: 'left' as const, configHash: config.resolvedHash, limitations: [] })) + return { cohortId: cohort.cohortId, configHash: config.resolvedHash, readings, run: { version: 'retained-waveform-run-v1' as const, build_hash: python, + decoded_records: 10, raw_records: 10, counts: { records: 10, accepted_frames: 10, empty_channel_frames: 0, pump_gated_frames: 0, preroll_outputs: 0, scored_outputs: readings.length }, + limitations: ['missing_pump_context'], config_hashes: { left: config.resolvedHash, right: work.job.waveform?.otherSide.resolvedHash ?? resolvedHash }, artifact_hashes: input.artifacts.map(row => row.artifact.sha256) } } + })) + return { hash: dataHash(outputs), outputs } + }) +}) +afterEach(async () => { + vi.clearAllMocks() + sqlite.close() + await rm(join(directory, 'promotion-qualifications.json'), { force: true }) +}) +const options = { owner: 'waveform-test', get workerPath() { + return join(directory, 'worker.cjs') +}, admit: async () => true } +const config = (label: 'default' | 'current' | 'candidate', revisionId: string | null = null) => ({ label, revisionId, resolved: defaultConfig, resolvedHash, manifestHash }) +const enqueue = (experimentProtocol: typeof protocol & { waveformStages?: boolean, stageObjectivePolicy?: typeof stageObjectiveTestPolicy }, configs = [config('default'), config('current')]) => { + const experiment = jobs.createExperiment(datasetId, experimentProtocol) + return jobs.enqueue({ evaluatorBuildHash, experimentId: experiment.id, idempotencyKey: 'waveform', split: 'tuning', configs, + waveform: { pythonBuildHash: python, otherSide: { manifestHash, resolvedHash, resolved: defaultConfig } } }).id +} + +it('replays retained waveforms through Python, re-verifies the pinned build and publishes a descriptive waveform report', async () => { + const jobId = enqueue(protocol) + expect(await executeNextBacktest(db, options)).toEqual({ state: 'succeeded', jobId }) + expect(executeWaveformInputs).toHaveBeenCalledTimes(1) + expect(executeWaveformInputs).toHaveBeenCalledWith(db, expect.objectContaining({ job: expect.objectContaining({ datasetId }) }), expect.objectContaining({ owner: 'waveform-test', signal: expect.any(AbortSignal) })) + expect(verifyWaveformJobBuild).toHaveBeenCalledTimes(1) + expect(jobs.result(jobId)?.report).toMatchObject({ version: 'waveform-backtest-report-v1', status: 'insufficient_evidence', qualificationWarning: null, + waveform: { pythonBuildHash: python, otherSideConfigHash: resolvedHash }, nights: [{ uniqueObservationCount: 10 }] }) +}) + +it('fails the job instead of publishing when the waveform build drifts after scoring', async () => { + const jobId = enqueue(protocol) + vi.mocked(verifyWaveformJobBuild).mockRejectedValueOnce(new Error('Waveform worker build changed since enqueue')) + expect(await executeNextBacktest(db, options)).toEqual({ state: 'stopped', jobId }) + expect(jobs.get(jobId)).toMatchObject({ state: 'failed' }) + expect(jobs.result(jobId)).toBeNull() +}) + +it.each([ + [['missing_pump_context'], null], + [[], 'Replay context is outside the installed qualification; this report is descriptive.'], +] as const)('keeps an installed waveform qualification only when every replay limitation %j was qualified', async (allowedLimitations, qualificationWarning) => { + const qualified = { ...protocol, waveformStages: true, stageObjectivePolicy: stageObjectiveTestPolicy } + const jobId = enqueue(qualified, [config('default'), config('current'), config('candidate', '33333333-3333-4333-8333-333333333333')]) + await writeFile(join(directory, 'promotion-qualifications.json'), JSON.stringify({ version: 'biometric-promotion-qualifications-v1', entries: [{ + qualificationId: stageObjectiveTestPolicy.qualificationId, manifestHash, evaluatorBuildHash, pythonBuildHash: python, protocol: qualified, + fixtureEvidenceHash: 'a'.repeat(64), podEvidenceHash: 'b'.repeat(64), waveformContext: { allowedLimitations: [...allowedLimitations] }, + }] })) + expect(await executeNextBacktest(db, options)).toEqual({ state: 'succeeded', jobId }) + expect(jobs.result(jobId)?.report).toMatchObject({ version: 'waveform-backtest-report-v1', status: 'no_material_change', qualificationWarning }) +}) diff --git a/src/biometrics/backtest/tests/runner.test.ts b/src/biometrics/backtest/tests/runner.test.ts new file mode 100644 index 00000000..f8eb5894 --- /dev/null +++ b/src/biometrics/backtest/tests/runner.test.ts @@ -0,0 +1,241 @@ +import { ReferenceStore } from '../../references/store' +import { previewReferenceImport } from '../../references/normalize' +import { configHash } from '../../algorithms/canonical' +import { defaultConfig, manifestHash } from '../../algorithms/registry' +import { stageObjectiveTestPolicy } from './fixtures/stage-objective' +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { execFile } from 'node:child_process' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { promisify } from 'node:util' +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest' +import * as schema from '@/src/db/biometrics-schema' +import { DatasetStore } from '../datasets' +import { BacktestJobStore } from '../jobs' +import { workerBuildHash } from '../child' +import { executeNextBacktest } from '../runner' +import { backtestFixture } from './fixtures/backtest' +let sqlite: Database.Database +let db: ReturnType> +let jobs: BacktestJobStore +let jobId: string +let directory: string +let evaluatorBuildHash: string +beforeAll(async () => { + directory = await mkdtemp(join(tmpdir(), 'biometric-runner-test-')) + await promisify(execFile)(process.execPath, ['scripts/build/biometrics-worker.mjs', directory]) + evaluatorBuildHash = await workerBuildHash(join(directory, 'worker.cjs')) +}) +afterAll(async () => rm(directory, { recursive: true, force: true })) +beforeEach(() => { + sqlite = new Database(':memory:') + db = drizzle(sqlite, { schema }) + migrate(db, { migrationsFolder: 'src/db/biometrics-migrations' }) + const fixture = backtestFixture() + const session = fixture.dataset.snapshot.sessions[0] + db.insert(schema.sleepRecords).values({ id: 1, side: 'left', enteredBedAt: new Date(session.startMs), leftBedAt: new Date(session.endMs), sleepDurationSeconds: 600, timesExitedBed: 0 }).run() + for (const row of session.vitals) db.insert(schema.vitals).values({ side: 'left', timestamp: new Date(row.timestampMs), heartRate: row.heartRate, hrv: row.hrv, breathingRate: row.breathingRate }).run() + const dataset = new DatasetStore(db).create({ side: 'left', subjectId: 'A', sessionIds: [1], referenceImportIds: [], timezone: 'UTC', idempotencyKey: 'run' }) + jobs = new BacktestJobStore(db) + const experiment = jobs.createExperiment(dataset.id, fixture.job.protocol) + jobId = jobs.enqueue({ evaluatorBuildHash, experimentId: experiment.id, configs: fixture.job.configs, + idempotencyKey: 'run', split: 'tuning' }).id +}) +afterEach(() => sqlite.close()) +const options = { owner: 'integration-test', get workerPath() { + return join(directory, 'worker.cjs') +}, admit: async () => true } + +describe('durable replay execution', () => { + it('runs retained data through a child and publishes an integrity-checked result without changing canonical observations', async () => { + const before = sqlite.prepare('SELECT * FROM vitals').all() + expect(await executeNextBacktest(db, options)).toEqual({ state: 'succeeded', jobId }) + expect(jobs.result(jobId)?.report).toMatchObject({ version: 'processed-backtest-report-v1', status: 'insufficient_evidence', nights: [{ uniqueObservationCount: 10 }] }) + expect(jobs.result(jobId)?.report).toMatchObject({ jobOperations: { ingestion: { allChecksMeasured: false, samples: 0, maximumObservedAgeMs: null } } }) + expect(sqlite.prepare('SELECT * FROM vitals').all()).toEqual(before) + expect(await executeNextBacktest(db, options)).toEqual({ state: 'idle' }) + }) + it('retains supervisor observations in the committed report and marks legacy boolean admission unmeasured', async () => { + let checks = 0 + expect(await executeNextBacktest(db, { ...options, admit: async () => { + checks++ + return true + }, + admissionReceipt: () => ({ allowed: true, sample: { qualificationId: 'test-policy', checkedAtMs: Date.now(), ingestionAgeMs: checks * 100 } }), + })).toEqual({ state: 'succeeded', jobId }) + expect(jobs.result(jobId)?.report).toMatchObject({ jobOperations: { version: 'backtest-job-operations-v1', + ingestion: { allChecksMeasured: true, checks: 2, samples: 2, maximumObservedAgeMs: 200, qualificationId: 'test-policy' } } }) + }) + it('finalizes retained stage objectives after the real scoring child completes', async () => { + const datasetId = jobs.getExperiment(jobs.get(jobId).experimentId).datasetId + jobs.cancel(jobId) + const fixture = backtestFixture() + const experiment = jobs.createExperiment(datasetId, { ...fixture.job.protocol, stageObjectivePolicy: stageObjectiveTestPolicy }) + const staged = jobs.enqueue({ evaluatorBuildHash, experimentId: experiment.id, idempotencyKey: 'stage-objective', split: 'tuning', + configs: [...fixture.job.configs, { ...fixture.job.configs[0], label: 'candidate' }] }) + expect(await executeNextBacktest(db, options)).toEqual({ state: 'succeeded', jobId: staged.id }) + expect(jobs.result(staged.id)?.report).toMatchObject({ status: 'no_material_change', stageObjectiveAssessments: [ + { baseline: 'default', status: 'insufficient_evidence', evidence: { operational: null } }, + { baseline: 'current', status: 'insufficient_evidence', evidence: { operational: null } }, + ] }) + }) + it.each([['wake', 'improved'], ['deep', 'regressed']] as const)('publishes %s candidate evidence as %s through the real scoring worker', async (stage, status) => { + jobs.cancel(jobId) + const first = 1791673200000 + const ids = Array.from({ length: 12 }, (_, index) => index + 10) + const rows = ids.flatMap((id, index) => { + const start = first + id * 86400000, end = start + 600000 + db.insert(schema.sleepRecords).values({ id, side: 'left', enteredBedAt: new Date(start), leftBedAt: new Date(end), sleepDurationSeconds: 600, timesExitedBed: 0 }).run() + for (let minute = 0; minute < 10; minute++) db.insert(schema.vitals).values({ side: 'left', timestamp: new Date(start + minute * 60000), heartRate: 60, hrv: 30, breathingRate: 14 }).run() + const middle = start + 360000 + index % 3 * 60000 + return [{ recordKey: `${id}-wake`, metric: 'sleep_stages', value: 'wake', unit: 'stage', startMs: start, endMs: middle }, + { recordKey: `${id}-light`, metric: 'sleep_stages', value: 'light', unit: 'stage', startMs: middle, endMs: end }] + }) + const input = { side: 'left' as const, source: 'synthetic-runner-test', subjectId: 'fixture', rows } + const reference = new ReferenceStore(db).commit({ import: input, previewHash: previewReferenceImport(input).summary.contentHash }) + const dataset = new DatasetStore(db).create({ side: 'left', subjectId: 'fixture', sessionIds: ids, referenceImportIds: [reference.id], timezone: 'UTC', idempotencyKey: 'qualified-test' }) + const policy = { ...stageObjectiveTestPolicy, requiredStages: ['wake' as const], maximumMacroF1Regression: 0.5, + operational: { maximumWallMs: 60000, maximumPeakRssBytes: 1000000000, maximumIngestionAgeMs: 1000, + observations: { version: 'sampled-process-group-v1' as const, maximumMemoryGapMs: 60000, maximumIngestionGapMs: 60000 } } } + const protocol = { ...backtestFixture().job.protocol, assumedStageCalibrationQuality: 1, stageObjectivePolicy: policy, + nightUncertainty: { version: 'paired-night-percentile-v1' as const, seed: 'runner-qualified', resamples: 200, minimumNights: 3 } } + const experiment = jobs.createExperiment(dataset.id, protocol) + const candidate = structuredClone(defaultConfig) + candidate.sleep_stages.ruleset.rules = [{ stage, when: [] }] + const configurations = [...backtestFixture().job.configs, { label: 'candidate' as const, revisionId: '33333333-3333-4333-8333-333333333333', + resolved: candidate, resolvedHash: configHash(candidate), manifestHash }] + const request = { evaluatorBuildHash, experimentId: experiment.id, configs: configurations, split: 'holdout' as const } + const staged = jobs.enqueue({ ...request, idempotencyKey: 'qualified' }) + const registry = join(directory, 'promotion-qualifications.json') + await writeFile(registry, JSON.stringify({ version: 'biometric-promotion-qualifications-v1', entries: [{ + qualificationId: policy.qualificationId, manifestHash, evaluatorBuildHash, protocol, + fixtureEvidenceHash: 'a'.repeat(64), podEvidenceHash: 'b'.repeat(64), waveformContext: null, + }] })) + const before = sqlite.prepare('SELECT * FROM vitals').all() + const measuredOptions = { ...options, admissionReceipt: () => ({ allowed: true, + sample: { qualificationId: policy.qualificationId, checkedAtMs: Date.now(), ingestionAgeMs: 10 }, + memory: { version: 'process-group-rss-observations-v1' as const, scope: 'sum_rss_of_replay_process_group' as const, + attempts: 2, samples: 2, allSamplesAvailable: true, peakObservedRssBytes: 1000, maximumObservationGapMs: 1, intervalMs: 1000 as const, exactKernelPeak: false as const } }) } + try { + expect(await executeNextBacktest(db, measuredOptions)).toEqual({ state: 'succeeded', jobId: staged.id }) + expect(jobs.result(staged.id)?.report).toMatchObject({ status, + stageObjectiveAssessments: [{ status, designated: true }, { status, designated: true }] }) + if (status === 'improved') expect(jobs.result(staged.id)?.report).toMatchObject({ evidence: 'qualified_reference_agreement' }) + expect(sqlite.prepare('SELECT * FROM vitals').all()).toEqual(before) + const reused = jobs.enqueue({ ...request, idempotencyKey: 'reused' }) + expect(await executeNextBacktest(db, measuredOptions)).toEqual({ state: 'succeeded', jobId: reused.id }) + expect(jobs.result(reused.id)?.report).toMatchObject({ status: 'insufficient_evidence' }) + } + finally { await rm(registry, { force: true }) } + }) + + it('refuses to execute a different worker bundle after enqueue', async () => { + sqlite.prepare('UPDATE biometric_backtest_jobs SET input = json_set(input, \'$.evaluatorBuildHash\', ?)').run('0'.repeat(64)) + expect(await executeNextBacktest(db, options)).toEqual({ state: 'stopped', jobId }) + expect(jobs.get(jobId).state).toBe('failed') + expect(jobs.result(jobId)).toBeNull() + }) + + it('defers work before claim when resource admission is denied or shutdown arrives', async () => { + expect(await executeNextBacktest(db, { ...options, admit: async () => false })).toEqual({ state: 'deferred' }) + expect(jobs.get(jobId).attempt).toBe(0) + const controller = new AbortController() + expect(await executeNextBacktest(db, { ...options, signal: controller.signal, admit: async () => { + controller.abort() + return true + } })).toEqual({ state: 'deferred' }) + expect(jobs.get(jobId).attempt).toBe(0) + }) + it('interrupts a running child when health declines and never publishes its result', async () => { + const stalledPath = join(directory, 'stalled.cjs') + await writeFile(stalledPath, 'process.stdin.resume(); setInterval(() => {}, 1000)') + const hash = await workerBuildHash(stalledPath) + sqlite.prepare('UPDATE biometric_backtest_jobs SET input = json_set(input, \'$.evaluatorBuildHash\', ?)').run(hash) + let checks = 0 + const result = await executeNextBacktest(db, { ...options, workerPath: stalledPath, admit: async () => ++checks === 1 }) + expect(result).toEqual({ state: 'stopped', jobId }) + expect(checks).toBe(2) + expect(jobs.get(jobId)).toMatchObject({ state: 'interrupted', error: 'Resource admission withdrawn' }) + expect(jobs.result(jobId)).toBeNull() + }, 10000) + + it('withholds publication when the final resource check fails', async () => { + let checks = 0 + expect(await executeNextBacktest(db, { ...options, admit: async () => { + if (++checks === 1) return true + throw new Error('Health collector unavailable') + } })).toEqual({ state: 'stopped', jobId }) + expect(checks).toBe(2) + expect(jobs.get(jobId)).toMatchObject({ state: 'interrupted', error: 'Resource admission withdrawn' }) + expect(jobs.result(jobId)).toBeNull() + }) +}) + +describe('lease, cancellation and worker protocol failures', () => { + /** A scripted worker standing in for the scoring bundle; the job is re-pinned to its hash. */ + async function scriptedWorker(name: string, body: string) { + const path = join(directory, name) + await writeFile(path, `let input='';process.stdin.on('data',c=>input+=c);process.stdin.on('end',()=>{const work=JSON.parse(input);const emit=e=>process.stdout.write(JSON.stringify(e)+'\\n');${body}});`) + sqlite.prepare('UPDATE biometric_backtest_jobs SET input = json_set(input, \'$.evaluatorBuildHash\', ?)').run(await workerBuildHash(path)) + return path + } + const progress = (completedNights: number, totalNights = 1) => `emit({type:'progress',completedNights:${completedNights},totalNights:${totalNights}});` + const identity = 'const report={version:"processed-backtest-report-v1",datasetHash:work.job.datasetHash,protocolHash:work.job.protocolHash,evaluatorBuildHash:work.job.evaluatorBuildHash,split:work.job.split,nights:[{}]};' + + it('interrupts a running replay when the host shuts down and leaves it for the next claimant', async () => { + const workerPath = await scriptedWorker('shutdown.cjs', 'setInterval(()=>{},1000)') + const controller = new AbortController() + setTimeout(() => controller.abort(), 300) + expect(await executeNextBacktest(db, { ...options, workerPath, signal: controller.signal })).toEqual({ state: 'stopped', jobId }) + expect(jobs.get(jobId)).toMatchObject({ state: 'interrupted', error: 'Worker host stopped' }) + expect(jobs.result(jobId)).toBeNull() + }) + + it('acknowledges a cancellation observed through a progress heartbeat', async () => { + const workerPath = await scriptedWorker('cancel.cjs', `setInterval(()=>{${progress(0)}},50)`) + setTimeout(() => jobs.cancel(jobId), 300) + expect(await executeNextBacktest(db, { ...options, workerPath })).toEqual({ state: 'stopped', jobId }) + expect(jobs.get(jobId)).toMatchObject({ state: 'cancelled', leaseOwner: null }) + expect(jobs.result(jobId)).toBeNull() + }) + + it('stops without touching job state once its lease is no longer authoritative', async () => { + const workerPath = await scriptedWorker('fenced.cjs', `setInterval(()=>{${progress(0)}},50)`) + setTimeout(() => sqlite.prepare('UPDATE biometric_backtest_jobs SET fencing_token = fencing_token + 1').run(), 300) + expect(await executeNextBacktest(db, { ...options, workerPath })).toEqual({ state: 'stopped', jobId }) + expect(jobs.get(jobId)).toMatchObject({ state: 'running', error: null }) + }) + + it.each([ + ['a different night count', progress(0, 5)], + ['regressing completion', progress(1) + progress(0)], + ['completion beyond the night count', progress(2)], + ])('fails a worker reporting %s', async (_, body) => { + const workerPath = await scriptedWorker('progress.cjs', `${body}setInterval(()=>{},1000)`) + expect(await executeNextBacktest(db, { ...options, workerPath })).toEqual({ state: 'stopped', jobId }) + expect(jobs.get(jobId)).toMatchObject({ state: 'failed', error: 'Replay failed validation, resources, or execution' }) + }) + + it.each([ + ['evaluator build', 'report.evaluatorBuildHash="0".repeat(64);'], + ['dataset', 'report.datasetHash="0".repeat(64);'], + ['protocol', 'report.protocolHash="0".repeat(64);'], + ['split', 'report.split="holdout";'], + ['night count', 'report.nights=[];'], + ])('rejects a report whose %s identity differs from the leased job', async (_, mutate) => { + const workerPath = await scriptedWorker('identity.cjs', `${identity}${mutate}${progress(1)}emit({type:'result',report});`) + expect(await executeNextBacktest(db, { ...options, workerPath })).toEqual({ state: 'stopped', jobId }) + expect(jobs.get(jobId)).toMatchObject({ state: 'failed' }) + expect(jobs.result(jobId)).toBeNull() + }) + + it('rejects a report published before every night was reported complete', async () => { + const workerPath = await scriptedWorker('incomplete.cjs', `${identity}emit({type:'result',report});`) + expect(await executeNextBacktest(db, { ...options, workerPath })).toEqual({ state: 'stopped', jobId }) + expect(jobs.get(jobId)).toMatchObject({ state: 'failed' }) + }) +}) diff --git a/src/biometrics/backtest/tests/submission-build.test.ts b/src/biometrics/backtest/tests/submission-build.test.ts new file mode 100644 index 00000000..1c9a6970 --- /dev/null +++ b/src/biometrics/backtest/tests/submission-build.test.ts @@ -0,0 +1,17 @@ +// @vitest-environment node +import { beforeEach, expect, it, vi } from 'vitest' +import { packagedWorkerPath } from '../location' +import { waveformEvaluatorBuildIdentity } from '../waveform-build' +import { resolveInstalledReplayBuild } from '../submission' + +vi.mock('../location', () => ({ packagedWorkerPath: vi.fn(), packagedCoordinatorPath: vi.fn() })) +vi.mock('../waveform-build', () => ({ waveformEvaluatorBuildIdentity: vi.fn() })) +beforeEach(() => vi.resetAllMocks()) + +it('pins the coordinator bundle that ships beside the worker, matching execution', async () => { + vi.mocked(packagedWorkerPath).mockReturnValue('/standalone/biometrics-worker/worker.cjs') + const build = { hash: 'a'.repeat(64), python: { hash: 'b'.repeat(64) } } as Awaited> + vi.mocked(waveformEvaluatorBuildIdentity).mockResolvedValue(build) + await expect(resolveInstalledReplayBuild('signal_replay')).resolves.toEqual({ evaluatorBuildHash: build.hash, pythonBuildHash: build.python.hash }) + expect(vi.mocked(waveformEvaluatorBuildIdentity).mock.calls[0]?.slice(0, 2)).toEqual(['/standalone/biometrics-worker/coordinator.cjs', '/standalone/biometrics-worker/worker.cjs']) +}) diff --git a/src/biometrics/backtest/tests/traces.test.ts b/src/biometrics/backtest/tests/traces.test.ts new file mode 100644 index 00000000..2225a445 --- /dev/null +++ b/src/biometrics/backtest/tests/traces.test.ts @@ -0,0 +1,108 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import * as schema from '@/src/db/biometrics-schema' +import { configHash } from '../../algorithms/canonical' +import { defaultConfig, manifestHash } from '../../algorithms/registry' +import { datasetManifestSchema } from '../contracts' +import { DatasetStore } from '../datasets' +import { BacktestJobStore } from '../jobs' +import { pageStageTraces, readNumericTraces, readReferenceTraces, readStageTraces } from '../traces' +const request = { id: '11111111-1111-4111-8111-111111111111', expectedContentHash: 'a'.repeat(64), configuration: 'candidate' as const, limit: 2 } +const report = { version: 'processed-backtest-report-v1', protocol: { mode: 'stage_replay', clockOffsetMs: 60000 }, nights: ['one', 'two'].map((cohortId, index) => ({ + cohortId, evaluations: [{ label: 'candidate', configHash: 'b'.repeat(64), revisionId: null, windows: [{ startMs: index * 300, endMs: index * 300 + 300, + blocks: [0, 1, 2].map(i => ({ start: index * 300 + i * 100, end: index * 300 + (i + 1) * 100, stage: 'light' })) }] }], +})) } +it('pages every retained interval exactly once with a stable hash cursor and no timestamp offset', () => { + const before = JSON.stringify(report) + const first = pageStageTraces(report, request) + const second = pageStageTraces(report, { ...request, offset: first.nextCursor?.offset }) + const third = pageStageTraces(report, { ...request, offset: second.nextCursor?.offset }) + expect(first).toMatchObject({ totalItems: 6, downsampled: false, clockOffsetMs: 60000, nextCursor: { offset: 2, contentHash: request.expectedContentHash } }) + expect([...first.items, ...second.items, ...third.items].map(row => row.startMs)).toEqual([0, 100, 200, 300, 400, 500]) + expect(third.nextCursor).toBeNull() + expect(pageStageTraces(report, { ...request, offset: 6 }).items).toEqual([]) + expect(JSON.stringify(report)).toBe(before) +}) +it('filters the cohort before counting and rejects out-of-range or missing selections', () => { + expect(pageStageTraces(report, { ...request, cohortId: 'two' })).toMatchObject({ totalItems: 3, items: [{ cohortId: 'two' }, { cohortId: 'two' }] }) + expect(() => pageStageTraces(report, { ...request, cohortId: 'missing' })).toThrow('not evaluated') + expect(() => pageStageTraces(report, { ...request, configuration: 'default' })).toThrow('not evaluated') + expect(() => pageStageTraces(report, { ...request, offset: 7 })).toThrow('cursor') + expect(() => pageStageTraces(report, { ...request, limit: 501 })).toThrow() + expect(() => pageStageTraces({ ...report, protocol: { ...report.protocol, mode: 'comparison' } }, request)).toThrow('did not produce') +}) +it('serves waveform stage traces only when the protocol replayed stages and rejects unreadable reports', () => { + expect(pageStageTraces({ ...report, version: 'waveform-backtest-report-v1', protocol: { ...report.protocol, mode: 'signal_replay', waveformStages: true } }, request)).toMatchObject({ totalItems: 6 }) + expect(() => pageStageTraces({ ...report, protocol: { ...report.protocol, mode: 'signal_replay', waveformStages: false } }, request)).toThrow('did not produce') + expect(() => pageStageTraces({ ...report, protocol: { ...report.protocol, mode: 'signal_replay' } }, request)).toThrow('did not produce') + expect(() => pageStageTraces({ ...report, nights: null }, request)).toThrow('format is unavailable') +}) + +const start = 1791673200000 +const numericReport = () => { + const hash = 'b'.repeat(64) + const reading = { side: 'left', sourceSide: 'left', configHash: hash, timestampMs: 1000, heartRate: 60, breathingRate: null, hrvIndex: null, + hrvMetric: 'window_ibi_variability_v1', qualityScore: 0, flags: null, limitations: [], hrRaw: 65 } + return { version: 'waveform-backtest-report-v1', side: 'left', protocol: { mode: 'signal_replay', clockOffsetMs: 0 }, + nights: [{ cohortId: 'one', evaluations: [{ label: 'candidate', configHash: hash, revisionId: null, windows: [{ startMs: 1000, endMs: 5000 }] }] }], + waveform: { outputs: [{ cohortId: 'one', configHash: hash, readings: [reading] }] } } +} +describe('durable trace reads', () => { + let sqlite: Database.Database + let db: ReturnType> + let jobs: BacktestJobStore + let dataset: ReturnType + beforeEach(() => { + sqlite = new Database(':memory:') + db = drizzle(sqlite, { schema }) + migrate(db, { migrationsFolder: 'src/db/biometrics-migrations' }) + db.insert(schema.sleepRecords).values({ id: 1, side: 'left', enteredBedAt: new Date(start), leftBedAt: new Date(start + 600000), sleepDurationSeconds: 600, timesExitedBed: 0 }).run() + const datasets = new DatasetStore(db) + dataset = datasets.read(datasets.create({ side: 'left', subjectId: 'A', sessionIds: [1], referenceImportIds: [], timezone: 'UTC', idempotencyKey: 'traces' }).id) + jobs = new BacktestJobStore(db) + }) + afterEach(() => sqlite.close()) + const config = { label: 'default' as const, revisionId: null, resolved: defaultConfig, resolvedHash: configHash(defaultConfig), manifestHash } + const enqueue = (idempotencyKey: string, experimentId = jobs.createExperiment(dataset.id, { version: 'descriptive-evaluation-v1', mode: 'stage_replay', assumedStageCalibrationQuality: 0, pointToleranceMs: 30000, clockOffsetMs: 0 }, idempotencyKey).id) => + jobs.enqueue({ experimentId, evaluatorBuildHash: 'f'.repeat(64), idempotencyKey, split: 'tuning', configs: [config] }) + const publish = (idempotencyKey: string, mode: 'stage_replay' | 'signal_replay', published: object) => { + const job = enqueue(idempotencyKey) + const claimed = jobs.claim('reader') + if (!claimed) throw new Error('Missing claim') + const result = jobs.complete({ id: job.id, owner: 'reader', token: claimed.fencingToken }, { ...published, protocol: { mode, clockOffsetMs: 0, pointToleranceMs: 30000 } }) + return { id: job.id, expectedContentHash: result.contentHash } + } + it('reads retained stage traces only from a verified, unreleased successful result', () => { + const published = publish('stage', 'stage_replay', report) + expect(readStageTraces(db, { ...published, configuration: 'candidate', limit: 2 })).toMatchObject({ totalItems: 6, clockOffsetMs: 0, items: [{ cohortId: 'one' }, { cohortId: 'one' }] }) + expect(() => readStageTraces(db, { ...published, expectedContentHash: '0'.repeat(64), configuration: 'candidate' })).toThrow('hash changed') + const queued = enqueue('queued', jobs.get(published.id).experimentId) + expect(() => readStageTraces(db, { ...published, id: queued.id, configuration: 'candidate' })).toThrow('successful replay') + jobs.releaseResult(published.id, published.expectedContentHash) + expect(() => readStageTraces(db, { ...published, configuration: 'candidate' })).toThrow('released') + }) + it('reads retained numeric traces with the same lifecycle checks', () => { + const published = publish('numeric', 'signal_replay', numericReport()) + expect(readNumericTraces(db, { ...published, configuration: 'candidate' })).toMatchObject({ totalItems: 1, side: 'left', items: [{ timestampMs: 1000 }] }) + expect(() => readNumericTraces(db, { ...published, expectedContentHash: '0'.repeat(64), configuration: 'candidate' })).toThrow('hash changed') + const queued = enqueue('queued', jobs.get(published.id).experimentId) + expect(() => readNumericTraces(db, { ...published, id: queued.id, configuration: 'candidate' })).toThrow('successful replay') + jobs.releaseResult(published.id, published.expectedContentHash) + expect(() => readNumericTraces(db, { ...published, configuration: 'candidate' })).toThrow('released') + }) + it('reads frozen reference traces for the retained dataset the report identifies', () => { + const cohortId = datasetManifestSchema.parse(dataset.manifest).splitProtocol.assignments[0].cohortId + const identity = { version: 'processed-backtest-report-v1', datasetId: dataset.id, datasetHash: dataset.contentHash, + nights: [{ cohortId, evaluations: [{ label: 'default', configHash: 'b'.repeat(64), revisionId: null, windows: [{ startMs: start, endMs: start + 600000, blocks: [] }] }] }] } + const published = publish('reference', 'stage_replay', identity) + expect(readReferenceTraces(db, { ...published, cohortId, metric: 'heart_rate' })).toMatchObject({ totalItems: 0, side: 'left', datasetHash: dataset.contentHash, provenance: 'frozen_dataset_reference' }) + expect(() => readReferenceTraces(db, { ...published, expectedContentHash: '0'.repeat(64), cohortId, metric: 'heart_rate' })).toThrow('hash changed') + const queued = enqueue('queued', jobs.get(published.id).experimentId) + expect(() => readReferenceTraces(db, { ...published, id: queued.id, cohortId, metric: 'heart_rate' })).toThrow('successful replay') + jobs.releaseResult(published.id, published.expectedContentHash) + expect(() => readReferenceTraces(db, { ...published, cohortId, metric: 'heart_rate' })).toThrow('released') + }) +}) diff --git a/src/biometrics/backtest/tests/uncertainty.test.ts b/src/biometrics/backtest/tests/uncertainty.test.ts new file mode 100644 index 00000000..a617374e --- /dev/null +++ b/src/biometrics/backtest/tests/uncertainty.test.ts @@ -0,0 +1,42 @@ +// @vitest-environment node +import { expect, it } from 'vitest' +import { nightUncertainty, type NightBootstrapPolicy } from '../uncertainty' +import { experimentProtocolSchema } from '../contracts' +import { configHash } from '../../algorithms/canonical' +const policy: NightBootstrapPolicy = { version: 'paired-night-percentile-v1', seed: 'fixture', resamples: 200, minimumNights: 3 } +const nights = [{ cohortId: 'a', difference: -0.5 }, { cohortId: 'b', difference: 0.2 }, { cohortId: 'c', difference: 0.8 }] +it('resamples paired nights reproducibly irrespective of input ordering', () => { + const result = nightUncertainty(nights, policy) + expect(result.status).toBe('estimated') + expect(result).toEqual(nightUncertainty([...nights].reverse(), policy)) + expect(result.interval?.lower).toBeLessThan(result.estimate ?? 0) + expect(result.interval?.upper).toBeGreaterThan(result.estimate ?? 0) + expect(result).toMatchObject({ scoredNights: 3, omittedNights: 0, samplingUnit: 'whole_night_cohort' }) +}) +it('does not fabricate precision from missing or identical nights', () => { + expect(nightUncertainty(nights)).toMatchObject({ status: 'not_requested', interval: null }) + expect(nightUncertainty([...nights.slice(0, 2), { cohortId: 'missing', difference: null }], policy)) + .toMatchObject({ status: 'insufficient_nights', scoredNights: 2, omittedNights: 1, interval: null }) + expect(nightUncertainty(nights.map(row => ({ ...row, difference: 0.5 })), policy)).toMatchObject({ status: 'degenerate', interval: null }) +}) +it('bounds computation and rejects invalid or duplicated evidence', () => { + expect(() => nightUncertainty([...nights, nights[0]], policy)).toThrow('Invalid paired-night') + expect(() => nightUncertainty([{ cohortId: 'bad', difference: NaN }], policy)).toThrow('Invalid paired-night') + expect(() => nightUncertainty(nights, { ...policy, resamples: 5001 })).toThrow() + expect(() => nightUncertainty(nights, { ...policy, minimumNights: 2 })).toThrow() +}) +it('pins uncertainty parameters in the experiment protocol hash', () => { + const base = { version: 'descriptive-evaluation-v1', mode: 'stage_replay', assumedStageCalibrationQuality: 0, pointToleranceMs: 30000, clockOffsetMs: 0 } + const first = experimentProtocolSchema.parse({ ...base, nightUncertainty: policy }) + const second = experimentProtocolSchema.parse({ ...base, nightUncertainty: { ...policy, seed: 'different' } }) + expect(configHash(first)).not.toBe(configHash(second)) + expect(experimentProtocolSchema.parse(base)).toEqual(base) +}) + +it('matches a separately computed Python SHA-256/quantile golden', () => { + const fixture = Array.from({ length: 12 }, (_, i) => ({ cohortId: String(i).padStart(2, '0'), difference: ((i * 7) % 11 - 5) / 10 })) + const result = nightUncertainty(fixture, policy) + expect(result.estimate).toBeCloseTo(-0.041666666666666664, 14) + expect(result.interval?.lower).toBeCloseTo(-0.22520833333333334, 14) + expect(result.interval?.upper).toBeCloseTo(0.125, 14) +}) diff --git a/src/biometrics/backtest/tests/waveform-dataset.test.ts b/src/biometrics/backtest/tests/waveform-dataset.test.ts new file mode 100644 index 00000000..48f61a75 --- /dev/null +++ b/src/biometrics/backtest/tests/waveform-dataset.test.ts @@ -0,0 +1,111 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { afterEach, beforeEach, expect, it } from 'vitest' +import * as schema from '@/src/db/biometrics-schema' +import { dataHash } from '../../algorithms/canonical' +import { DatasetStore } from '../datasets' +import { datasetManifestSchema } from '../contracts' +import { prepareWaveformDataset } from '../waveform-dataset' +import { retainedWaveformInputSchema, waveformDatasetInputsSchema } from '../waveform-contracts' + +let sqlite: Database.Database +let db: ReturnType> +let datasets: DatasetStore +let base: ReturnType +const start = 1791673200000 +beforeEach(() => { + sqlite = new Database(':memory:') + db = drizzle(sqlite, { schema }) + migrate(db, { migrationsFolder: 'src/db/biometrics-migrations' }) + for (const id of [1, 2, 3]) db.insert(schema.sleepRecords).values({ id, side: 'left', + enteredBedAt: new Date(start + id * 86400000), leftBedAt: new Date(start + id * 86400000 + 3600000), + sleepDurationSeconds: 3600, timesExitedBed: 0 }).run() + datasets = new DatasetStore(db) + base = datasets.read(datasets.create({ side: 'left', subjectId: 'fixture', sessionIds: [1, 2, 3], + referenceImportIds: [], timezone: 'UTC', idempotencyKey: 'base' }).id) +}) +afterEach(() => sqlite.close()) +function inputs() { + return datasetManifestSchema.parse(base.manifest).splitProtocol.assignments.map((cohort, index) => ({ + cohortId: cohort.cohortId, + input: { version: 'retained-waveform-input-v1' as const, retained_start_ms: cohort.startMs - 300000, + score_start_ms: cohort.startMs, score_end_ms: cohort.endMs, contexts: [], + artifacts: [{ artifact: { name: `${String(index + 1).repeat(64)}.RAW.gz`, sha256: String(index + 1).repeat(64), compressed_bytes: 100, expanded_bytes: 200 }, + first_event_ms: cohort.startMs - 300000, last_event_ms: cohort.endMs - 1 }] }, + })) +} +function source() { + return { ...base, manifest: datasetManifestSchema.parse(base.manifest) } +} + +it('creates a separate immutable snapshot with stable request/content hashes and original observations', () => { + const before = structuredClone(base) + const first = prepareWaveformDataset(source(), inputs().reverse(), 'raw', 1000) + const retry = prepareWaveformDataset(source(), inputs(), 'raw', 2000) + expect(first.requestHash).toBe(retry.requestHash) + expect(first.contentHash).toBe(retry.contentHash) + expect(first.id).not.toBe(retry.id) + expect(base).toEqual(before) + expect(first.snapshot.sessions).toEqual(base.snapshot.sessions) + expect(first.manifest.parentDataset).toEqual({ id: base.id, contentHash: base.contentHash }) + expect(first.manifest.capabilities.signalReplay).toBe(true) + expect(first.contentHash).toBe(dataHash({ manifest: first.manifest, snapshot: first.snapshot })) + expect(datasets.list('left')).toHaveLength(1) // Preparation performs no publication. +}) + +it('preserves waveform inputs through the shared dataset reader and detects later corruption', () => { + const prepared = prepareWaveformDataset(source(), inputs(), 'raw') + db.insert(schema.biometricDatasets).values({ ...prepared, + byteLength: Buffer.byteLength(JSON.stringify(prepared.manifest)) + Buffer.byteLength(JSON.stringify(prepared.snapshot)) }).run() + const retained = datasets.read(prepared.id) + expect(retained.snapshot).toEqual(prepared.snapshot) + sqlite.prepare('UPDATE biometric_datasets SET snapshot = json_set(snapshot, \'$.waveformInputs[0].input.artifacts[0].artifact.expanded_bytes\', 999) WHERE id = ?').run(prepared.id) + expect(() => datasets.read(prepared.id)).toThrow('integrity') +}) + +it('rejects missing, duplicated and mismatched whole-night inputs and forged source identity', () => { + expect(() => prepareWaveformDataset(source(), inputs().slice(1), 'raw')).toThrow('every whole-night') + const duplicated = inputs() + duplicated[1] = duplicated[0] + expect(() => prepareWaveformDataset(source(), duplicated, 'raw')).toThrow('each waveform cohort once') + const wrongWindow = inputs() + wrongWindow[0].input.score_start_ms++ + expect(() => prepareWaveformDataset(source(), wrongWindow, 'raw')).toThrow('whole-night cohort') + expect(() => prepareWaveformDataset({ ...source(), contentHash: '0'.repeat(64) }, inputs(), 'raw')).toThrow('integrity') +}) + +it('rejects overlapping source/context windows, invalid modes and inconsistent shared evidence', () => { + const input = inputs()[0].input + expect(() => retainedWaveformInputSchema.parse({ ...input, artifacts: [...input.artifacts, input.artifacts[0]] })).toThrow('distinct') + const context = { start_ms: input.retained_start_ms, end_ms: input.score_end_ms, + active_sides: ['right', 'left'], left_occupied: null, right_occupied: null, pump_context_known: false } + expect(() => retainedWaveformInputSchema.parse({ ...input, contexts: [context] })).toThrow('sleeper mode') + expect(() => retainedWaveformInputSchema.parse({ ...input, contexts: [{ ...context, active_sides: null }, { ...context, active_sides: null }] })).toThrow('nonoverlapping') + const shared = [{ cohortId: 'one', input }, { cohortId: 'two', input: structuredClone(input) }] + expect(waveformDatasetInputsSchema.parse(shared)).toEqual(shared) + shared[1].input.artifacts[0].artifact.expanded_bytes++ + expect(() => waveformDatasetInputsSchema.parse(shared)).toThrow('Conflicting evidence') +}) + +it('accepts only processed sources and bounds the retained dataset size', () => { + const prepared = prepareWaveformDataset(source(), inputs(), 'raw') + expect(() => prepareWaveformDataset(prepared, inputs(), 'again')).toThrow('processed source dataset') + expect(() => prepareWaveformDataset({ ...source(), manifest: prepared.manifest }, inputs(), 'again')).toThrow('processed source dataset') + const oversized = source() + oversized.snapshot.sessions[0].presentIntervals = Array.from({ length: 130 }, () => 'x'.repeat(65536)) + oversized.contentHash = dataHash({ manifest: oversized.manifest, snapshot: oversized.snapshot }) + expect(() => prepareWaveformDataset(oversized, inputs(), 'raw')).toThrow('byte budget') +}) + +it('detaches opaque historical context from caller-owned objects before hashing', () => { + const mutable = source() + const interval = { start: start + 1, end: start + 2 } + mutable.snapshot.sessions[0].presentIntervals = [interval] + mutable.contentHash = dataHash({ manifest: mutable.manifest, snapshot: mutable.snapshot }) + const prepared = prepareWaveformDataset(mutable, inputs(), 'raw') + interval.end++ + expect(prepared.snapshot.sessions[0].presentIntervals).toEqual([{ start: start + 1, end: start + 2 }]) + expect(dataHash({ manifest: prepared.manifest, snapshot: prepared.snapshot })).toBe(prepared.contentHash) +}) diff --git a/src/biometrics/backtest/tests/waveform-evaluator.test.ts b/src/biometrics/backtest/tests/waveform-evaluator.test.ts new file mode 100644 index 00000000..9b020873 --- /dev/null +++ b/src/biometrics/backtest/tests/waveform-evaluator.test.ts @@ -0,0 +1,150 @@ +import { previewReferenceImport } from '../../references/normalize' +// @vitest-environment node +import { expect, it } from 'vitest' +import { backtestFixture } from './fixtures/backtest' +import { dataHash, configHash } from '../../algorithms/canonical' +import { evaluateBacktest } from '../evaluator' +import { summarizeBacktestResult } from '../report-summary' +import { defaultConfig, manifestHash } from '../../algorithms/registry' + +function fixture() { + const base = backtestFixture() + const cohort = base.dataset.manifest.splitProtocol.assignments[0] + const digest = 'a'.repeat(64), build = 'b'.repeat(64), resolvedHash = configHash(defaultConfig) + const dataset = { ...base.dataset, snapshot: { ...base.dataset.snapshot, version: 'waveform-dataset-v1' as const, + waveformInputs: [{ cohortId: cohort.cohortId, input: { version: 'retained-waveform-input-v1' as const, + retained_start_ms: cohort.startMs, score_start_ms: cohort.startMs, score_end_ms: cohort.endMs, contexts: [], + artifacts: [{ artifact: { sha256: digest, name: `${digest}.RAW.gz`, compressed_bytes: 1, expanded_bytes: 2 }, + first_event_ms: cohort.startMs, last_event_ms: cohort.endMs - 1 }] } }] } } + const protocol = { ...base.job.protocol, mode: 'signal_replay' as const, waveformStages: false } + const readings = [{ side: 'left' as const, timestampMs: cohort.startMs, heartRate: 62, hrvIndex: null, + hrvMetric: 'window_ibi_variability_v1' as const, breathingRate: 12, qualityScore: 1, flags: [], hrRaw: 62, + sourceSide: 'left' as const, configHash: resolvedHash, limitations: ['unknown_context'] }] + const outputs = [{ cohortId: cohort.cohortId, configHash: resolvedHash, readings, run: { + version: 'retained-waveform-run-v1' as const, build_hash: build, decoded_records: 1, raw_records: 1, + counts: { records: 1, accepted_frames: 1, empty_channel_frames: 0, pump_gated_frames: 0, preroll_outputs: 0, scored_outputs: 1 }, + limitations: ['unknown_context'], config_hashes: { left: resolvedHash, right: resolvedHash }, artifact_hashes: [digest], + } }] + return { dataset, job: { ...base.job, datasetHash: dataHash(dataset), protocol, protocolHash: configHash(protocol), + waveform: { pythonBuildHash: build, otherSide: { manifestHash, resolvedHash, resolved: defaultConfig } } }, + waveform: { hash: dataHash(outputs), outputs } } +} +it('scores derived readings for matching labels without replacing immutable dataset identity', () => { + const work = fixture() + const before = structuredClone(work) + const report = evaluateBacktest(work) + expect(report.version).toBe('waveform-backtest-report-v1') + expect(report.datasetHash).toBe(work.job.datasetHash) + expect(report.nights[0].uniqueObservationCount).toBe(1) + expect(report.nights[0].evaluations[0].windows[0].observationCount).toBe(1) + expect(report.nights[0].evaluations[0].windows[0].numeric[0]).toMatchObject({ pairedCount: 1, metrics: { mae: 0 } }) + expect(report.nights[0].evaluations[1].windows).toEqual(report.nights[0].evaluations[0].windows) + expect(report.nights[0].evaluations[0].windows[0].blocks).toEqual([]) + expect(report.nights[0].numericComparisons?.[0].metrics).toEqual(expect.arrayContaining([ + expect.objectContaining({ metric: 'heart_rate', baseline: 'default', candidate: 'current', + commonObservationCount: 1, differences: expect.objectContaining({ commonMae: 0 }), coverageReduced: false }), + ])) + const summary = summarizeBacktestResult({ available: true, report }) + expect(summary?.report).toMatchObject({ nights: [expect.objectContaining({ numericComparisons: report.nights[0].numericComparisons })] }) + expect(report.numericNightSummary?.[0]).toMatchObject({ metric: 'heart_rate', scoredNights: 1, equalNightMeanMaeDifference: 0, uncertainty: { statistic: 'equal_night_mean_common_mae_difference', status: 'not_requested' } }) + expect(work).toEqual(before) +}) +it('rejects incomplete, mismatched or extraneous derived waveform evidence', () => { + for (const mutate of [ + (work: ReturnType) => { work.waveform.hash = '0'.repeat(64) }, + (work: ReturnType) => { work.waveform.outputs = [] }, + (work: ReturnType) => { work.waveform.outputs.push(structuredClone(work.waveform.outputs[0])) }, + (work: ReturnType) => { work.waveform.outputs[0].run.build_hash = '0'.repeat(64) }, + (work: ReturnType) => { work.waveform.outputs[0].run.config_hashes.right = '0'.repeat(64) }, + (work: ReturnType) => { work.waveform.outputs[0].run.artifact_hashes = ['0'.repeat(64)] }, + (work: ReturnType) => { work.waveform.outputs[0].readings[0].timestampMs-- }, + (work: ReturnType) => { work.waveform.outputs[0].readings[0].configHash = '0'.repeat(64) }, + ]) { + const work = fixture() + mutate(work) + if (work.waveform.hash !== '0'.repeat(64)) work.waveform.hash = dataHash(work.waveform.outputs) + expect(() => evaluateBacktest(work)).toThrow() + } +}) +it('rejects waveform evidence supplied to processed evaluation and missing evidence in signal replay', () => { + expect(() => evaluateBacktest({ ...backtestFixture(), waveform: fixture().waveform })).toThrow('Unexpected waveform') + const work = fixture() + expect(() => evaluateBacktest({ ...work, waveform: undefined })).toThrow('Missing or corrupt') +}) + +it('retains objective thresholds but cannot promote without qualified measured evidence', () => { + const work = fixture() + const protocol = { ...work.job.protocol, objectivePolicy: { version: 'numeric-objective-v1' as const, qualificationId: 'test-only', metric: 'heart_rate' as const, + minimumNights: 3, minimumPairedReferencesPerNight: 10, maximumLostObservationFraction: 0, minimumMaeImprovement: 2, + maximumNightMaeRegression: 1, maximumRegressedNightFraction: 0, maximumSubgroupMaeRegression: 1, requiredSubgroups: [], + operational: { maximumWallMs: 1000, maximumPeakRssBytes: 1000, maximumIngestionAgeMs: 1000 } } } + const report = evaluateBacktest({ ...work, job: { ...work.job, protocol, protocolHash: configHash(protocol), + configs: [...work.job.configs, { ...work.job.configs[0], label: 'candidate' }] } }) + expect(report.objectiveAssessments).toHaveLength(2) + expect(report.objectiveAssessments[0]).toMatchObject({ status: 'insufficient_evidence', designated: false, + missingEvidence: expect.arrayContaining(['policy_not_designated', 'operational_evidence_unavailable']) }) + expect(report.protocol.objectivePolicy).toEqual(protocol.objectivePolicy) +}) + +it('retains measured waveform costs through compact report projection without claiming whole-job health', () => { + const work = fixture() + const outputs = work.waveform.outputs.map(output => ({ ...output, run: { ...output.run, resources: { + version: 'waveform-process-resources-v1' as const, wallMs: 120, cpuMs: 80, peakRssBytes: 102400, + scope: 'python_process_until_result' as const, memoryScope: 'process_lifetime_high_water_mark' as const, + } } })) + const report = evaluateBacktest({ ...work, waveform: { hash: dataHash(outputs), outputs } }) + const summary = summarizeBacktestResult({ available: true, report }) + expect(summary?.report).toMatchObject({ waveform: { resources: { + expectedProcesses: outputs.length, measuredProcesses: outputs.length, totalWallMs: outputs.length * 120, + totalCpuMs: outputs.length * 80, largestProcessPeakRssBytes: 102400, ingestionHealthMeasured: false, + } } }) +}) + +it('exposes a late-window regression hidden by lower whole-night error', () => { + const work = fixture(), start = work.dataset.snapshot.sessions[0].startMs + const baseline = work.waveform.outputs[0] + baseline.readings = [80, 60].map((heartRate, index) => ({ ...baseline.readings[0], heartRate, timestampMs: start + index * 300000 })) + baseline.run.counts.scored_outputs = 2 + const candidate = structuredClone(defaultConfig) + candidate.heart_rate.trackingToleranceBpm = 20 + const candidateHash = configHash(candidate) + const output = { ...baseline, configHash: candidateHash, + readings: baseline.readings.map((row, index) => ({ ...row, heartRate: index ? 65 : 60, configHash: candidateHash })), + run: { ...baseline.run, config_hashes: { ...baseline.run.config_hashes, left: candidateHash } } } + work.dataset.snapshot.references[0].measurements = previewReferenceImport({ side: 'left', source: 'fixture', subjectId: 'A', rows: [ + { recordKey: 'early', metric: 'heart_rate', timestampMs: start, value: 60, unit: 'bpm' }, + { recordKey: 'late', metric: 'heart_rate', timestampMs: start + 300000, value: 60, unit: 'bpm' }, + ] }).measurements + const protocol = { ...work.job.protocol, objectivePolicy: { version: 'numeric-objective-v1' as const, qualificationId: 'test-only', metric: 'heart_rate' as const, + minimumNights: 3, minimumPairedReferencesPerNight: 1, maximumLostObservationFraction: 0, minimumMaeImprovement: 2, + maximumNightMaeRegression: 1, maximumRegressedNightFraction: 0, maximumSubgroupMaeRegression: 1, requiredSubgroups: ['window_second_half'], + operational: { maximumWallMs: 1000, maximumPeakRssBytes: 1000, maximumIngestionAgeMs: 1000 } } } + const outputs = [baseline, output] + const report = evaluateBacktest({ ...work, waveform: { outputs, hash: dataHash(outputs) }, job: { ...work.job, + datasetHash: dataHash(work.dataset), protocol, protocolHash: configHash(protocol), + configs: [...work.job.configs, { ...work.job.configs[0], label: 'candidate', resolved: candidate, resolvedHash: candidateHash }] } }) + expect(report.numericNightSummary?.find(row => row.baseline === 'default' && row.candidate === 'candidate' && row.metric === 'heart_rate')?.equalNightMeanMaeDifference).toBe(-7.5) + expect(report.numericSubgroupSummary?.find(row => row.name === 'window_second_half' && row.summary.baseline === 'default' && row.summary.candidate === 'candidate')?.summary.equalNightMeanMaeDifference).toBe(5) + expect(report.objectiveAssessments[0].evidence.subgroups).toContainEqual({ name: 'window_second_half', pairedNights: 1, maeDifference: 5 }) + expect(report.objectiveAssessments[0].evidence.subgroups).toContainEqual({ name: 'reference_intervals', pairedNights: 0, maeDifference: null }) +}) + +it('evaluates each numeric policy against both baselines without mixing metric subgroups', () => { + const work = fixture() + const objectivePolicies = (['heart_rate', 'breathing_rate', 'window_ibi_variability_v1'] as const).map(metric => ({ + version: 'numeric-objective-v1' as const, qualificationId: 'multi-test', metric, minimumNights: 3, + minimumPairedReferencesPerNight: 1, maximumLostObservationFraction: 0, minimumMaeImprovement: 1, + maximumNightMaeRegression: 1, maximumRegressedNightFraction: 0, maximumSubgroupMaeRegression: 1, + requiredSubgroups: ['reference_points'], operational: { maximumWallMs: 1000, maximumPeakRssBytes: 1000, maximumIngestionAgeMs: 1000 }, + })) + const protocol = { ...work.job.protocol, objectivePolicies } + const report = evaluateBacktest({ ...work, job: { ...work.job, protocol, protocolHash: configHash(protocol), + configs: [...work.job.configs, { ...work.job.configs[0], label: 'candidate' }] } }) + expect(report.objectiveAssessments).toHaveLength(6) + for (const policy of objectivePolicies) { + const assessments = report.objectiveAssessments.filter(row => row.policy.metric === policy.metric) + expect(assessments.map(row => row.baseline).sort()).toEqual(['current', 'default']) + expect(assessments.every(row => row.status === 'insufficient_evidence' && !row.designated)).toBe(true) + expect(report.numericSubgroupSummary?.filter(row => row.summary.metric === policy.metric)).toHaveLength(12) + } +}) diff --git a/src/biometrics/backtest/tests/waveform-execution.test.ts b/src/biometrics/backtest/tests/waveform-execution.test.ts new file mode 100644 index 00000000..38b350da --- /dev/null +++ b/src/biometrics/backtest/tests/waveform-execution.test.ts @@ -0,0 +1,114 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { mkdtemp, realpath, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import * as schema from '@/src/db/biometrics-schema' +import { configHash, dataHash } from '../../algorithms/canonical' +import { defaultConfig, manifestHash } from '../../algorithms/registry' +import { datasetManifestSchema } from '../contracts' +import { DatasetStore } from '../datasets' +import { podRawInspectionLocation } from '../raw-inspection-child' +import { prepareWaveformDataset } from '../waveform-dataset' +import { waveformEvaluatorBuildIdentity } from '../waveform-build' +import { runWaveformChild } from '../waveform-child' +import { executeWaveformInputs, verifyWaveformJobBuild } from '../waveform-execution' +import { backtestFixture } from './fixtures/backtest' + +vi.mock('../waveform-build', () => ({ waveformEvaluatorBuildIdentity: vi.fn() })) +vi.mock('../waveform-child', () => ({ runWaveformChild: vi.fn() })) + +const build = 'b'.repeat(64), python = 'c'.repeat(64), digest = 'a'.repeat(64), resolvedHash = configHash(defaultConfig) +let retained: ReturnType +const start = 1791673200000 +function fixture(side: 'left' | 'right' = 'left', labels: Array<'default' | 'current' | 'candidate'> = ['default', 'current']) { + const base = backtestFixture() + const dataset = structuredClone({ manifest: retained.manifest, snapshot: { ...retained.snapshot, side } }) + const protocol = { ...base.job.protocol, mode: 'signal_replay' as const } + return { dataset, job: { ...base.job, evaluatorBuildHash: build, datasetHash: dataHash(dataset), protocol, protocolHash: configHash(protocol), + configs: labels.map(label => ({ ...base.job.configs[0], label })), + waveform: { pythonBuildHash: python, otherSide: { manifestHash, resolvedHash, resolved: defaultConfig } } } } +} +const reading = (timestampMs: number) => ({ side: 'left' as const, timestampMs, heartRate: 62, hrvIndex: null, hrvMetric: 'window_ibi_variability_v1' as const, + breathingRate: 12, qualityScore: 1, flags: [], hrRaw: 62, sourceSide: 'left' as const, configHash: resolvedHash, limitations: [] }) +const run = { version: 'retained-waveform-run-v1' as const, build_hash: python, decoded_records: 1, raw_records: 1, + counts: { records: 1, accepted_frames: 1, empty_channel_frames: 0, pump_gated_frames: 0, preroll_outputs: 0, scored_outputs: 1 }, + limitations: [], config_hashes: { left: resolvedHash, right: resolvedHash }, artifact_hashes: [digest] } +const rawLocation = { pythonPath: '/python', modulesPath: '/modules', archiveDirectory: '/archive' } +let directory: string, sqlite: Database.Database, db: ReturnType> +beforeEach(async () => { + directory = await realpath(await mkdtemp(join(tmpdir(), 'waveform-execution-'))) + sqlite = new Database(join(directory, 'biometrics.db')) + db = drizzle(sqlite, { schema }) + migrate(db, { migrationsFolder: 'src/db/biometrics-migrations' }) + db.insert(schema.sleepRecords).values({ id: 1, side: 'left', enteredBedAt: new Date(start), leftBedAt: new Date(start + 600000), sleepDurationSeconds: 600, timesExitedBed: 0 }).run() + const datasets = new DatasetStore(db) + const source = datasets.read(datasets.create({ side: 'left', subjectId: 'A', sessionIds: [1], referenceImportIds: [], timezone: 'UTC', idempotencyKey: 'source' }).id) + const cohort = datasetManifestSchema.parse(source.manifest).splitProtocol.assignments[0] + retained = prepareWaveformDataset({ ...source, manifest: datasetManifestSchema.parse(source.manifest) }, [{ cohortId: cohort.cohortId, + input: { version: 'retained-waveform-input-v1', retained_start_ms: cohort.startMs, score_start_ms: cohort.startMs, score_end_ms: cohort.endMs, contexts: [], + artifacts: [{ artifact: { sha256: digest, name: `${digest}.RAW.gz`, compressed_bytes: 1, expanded_bytes: 2 }, first_event_ms: cohort.startMs, last_event_ms: cohort.endMs - 1 }] } }], 'raw') + vi.mocked(waveformEvaluatorBuildIdentity).mockResolvedValue({ hash: build, python: { hash: python } } as Awaited>) + vi.mocked(runWaveformChild).mockResolvedValue({ readings: [reading(start)], run }) +}) +afterEach(async () => { + vi.clearAllMocks() + sqlite.close() + await rm(directory, { recursive: true, force: true }) +}) +const options = () => ({ workerPath: join(directory, 'worker.cjs'), signal: new AbortController().signal }) + +it('pins the coordinator beside the worker and rejects any build drift since enqueue', async () => { + const work = fixture() + await verifyWaveformJobBuild(work, options()) + expect(waveformEvaluatorBuildIdentity).toHaveBeenLastCalledWith(join(directory, 'coordinator.cjs'), join(directory, 'worker.cjs'), podRawInspectionLocation, expect.any(AbortSignal)) + await verifyWaveformJobBuild(work, { ...options(), rawLocation }) + expect(waveformEvaluatorBuildIdentity).toHaveBeenLastCalledWith(expect.any(String), expect.any(String), rawLocation, expect.any(AbortSignal)) + vi.mocked(waveformEvaluatorBuildIdentity).mockResolvedValueOnce({ hash: '0'.repeat(64), python: { hash: python } } as Awaited>) + await expect(verifyWaveformJobBuild(work, options())).rejects.toThrow('build changed') + vi.mocked(waveformEvaluatorBuildIdentity).mockResolvedValueOnce({ hash: build, python: { hash: '0'.repeat(64) } } as Awaited>) + await expect(verifyWaveformJobBuild(work, options())).rejects.toThrow('build changed') +}) + +it('runs each cohort and distinct configuration through Python beside the persistent database', async () => { + const work = fixture('left', ['default', 'current', 'candidate']) + const result = await executeWaveformInputs(db, work, { ...options(), rawLocation, timeoutMs: 5000 }) + expect(runWaveformChild).toHaveBeenCalledTimes(1) + expect(runWaveformChild).toHaveBeenCalledWith({ ...rawLocation, retainedDirectory: join(directory, 'biometric-raw') }, + { version: 'retained-waveform-work-v1', input: expect.objectContaining({ version: 'retained-waveform-input-v1', score_start_ms: start }), configs: { left: defaultConfig, right: defaultConfig } }, + { signal: expect.any(AbortSignal), timeoutMs: 5000, cpuSeconds: undefined, expectedBuildHash: python }) + expect(result.outputs).toEqual([{ cohortId: retained.manifest.splitProtocol.assignments[0].cohortId, configHash: resolvedHash, readings: [reading(start)], run }]) + expect(result.hash).toBe(dataHash(result.outputs)) + expect(waveformEvaluatorBuildIdentity).toHaveBeenCalledTimes(2) +}) + +it('routes the retained side to the candidate configuration and the replay budget to the child', async () => { + const work = fixture('right') + const replayBudget = { jobWallMs: 60000, waveformWallMs: 30000, waveformCpuSeconds: 20 } + vi.mocked(runWaveformChild).mockResolvedValueOnce({ readings: [{ ...reading(start), side: 'right', sourceSide: 'right' }], run }) + await executeWaveformInputs(db, work, { ...options(), replayBudget }) + expect(runWaveformChild).toHaveBeenCalledWith({ ...podRawInspectionLocation, retainedDirectory: join(directory, 'biometric-raw') }, + expect.objectContaining({ configs: { left: defaultConfig, right: defaultConfig } }), + expect.objectContaining({ timeoutMs: 30000, cpuSeconds: 20 })) +}) + +it('rejects processed jobs, in-memory databases, missing cohorts, cancellation and reading budget overruns', async () => { + const processed = backtestFixture() + await expect(executeWaveformInputs(db, processed, options())).rejects.toThrow('Missing retained waveform job inputs') + await expect(executeWaveformInputs(db, { ...fixture(), job: { ...fixture().job, waveform: undefined } }, options())).rejects.toThrow('Missing retained waveform job inputs') + await expect(executeWaveformInputs(db, { ...fixture(), dataset: processed.dataset }, options())).rejects.toThrow('Missing retained waveform job inputs') + const memory = drizzle(new Database(':memory:'), { schema }) + await expect(executeWaveformInputs(memory, fixture(), options())).rejects.toThrow('persistent database') + const orphan = fixture() + if (orphan.dataset.snapshot.version === 'waveform-dataset-v1') orphan.dataset.snapshot.waveformInputs[0].cohortId = 'other' + await expect(executeWaveformInputs(db, orphan, options())).rejects.toThrow('Missing retained waveform cohort') + const controller = new AbortController() + controller.abort() + await expect(executeWaveformInputs(db, fixture(), { ...options(), signal: controller.signal })).rejects.toThrow('cancelled') + expect(runWaveformChild).not.toHaveBeenCalled() + vi.mocked(runWaveformChild).mockResolvedValueOnce({ readings: Array.from({ length: 20001 }, (_, index) => reading(start + index)), run }) + await expect(executeWaveformInputs(db, fixture(), options())).rejects.toThrow('reading budget exceeded') +}) diff --git a/src/biometrics/backtest/tests/waveform-selection.test.ts b/src/biometrics/backtest/tests/waveform-selection.test.ts new file mode 100644 index 00000000..3c2db220 --- /dev/null +++ b/src/biometrics/backtest/tests/waveform-selection.test.ts @@ -0,0 +1,155 @@ +// @vitest-environment node +import Database from 'better-sqlite3' +import { drizzle } from 'drizzle-orm/better-sqlite3' +import { migrate } from 'drizzle-orm/better-sqlite3/migrator' +import { afterEach, beforeEach, expect, it } from 'vitest' +import * as schema from '@/src/db/biometrics-schema' +import { DatasetStore } from '../datasets' +import { RawSourceCatalog } from '../raw-catalog' +import { createWaveformDatasetSchema, prepareSelectedWaveformDataset, recordedWaveformContexts, selectedWaveformRetry, waveformSelectionRequestHash } from '../waveform-selection' +import { datasetManifestSchema } from '../contracts' + +let sqlite: Database.Database +let db: ReturnType> +let parent: ReturnType +const start = 1700000000000 +let sourceIds: string[] +beforeEach(() => { + sqlite = new Database(':memory:') + db = drizzle(sqlite, { schema }) + migrate(db, { migrationsFolder: 'src/db/biometrics-migrations' }) + db.insert(schema.sleepRecords).values({ id: 1, side: 'left', enteredBedAt: new Date(start), leftBedAt: new Date(start + 600000), + sleepDurationSeconds: 600, timesExitedBed: 1, presentIntervals: [[start / 1000, start / 1000 + 60]], + notPresentIntervals: [[start / 1000 + 60, start / 1000 + 120]] }).run() + const store = new DatasetStore(db) + parent = store.read(store.create({ side: 'left', subjectId: 'test', sessionIds: [1], referenceImportIds: [], timezone: 'UTC', idempotencyKey: 'parent' }).id) + sourceIds = [0, 1].map((index) => { + const sha256 = String(index + 1).repeat(64) + return new RawSourceCatalog(db).record({ source_name: `${index}.RAW.gz`, retained: false, + artifact: { name: `${sha256}.RAW.gz`, sha256, compressed_bytes: 100, expanded_bytes: 200 }, + inventory: { version: 'retained-raw-inventory-v1', artifact_sha256: sha256, decoded_records: 2, piezo_records: 1, pump_records: 1, + usable_dual_frames: 1, empty_channel_frames: 0, oversize_channel_frames: 0, invalid_timestamps: 0, timestamp_reversals: 0, + duplicate_timestamps: 0, first_event_ms: start - 300000 + index * 450000, last_event_ms: start + 149999 + index * 450000, + channels: { left1: { samples: 10, lost_markers: 0, truncated_tail_bytes: 0 }, right1: { samples: 10, lost_markers: 0, truncated_tail_bytes: 0 } }, + ordered_waveform_available: true, limitations: [] } }) + }) +}) +afterEach(() => sqlite.close()) +const request = () => ({ parentDatasetId: parent.id, expectedContentHash: parent.contentHash, idempotencyKey: 'waveform', + cohorts: [{ cohortId: datasetManifestSchema.parse(parent.manifest).splitProtocol.assignments[0].cohortId, sourceIds: [...sourceIds].reverse() }] }) + +it('resolves ordered immutable source identities and recorded occupancy without reading current state', () => { + sqlite.prepare('UPDATE sleep_records SET present_intervals = NULL, not_present_intervals = NULL').run() + const result = prepareSelectedWaveformDataset(db, request(), 123) + expect(result.selections.map(item => item.source_name)).toEqual(['0.RAW.gz', '1.RAW.gz']) + if (result.prepared.snapshot.version !== 'waveform-dataset-v1') throw new Error('Wrong snapshot') + expect(result.prepared.snapshot.waveformInputs[0].input).toMatchObject({ retained_start_ms: start - 300000, + score_start_ms: start, score_end_ms: start + 600000, contexts: [ + { start_ms: start, end_ms: start + 60000, left_occupied: true, right_occupied: null, active_sides: null, pump_context_known: false }, + { start_ms: start + 60000, end_ms: start + 120000, left_occupied: false, right_occupied: null, active_sides: null, pump_context_known: false }, + ] }) + const retry = request() + retry.cohorts[0].sourceIds.reverse() + expect(prepareSelectedWaveformDataset(db, retry, 456).prepared.contentHash).toBe(result.prepared.contentHash) + expect(sqlite.prepare('SELECT count(*) n FROM biometric_datasets').get()).toEqual({ n: 1 }) + expect(sqlite.prepare('SELECT count(*) n FROM biometric_raw_artifacts').get()).toEqual({ n: 0 }) +}) + +it('rejects stale, duplicate, incomplete and forged selections', () => { + expect(() => prepareSelectedWaveformDataset(db, { ...request(), expectedContentHash: '0'.repeat(64) })).toThrow('expected immutable') + expect(() => prepareSelectedWaveformDataset(db, { ...request(), cohorts: [...request().cohorts, ...request().cohorts] })).toThrow('each whole-night') + const duplicate = request() + duplicate.cohorts[0].sourceIds = [sourceIds[0], sourceIds[0]] + expect(() => prepareSelectedWaveformDataset(db, duplicate)).toThrow('each RAW source') + const missing = request() + missing.cohorts[0].cohortId = 'missing' + expect(() => prepareSelectedWaveformDataset(db, missing)).toThrow('Missing whole-night') + expect(() => createWaveformDatasetSchema.parse({ ...request(), archiveDirectory: '/tmp' })).toThrow() + new RawSourceCatalog(db).markUnavailable(sourceIds[0]) + expect(() => prepareSelectedWaveformDataset(db, request())).toThrow('unavailable') +}) + +it('retains unknown context for gaps, malformed evidence and contradictory intervals', () => { + const snapshot = structuredClone(parent.snapshot) + snapshot.sessions[0].presentIntervals = [[start / 1000 - 10, start / 1000 + 60], ['bad', 10], [Infinity, 50], [10, 0]] + snapshot.sessions[0].notPresentIntervals = [[start / 1000 + 30, start / 1000 + 90]] + expect(recordedWaveformContexts(snapshot, start - 300000, start + 600000)).toEqual([ + { start_ms: start, end_ms: start + 30000, left_occupied: true, right_occupied: null, active_sides: null, pump_context_known: false }, + { start_ms: start + 60000, end_ms: start + 90000, left_occupied: false, right_occupied: null, active_sides: null, pump_context_known: false }, + ]) + snapshot.side = 'right' + expect(recordedWaveformContexts(snapshot, start + 10000, start + 20000)).toEqual([ + { start_ms: start + 10000, end_ms: start + 20000, right_occupied: true, left_occupied: null, active_sides: null, pump_context_known: false }, + ]) + snapshot.sessions[0].presentIntervals = null + snapshot.sessions[0].notPresentIntervals = { unrecognized: true } + expect(recordedWaveformContexts(snapshot, start, start + 600000)).toEqual([]) +}) + +it('merges consistent overlapping intervals, preserves fractional seconds and bounds recorded evidence', () => { + const snapshot = structuredClone(parent.snapshot) + snapshot.sessions[0].presentIntervals = [[start / 1000 + 0.125, start / 1000 + 40], [start / 1000 + 30, start / 1000 + 60]] + snapshot.sessions[0].notPresentIntervals = null + expect(recordedWaveformContexts(snapshot, start, start + 600000)).toEqual([ + { start_ms: start + 125, end_ms: start + 60000, left_occupied: true, right_occupied: null, active_sides: null, pump_context_known: false }, + ]) + snapshot.sessions[0].presentIntervals = Array(20001).fill([1, 2]) + expect(() => recordedWaveformContexts(snapshot, start, start + 600000)).toThrow('budget') +}) + +const recordSource = (source_name: string, sha256: string, first_event_ms: number, last_event_ms: number) => new RawSourceCatalog(db).record({ source_name, retained: false, + artifact: { name: `${sha256}.RAW.gz`, sha256, compressed_bytes: 100, expanded_bytes: 200 }, + inventory: { version: 'retained-raw-inventory-v1', artifact_sha256: sha256, decoded_records: 2, piezo_records: 1, pump_records: 1, + usable_dual_frames: 1, empty_channel_frames: 0, oversize_channel_frames: 0, invalid_timestamps: 0, timestamp_reversals: 0, + duplicate_timestamps: 0, first_event_ms, last_event_ms, + channels: { left1: { samples: 10, lost_markers: 0, truncated_tail_bytes: 0 }, right1: { samples: 10, lost_markers: 0, truncated_tail_bytes: 0 } }, + ordered_waveform_available: true, limitations: [] } }) + +it('hashes a selection independently of cohort and source order', () => { + const base = { parentDatasetId: parent.id, expectedContentHash: parent.contentHash, idempotencyKey: 'order' } + const a = { cohortId: 'a', sourceIds: [sourceIds[1], sourceIds[0]] }, b = { cohortId: 'b', sourceIds: [sourceIds[0]] } + expect(waveformSelectionRequestHash({ ...base, cohorts: [b, a] })).toBe(waveformSelectionRequestHash({ ...base, cohorts: [{ ...a, sourceIds: [sourceIds[0], sourceIds[1]] }, b] })) + expect(waveformSelectionRequestHash({ ...base, cohorts: [a, a] })).toBe(waveformSelectionRequestHash({ ...base, cohorts: [a, a] })) + expect(waveformSelectionRequestHash({ ...base, cohorts: [a] })).not.toBe(waveformSelectionRequestHash({ ...base, cohorts: [b] })) +}) + +it('replays an identical published selection by idempotency key and rejects a reused key', () => { + expect(selectedWaveformRetry(db, request())).toBeNull() + const { prepared } = prepareSelectedWaveformDataset(db, request(), 123) + db.insert(schema.biometricDatasets).values({ ...prepared, byteLength: Buffer.byteLength(JSON.stringify(prepared.manifest)) + Buffer.byteLength(JSON.stringify(prepared.snapshot)) }).run() + expect(selectedWaveformRetry(db, request())).toMatchObject({ id: prepared.id, contentHash: prepared.contentHash }) + const changed = request() + changed.cohorts[0].sourceIds = [sourceIds[0]] + expect(() => selectedWaveformRetry(db, changed)).toThrow('already used') +}) + +it('reuses a retained artifact for an unavailable source only when its recorded descriptor matches', () => { + new RawSourceCatalog(db).markUnavailable(sourceIds[0]) + db.insert(schema.biometricRawArtifacts).values({ contentHash: '1'.repeat(64), name: `${'1'.repeat(64)}.RAW.gz`, compressedBytes: 100, expandedBytes: 999, createdAtMs: 1 }).run() + expect(() => prepareSelectedWaveformDataset(db, request())).toThrow('unavailable') + sqlite.prepare('UPDATE biometric_raw_artifacts SET expanded_bytes = 200').run() + expect(prepareSelectedWaveformDataset(db, request()).selections.map(item => item.source_name)).toEqual(['0.RAW.gz', '1.RAW.gz']) +}) + +it('rejects sources whose event bounds tie or overlap inside one cohort', () => { + const tie = recordSource('tie.RAW.gz', '4'.repeat(64), start - 300000, start + 149999) + const selection = request() + selection.cohorts[0].sourceIds = [sourceIds[0], tie] + expect(() => prepareSelectedWaveformDataset(db, selection)).toThrow('distinct') +}) + +it('requires a selection for every cohort and attributes a shared artifact to its earliest source name', () => { + db.insert(schema.sleepRecords).values({ id: 2, side: 'left', enteredBedAt: new Date(start + 86400000), leftBedAt: new Date(start + 86400000 + 600000), sleepDurationSeconds: 600, timesExitedBed: 0 }).run() + const store = new DatasetStore(db) + const twoNights = store.read(store.create({ side: 'left', subjectId: 'test', sessionIds: [1, 2], referenceImportIds: [], timezone: 'UTC', idempotencyKey: 'two-nights' }).id) + const cohorts = datasetManifestSchema.parse(twoNights.manifest).splitProtocol.assignments + expect(cohorts).toHaveLength(2) + const sha256 = '3'.repeat(64) + const [later, earlier] = ['z.RAW.gz', 'b.RAW.gz'].map(name => recordSource(name, sha256, start - 300000, start + 86400000 + 599999)) + const base = { parentDatasetId: twoNights.id, expectedContentHash: twoNights.contentHash, idempotencyKey: 'shared' } + expect(() => prepareSelectedWaveformDataset(db, { ...base, cohorts: [{ cohortId: cohorts[0].cohortId, sourceIds: [later] }] })).toThrow('every whole-night cohort') + for (const order of [[later, earlier], [earlier, later]]) { + const result = prepareSelectedWaveformDataset(db, { ...base, cohorts: cohorts.map((cohort, index) => ({ cohortId: cohort.cohortId, sourceIds: [order[index]] })) }) + expect(result.selections.map(item => item.source_name)).toEqual(['b.RAW.gz']) + } +}) diff --git a/src/biometrics/backtest/trace-contracts.ts b/src/biometrics/backtest/trace-contracts.ts new file mode 100644 index 00000000..06fd06bc --- /dev/null +++ b/src/biometrics/backtest/trace-contracts.ts @@ -0,0 +1,12 @@ +import { z } from 'zod' +export const traceRequestSchema = z.object({ id: z.string().uuid(), expectedContentHash: z.string().regex(/^[a-f0-9]{64}$/), + configuration: z.enum(['default', 'current', 'candidate']), cohortId: z.string().min(1).max(200).optional(), + offset: z.number().int().min(0).max(1000000).default(0), limit: z.number().int().min(1).max(500).default(200), +}).strict() +export const tracePageSchema = z.object({ jobId: z.string().uuid(), contentHash: z.string(), configuration: z.enum(['default', 'current', 'candidate']), + provenance: z.literal('versioned_stage_replay'), timestampBasis: z.literal('original_trace_before_reference_clock_offset'), clockOffsetMs: z.number(), + totalItems: z.number().int(), offset: z.number().int(), limit: z.number().int(), downsampled: z.literal(false), + items: z.array(z.object({ cohortId: z.string(), configurationHash: z.string(), revisionId: z.string().uuid().nullable(), + windowStartMs: z.number(), windowEndMs: z.number(), startMs: z.number(), endMs: z.number(), stage: z.enum(['wake', 'light', 'deep', 'rem']) })), + nextCursor: z.object({ offset: z.number().int(), contentHash: z.string() }).nullable(), +}) diff --git a/src/biometrics/backtest/trace-page.ts b/src/biometrics/backtest/trace-page.ts new file mode 100644 index 00000000..ba8825b7 --- /dev/null +++ b/src/biometrics/backtest/trace-page.ts @@ -0,0 +1,47 @@ +import { z } from 'zod' +import { ReferenceInputError } from '../references/time' +import { traceRequestSchema, tracePageSchema } from './trace-contracts' + +const reportSchema = z.object({ version: z.enum(['processed-backtest-report-v1', 'waveform-backtest-report-v1']), + protocol: z.object({ mode: z.enum(['comparison', 'stage_replay', 'signal_replay']), waveformStages: z.boolean().optional(), clockOffsetMs: z.number().int() }), + nights: z.array(z.object({ cohortId: z.string(), evaluations: z.array(z.object({ label: z.enum(['default', 'current', 'candidate']), + configHash: z.string(), revisionId: z.string().uuid().nullable(), windows: z.array(z.object({ startMs: z.number().int(), endMs: z.number().int(), + blocks: z.array(z.object({ start: z.number().int(), end: z.number().int(), stage: z.enum(['wake', 'light', 'deep', 'rem']) })), + })), + })).max(3) })).max(60), +}) + +/** Selection changes only returned detail, never the retained evaluation or + * its scoring. Offset cursors are pinned to an immutable verified report hash. */ +export function pageStageTraces(report: unknown, raw: z.input) { + const input = traceRequestSchema.parse(raw) + const parsed = reportSchema.safeParse(report) + if (!parsed.success) throw new ReferenceInputError('RESULT_CORRUPT', 'Report trace format is unavailable') + const data = parsed.data + if (data.protocol.mode !== 'stage_replay' && !(data.protocol.mode === 'signal_replay' && data.protocol.waveformStages)) throw new ReferenceInputError('INCOMPATIBLE_METRIC', 'This comparison did not produce stage replay traces') + const nights = input.cohortId === undefined ? data.nights : data.nights.filter(night => night.cohortId === input.cohortId) + if (!nights.length || !nights.some(night => night.evaluations.some(evaluation => evaluation.label === input.configuration))) { + throw new ReferenceInputError('NOT_FOUND', 'Requested cohort or configuration was not evaluated') + } + const items: z.infer['items'] = [] + let totalItems = 0 + for (const night of nights) { + for (const evaluation of night.evaluations.filter(row => row.label === input.configuration)) { + for (const window of evaluation.windows) { + for (const block of window.blocks) { + if (totalItems >= input.offset && items.length < input.limit) items.push({ cohortId: night.cohortId, configurationHash: evaluation.configHash, + revisionId: evaluation.revisionId, windowStartMs: window.startMs, windowEndMs: window.endMs, + startMs: block.start, endMs: block.end, stage: block.stage }) + totalItems++ + } + } + } + } + if (input.offset > totalItems) throw new ReferenceInputError('INVALID_CURSOR', 'Trace cursor exceeds the selected report') + const next = input.offset + items.length + return tracePageSchema.parse({ jobId: input.id, contentHash: input.expectedContentHash, configuration: input.configuration, + provenance: 'versioned_stage_replay', timestampBasis: 'original_trace_before_reference_clock_offset', clockOffsetMs: data.protocol.clockOffsetMs, + items, totalItems, offset: input.offset, limit: input.limit, downsampled: false, + nextCursor: next < totalItems ? { offset: next, contentHash: input.expectedContentHash } : null }) +} + diff --git a/src/biometrics/backtest/traces.ts b/src/biometrics/backtest/traces.ts new file mode 100644 index 00000000..0b5be7b0 --- /dev/null +++ b/src/biometrics/backtest/traces.ts @@ -0,0 +1,40 @@ +import { DatasetStore } from './datasets' +import { pageReferenceTraces, referenceTraceIdentity, referenceTraceRequestSchema } from './reference-trace-page' +import { pageNumericTraces } from './numeric-trace-page' +import type { z } from 'zod' +import { pageStageTraces } from './trace-page' +export { pageStageTraces } from './trace-page' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import type * as schema from '@/src/db/biometrics-schema' +import { ReferenceInputError } from '../references/time' +import { BacktestJobStore } from './jobs' +import { traceRequestSchema } from './trace-contracts' + +export function readStageTraces(db: BetterSQLite3Database, raw: z.input) { + const input = traceRequestSchema.parse(raw) + const result = new BacktestJobStore(db).result(input.id) + if (!result) throw new ReferenceInputError('INVALID_STATE', 'Wait for a successful replay before reading traces') + if (result.contentHash !== input.expectedContentHash) throw new ReferenceInputError('REVISION_CONFLICT', 'Trace report hash changed') + if (!result.available) throw new ReferenceInputError('RESULT_RELEASED', 'Replay report was released') + return pageStageTraces(result.report, input) +} + +export function readNumericTraces(db: BetterSQLite3Database, raw: z.input) { + const input = traceRequestSchema.parse(raw) + const result = new BacktestJobStore(db).result(input.id) + if (!result) throw new ReferenceInputError('INVALID_STATE', 'Wait for a successful replay before reading traces') + if (result.contentHash !== input.expectedContentHash) throw new ReferenceInputError('REVISION_CONFLICT', 'Trace report hash changed') + if (!result.available) throw new ReferenceInputError('RESULT_RELEASED', 'Replay report was released') + return pageNumericTraces(result.report, input) +} + +export function readReferenceTraces(db: BetterSQLite3Database, raw: z.input) { + const input = referenceTraceRequestSchema.parse(raw) + const result = new BacktestJobStore(db).result(input.id) + if (!result) throw new ReferenceInputError('INVALID_STATE', 'Wait for a successful replay before reading references') + if (result.contentHash !== input.expectedContentHash) throw new ReferenceInputError('REVISION_CONFLICT', 'Reference report hash changed') + if (!result.available) throw new ReferenceInputError('RESULT_RELEASED', 'Replay report was released') + const identity = referenceTraceIdentity(result.report) + const dataset = new DatasetStore(db).read(identity.datasetId) + return pageReferenceTraces(result.report, dataset, input) +} diff --git a/src/biometrics/backtest/uncertainty.ts b/src/biometrics/backtest/uncertainty.ts new file mode 100644 index 00000000..48182d82 --- /dev/null +++ b/src/biometrics/backtest/uncertainty.ts @@ -0,0 +1,4 @@ +import { createHash } from 'node:crypto' +import { createNightUncertainty } from './uncertainty-core' +export type { NightBootstrapPolicy } from './uncertainty-policy' +export const nightUncertainty = createNightUncertainty(text => createHash('sha256').update(text).digest()) diff --git a/src/biometrics/backtest/waveform-dataset.ts b/src/biometrics/backtest/waveform-dataset.ts new file mode 100644 index 00000000..e0bf924e --- /dev/null +++ b/src/biometrics/backtest/waveform-dataset.ts @@ -0,0 +1,45 @@ +import { randomUUID } from 'node:crypto' +import { z } from 'zod' +import { configHash, dataHash } from '../algorithms/canonical' +import { datasetManifestSchema, datasetSnapshotSchema } from './contracts' +import { waveformDatasetInputsSchema } from './waveform-contracts' + +const sourceSchema = z.object({ id: z.string().uuid(), contentHash: z.string().regex(/^[a-f0-9]{64}$/), + manifest: datasetManifestSchema, snapshot: datasetSnapshotSchema }) + +/** Server-internal preparation. Sources and context must come from retained evidence. */ +export function prepareWaveformDataset(source: z.input, rawInputs: unknown, idempotencyKey: string, now = Date.now()) { + const base = sourceSchema.parse(structuredClone(source)) + if (base.snapshot.version !== 'processed-dataset-v1' || base.manifest.version !== 'processed-dataset-manifest-v1') throw new Error('Select a processed source dataset') + if (dataHash({ manifest: base.manifest, snapshot: base.snapshot }) !== base.contentHash) throw new Error('Source dataset integrity mismatch') + z.string().min(1).max(200).parse(idempotencyKey) + z.number().int().min(0).max(Number.MAX_SAFE_INTEGER).parse(now) + const inputs = waveformDatasetInputsSchema.parse(rawInputs) + const cohorts = new Map(base.manifest.splitProtocol.assignments.map(item => [item.cohortId, item])) + if (cohorts.size !== base.manifest.splitProtocol.assignments.length || inputs.length !== cohorts.size) throw new Error('Retain waveform inputs for every whole-night cohort') + for (const item of inputs) { + const cohort = cohorts.get(item.cohortId) + if (!cohort || item.input.score_start_ms !== cohort.startMs || item.input.score_end_ms !== cohort.endMs) throw new Error('Waveform input must match its whole-night cohort') + } + // Follow the immutable split assignment order, independent of selection UI order. + const order = [...cohorts.keys()] + inputs.sort((a, b) => order.indexOf(a.cohortId) - order.indexOf(b.cohortId)) + const snapshot = datasetSnapshotSchema.parse({ ...base.snapshot, version: 'waveform-dataset-v1', waveformInputs: inputs }) + const manifest = datasetManifestSchema.parse({ ...base.manifest, version: 'waveform-dataset-manifest-v1', + parentDataset: { id: base.id, contentHash: base.contentHash }, + capabilities: { ...base.manifest.capabilities, signalReplay: true }, + limitations: ['Waveform files and supplied context intervals are frozen; execution requires configured deployment admission and current resource health.', + 'Event bounds do not establish continuous waveform coverage; gaps in historical context remain unknown.', + 'Replay uses event time and a cold start with retained pre-roll, not serialized historical processor state.', + 'Measured calibration is not reconstructed from current calibration settings.', + 'Historical stage replay must declare its calibration quality assumption when original calibration is unavailable.', + 'Reference deletion does not erase this separately retained dataset.', + 'Historical occupant identity is asserted by the dataset creator, not measured by the pod.'], + }) + const contentHash = dataHash({ manifest, snapshot }) + const prepared = { id: randomUUID(), side: snapshot.side, subjectId: snapshot.subjectId, + idempotencyKey, requestHash: configHash({ version: 'waveform-dataset-request-v1', parentId: base.id, + parentHash: base.contentHash, inputsHash: dataHash(inputs), idempotencyKey }), contentHash, manifest, snapshot, createdAtMs: now } + if (Buffer.byteLength(JSON.stringify(prepared)) > 8 * 1024 * 1024) throw new Error('Waveform dataset exceeds its byte budget') + return prepared +} diff --git a/src/biometrics/backtest/waveform-execution.ts b/src/biometrics/backtest/waveform-execution.ts new file mode 100644 index 00000000..9cf09ea6 --- /dev/null +++ b/src/biometrics/backtest/waveform-execution.ts @@ -0,0 +1,63 @@ +import type { ReplayBudget } from './execution-budget' +import { sql } from 'drizzle-orm' +import { dirname, join } from 'node:path' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import type { z } from 'zod' +import type * as schema from '@/src/db/biometrics-schema' +import { dataHash } from '../algorithms/canonical' +import type { backtestWorkSchema } from './evaluator' +import { waveformEvaluationSchema } from './evaluator' +import { datasetManifestSchema } from './contracts' +import { podRawInspectionLocation, type RawInspectionLocation } from './raw-inspection-child' +import { waveformEvaluatorBuildIdentity } from './waveform-build' +import { runWaveformChild } from './waveform-child' + +export async function verifyWaveformJobBuild(work: z.infer, options: { + workerPath: string + rawLocation?: RawInspectionLocation + signal: AbortSignal +}) { + const build = await waveformEvaluatorBuildIdentity(join(dirname(options.workerPath), 'coordinator.cjs'), options.workerPath, + options.rawLocation ?? podRawInspectionLocation, options.signal) + if (build.hash !== work.job.evaluatorBuildHash || build.python.hash !== work.job.waveform?.pythonBuildHash) throw new Error('Waveform worker build changed since enqueue') +} + +/** Run one cohort/configuration at a time. Raw bytes stay in Python; only + * bounded derived rows reach the isolated Node scorer and retained report. */ +export async function executeWaveformInputs(db: BetterSQLite3Database, work: z.infer, options: { + workerPath: string + rawLocation?: RawInspectionLocation + signal: AbortSignal + timeoutMs?: number + replayBudget?: ReplayBudget +}) { + const { job, dataset } = work + if (job.protocol.mode !== 'signal_replay' || !job.waveform || dataset.snapshot.version !== 'waveform-dataset-v1') throw new Error('Missing retained waveform job inputs') + const databasePath = db.all<{ name: string, file: string }>(sql`PRAGMA database_list`).find(row => row.name === 'main')?.file + if (!databasePath) throw new Error('Waveform execution requires a persistent database') + const location = { ...(options.rawLocation ?? podRawInspectionLocation), retainedDirectory: join(dirname(databasePath), 'biometric-raw') } + await verifyWaveformJobBuild(work, options) + const assignments = datasetManifestSchema.parse(dataset.manifest).splitProtocol.assignments.filter(row => row.split === job.split) + const configurations = [...new Map(job.configs.map(config => [config.resolvedHash, config])).values()] + const outputs: z.infer['outputs'] = [] + let count = 0 + for (const cohort of assignments) { + const input = dataset.snapshot.waveformInputs.find(row => row.cohortId === cohort.cohortId)?.input + if (!input) throw new Error('Missing retained waveform cohort') + for (const config of configurations) { + if (options.signal.aborted) throw new Error('Waveform job cancelled') + const configs = dataset.snapshot.side === 'left' + ? { left: config.resolved, right: job.waveform.otherSide.resolved } + : { left: job.waveform.otherSide.resolved, right: config.resolved } + const output = await runWaveformChild(location, { version: 'retained-waveform-work-v1', input, configs }, { + signal: options.signal, timeoutMs: options.replayBudget?.waveformWallMs ?? options.timeoutMs, + cpuSeconds: options.replayBudget?.waveformCpuSeconds, expectedBuildHash: job.waveform.pythonBuildHash, + }) + count += output.readings.length + if (count > 20000) throw new Error('Waveform job derived reading budget exceeded') + outputs.push({ cohortId: cohort.cohortId, configHash: config.resolvedHash, ...output }) + } + } + await verifyWaveformJobBuild(work, options) + return waveformEvaluationSchema.parse({ hash: dataHash(outputs), outputs }) +} diff --git a/src/biometrics/backtest/waveform-selection.ts b/src/biometrics/backtest/waveform-selection.ts new file mode 100644 index 00000000..133d20d7 --- /dev/null +++ b/src/biometrics/backtest/waveform-selection.ts @@ -0,0 +1,131 @@ +import { z } from 'zod' +import type { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' +import { eq } from 'drizzle-orm' +import { configHash } from '../algorithms/canonical' +import * as schema from '@/src/db/biometrics-schema' +import { ReferenceInputError } from '../references/time' +import { DatasetStore } from './datasets' +import { datasetManifestSchema, type DatasetSnapshot } from './contracts' +import { RawSourceCatalog } from './raw-catalog' +import { prepareWaveformDataset } from './waveform-dataset' + +const hash = z.string().regex(/^[a-f0-9]{64}$/) +export const createWaveformDatasetSchema = z.object({ + parentDatasetId: z.string().uuid(), expectedContentHash: hash, + cohorts: z.array(z.object({ cohortId: z.string().min(1).max(200), + sourceIds: z.array(hash).min(1).max(64), + }).strict()).min(1).max(60), + idempotencyKey: z.string().min(1).max(200), +}).strict() + +/** Retry identity depends on the requested immutable IDs, not mutable catalog availability. */ +export function waveformSelectionRequestHash(raw: z.input) { + const input = createWaveformDatasetSchema.parse(raw) + const cohorts = input.cohorts.map(item => ({ ...item, sourceIds: [...item.sourceIds].sort() })) + .sort((a, b) => a.cohortId < b.cohortId ? -1 : a.cohortId > b.cohortId ? 1 : 0) + return configHash({ version: 'selected-waveform-dataset-request-v1', ...input, cohorts }) +} + +export function selectedWaveformRetry(db: BetterSQLite3Database, raw: z.input) { + const input = createWaveformDatasetSchema.parse(raw) + const row = db.select().from(schema.biometricDatasets).where(eq(schema.biometricDatasets.idempotencyKey, input.idempotencyKey)).get() + if (!row) return null + if (row.requestHash !== waveformSelectionRequestHash(input)) throw new ReferenceInputError('REVISION_CONFLICT', 'Dataset idempotency key was already used for a different selection') + return new DatasetStore(db).descriptor(row.id) +} + +/** Only intervals actually recorded in the frozen parent are evidence. Session + * boundaries are not evidence of continuous occupancy, nor of sleeper mode. */ +export function recordedWaveformContexts(snapshot: DatasetSnapshot, startMs: number, endMs: number) { + const events = new Map() + let count = 0 + for (const session of snapshot.sessions) { + for (const [raw, state] of [[session.presentIntervals, 0], [session.notPresentIntervals, 1]] as const) { + if (!Array.isArray(raw)) continue + for (const item of raw) { + if (++count > 20000) throw new ReferenceInputError('RESOURCE_LIMIT', 'Recorded occupancy interval budget exceeded') + // The sleep detector stores epoch seconds (including fractional seconds). + if (!Array.isArray(item) || item.length !== 2 || item.some(value => typeof value !== 'number' || !Number.isFinite(value) || value < 0)) continue + const first = Math.round(item[0] * 1000), last = Math.round(item[1] * 1000) + if (!Number.isSafeInteger(first) || !Number.isSafeInteger(last) || first >= last) continue + const start = Math.max(first, startMs, session.startMs), end = Math.min(last, endMs, session.endMs) + if (start >= end) continue + for (const [time, delta] of [[start, 1], [end, -1]]) { + const event = events.get(time) ?? [0, 0] + event[state] += delta + events.set(time, event) + } + } + } + } + const times = [...events.keys()].sort((a, b) => a - b) + let present = 0, absent = 0 + const contexts: Array<{ + start_ms: number + end_ms: number + active_sides: null + left_occupied: boolean | null + right_occupied: boolean | null + pump_context_known: false + }> = [] + for (let index = 0; index < times.length - 1; index++) { + const [yes, no] = events.get(times[index]) ?? [0, 0] + present += yes + absent += no + // Conflicting evidence and unrecorded gaps both remain unknown. + if ((present > 0) === (absent > 0)) continue + const occupied = present > 0 + const last = contexts.at(-1) + if (last && last.end_ms === times[index] && last[`${snapshot.side}_occupied`] === occupied) last.end_ms = times[index + 1] + else contexts.push({ start_ms: times[index], end_ms: times[index + 1], active_sides: null, + left_occupied: snapshot.side === 'left' ? occupied : null, + right_occupied: snapshot.side === 'right' ? occupied : null, pump_context_known: false }) + } + if (contexts.length > 20000) throw new ReferenceInputError('RESOURCE_LIMIT', 'Recorded occupancy context budget exceeded') + return contexts +} + +/** Resolve IDs to verified catalog evidence and frozen historical context. No + * archive reads or live configuration queries occur while the DB snapshot is held. */ +export function prepareSelectedWaveformDataset(db: BetterSQLite3Database, raw: z.input, now = Date.now()) { + const input = createWaveformDatasetSchema.parse(raw) + if (new Set(input.cohorts.map(item => item.cohortId)).size !== input.cohorts.length) throw new ReferenceInputError('DUPLICATE_SELECTION', 'Select each whole-night cohort once') + return db.transaction(() => { + const parent = new DatasetStore(db).read(input.parentDatasetId) + if (parent.contentHash !== input.expectedContentHash) throw new ReferenceInputError('DATASET_CHANGED', 'Select the expected immutable parent dataset') + const manifest = datasetManifestSchema.parse(parent.manifest) + if (input.cohorts.length !== manifest.splitProtocol.assignments.length) throw new ReferenceInputError('INSUFFICIENT_RAW_DATA', 'Select RAW inputs for every whole-night cohort') + const catalog = new RawSourceCatalog(db) + const selections = new Map['inspection']['artifact'] }>() + const inputs = manifest.splitProtocol.assignments.map((cohort) => { + const selection = input.cohorts.find(item => item.cohortId === cohort.cohortId) + if (!selection) throw new ReferenceInputError('INSUFFICIENT_RAW_DATA', 'Missing whole-night RAW selection') + if (new Set(selection.sourceIds).size !== selection.sourceIds.length) throw new ReferenceInputError('DUPLICATE_SELECTION', 'Select each RAW source once per cohort') + const artifacts = selection.sourceIds.map((id) => { + const source = catalog.read(id) + const { inventory, artifact, source_name } = source.inspection + const cached = source.unavailableAtMs === null + ? undefined + : db.select().from(schema.biometricRawArtifacts) + .where(eq(schema.biometricRawArtifacts.contentHash, artifact.sha256)).get() + const mayReuse = cached?.name === artifact.name && cached.compressedBytes === artifact.compressed_bytes + && cached.expandedBytes === artifact.expanded_bytes + // Metadata only permits attempting reuse; Python still verifies the complete retained file. + if ((source.unavailableAtMs !== null && !mayReuse) || !inventory.ordered_waveform_available + || inventory.first_event_ms === null || inventory.last_event_ms === null) { + throw new ReferenceInputError('INSUFFICIENT_RAW_DATA', 'Selected RAW source is unavailable or cannot provide ordered waveforms') + } + const previous = selections.get(artifact.sha256) + if (!previous || source_name < previous.source_name) selections.set(artifact.sha256, { source_name, artifact }) + return { artifact, first_event_ms: inventory.first_event_ms, last_event_ms: inventory.last_event_ms } + }).sort((a, b) => a.first_event_ms - b.first_event_ms || a.artifact.sha256.localeCompare(b.artifact.sha256)) + const start = Math.max(0, cohort.startMs - 300000) + return { cohortId: cohort.cohortId, input: { version: 'retained-waveform-input-v1' as const, + retained_start_ms: start, score_start_ms: cohort.startMs, score_end_ms: cohort.endMs, + artifacts, contexts: recordedWaveformContexts(parent.snapshot, start, cohort.endMs) } } + }) + const prepared = prepareWaveformDataset({ ...parent, manifest }, inputs, input.idempotencyKey, now) + prepared.requestHash = waveformSelectionRequestHash(input) + return { prepared, selections: [...selections.values()].sort((a, b) => a.artifact.sha256.localeCompare(b.artifact.sha256)) } + }) +} diff --git a/src/biometrics/packages/README.md b/src/biometrics/packages/README.md new file mode 100644 index 00000000..e387e914 --- /dev/null +++ b/src/biometrics/packages/README.md @@ -0,0 +1,71 @@ +# Algorithm package proposals + +`manifest.ts` validates inert proposals for the algorithm-development workflow. +It does not import, execute, install, or approve their code. No API currently +accepts these proposals. The [off-pod experiment workflow](../../../scripts/algorithm-experiments/README.md) +provides Docker execution, deterministic/noise checks, managed whole-night +comparisons, resource assessments and combined external qualification evidence. +That evidence cannot install code. Promotion requires representative-fixture +review, matching target-pod and prospective evidence, and a reviewed release +into the shipped registry. + +The versioned manifest declares engine identity, frame schema identifiers, +metric definitions and units, supported hardware/sample rates, causal or +retrospective behavior, deterministic initialization and reset, JSON state +serialization, resource requests, an entrypoint, and a dependency lock. Every +file is listed with its exact byte length and SHA-256 digest. Package identity +is SHA-256 over the complete manifest using `finite-binary64-json-v1` canonical +JSON. Artifact array order is part of that identity. + +`verifyAlgorithmPackage` takes a map of bytes, avoiding filesystem resolution +or archive extraction. It rejects unexpected files, missing files, mismatched +sizes/hashes, traversal paths, duplicate identities, and oversized proposals. +The 32 MiB intake ceiling is a storage bound, not a qualified runtime budget. +A future transport must enforce size limits before buffering. Intake verifies +only the dependency-lock artifact identity. The external +executor additionally requires an exact Python environment lock and independently +probes its reviewed runtime image; see the executor README for that format. + +Execution must use an external isolation boundary without pod credentials, +network access by default, writable input, or access to live sinks. Resource +requests must be checked against server-owned limits and enforced outside the +candidate process. Deterministic replay, artifact/noise fixtures, resource +measurements, held-out comparisons, and compatibility checks remain required. +An experiment result never grants installation rights: promotion goes through +code review and a release into the shipped registry. + +## Streaming frame contract + +`protocol.ts` defines the host-side exchange. A run names the exact package and +input hashes, hardware, sample rate, side, seed, and finite JSON configuration. +Its canonical hash binds all those choices. Create a fresh validator and reset +candidate state for each independent run; never carry state between nights or +qualification repetitions unless a future protocol explicitly declares it. + +Frames carry increasing sequence numbers from zero, explicit sample-start +timestamps, one or both channels of at most 4096 finite samples, and explicit +nullable occupancy/pump context. Missing context is unknown, not false. +Channels must agree in length. Sample duration determines the next permissible +frame time; gaps are permitted, overlaps are rejected. The executor must call +`validateFrame` before exposing each frame to the candidate. + +A response echoes the run hash, canonical input-frame hash, and sequence. It +contains bounded numeric/null metric readings and JSON state. The host checks +metric definitions, side, ranges, strictly increasing timestamps per metric, +observed time bounds, declared retrospective lookahead, state bytes, and the +cumulative canonical response-byte budget. Validation failure does not advance +the run. No response can authorize database or hardware writes. + +The external executor must bound transport bytes before JSON decoding (including +whitespace), enforce process resources independently, and expose only the current +frame and previously returned state. The validator cannot prove that code obeys +causality, initialization, reset, or determinism; those need the executor and +repeated qualification runs. This protocol is not itself an isolation boundary. + +A metric can optionally declare a structured `reference` contract for external +comparison: `metric` is one of `heart_rate`, `breathing_rate`, +`window_ibi_variability_v1`, `hrv_rmssd`, or `hrv_sdnn`; `windowMs` is required and +may be null only for non-HRV metrics. The declaration requires canonical units +and participates in package identity. Metrics without this declaration remain +valid experimental outputs but cannot enter reference comparison based on their +name or prose definition alone. diff --git a/src/biometrics/packages/manifest.ts b/src/biometrics/packages/manifest.ts new file mode 100644 index 00000000..50c173b8 --- /dev/null +++ b/src/biometrics/packages/manifest.ts @@ -0,0 +1,75 @@ +import { createHash } from 'node:crypto' +import { z } from 'zod' +import { configHash } from '../algorithms/canonical' + +const hash = z.string().regex(/^[a-f0-9]{64}$/) +const identifier = z.string().regex(/^[a-z][a-z0-9_.-]{0,99}$/) +const positiveInteger = z.number().int().positive().max(Number.MAX_SAFE_INTEGER) +const relativePath = z.string().min(1).max(240).refine(path => + path.split('/').every(part => /^[A-Za-z0-9_][A-Za-z0-9_.-]*$/.test(part)), 'Expected a relative package file path') +const artifactSchema = z.object({ path: relativePath, sha256: hash, bytes: positiveInteger }).strict() +const referenceContractSchema = z.object({ + metric: z.enum(['heart_rate', 'breathing_rate', 'window_ibi_variability_v1', 'hrv_rmssd', 'hrv_sdnn']), + windowMs: z.number().int().positive().max(86400000).nullable(), +}).strict().refine(value => !['window_ibi_variability_v1', 'hrv_rmssd', 'hrv_sdnn'].includes(value.metric) || value.windowMs !== null, 'HRV comparison requires an explicit window') +const metricSchema = z.object({ + id: identifier, definition: z.string().min(1).max(2000), unit: z.string().min(1).max(80), + minimum: z.number().finite(), maximum: z.number().finite(), + reference: referenceContractSchema.optional(), +}).strict().refine(metric => metric.minimum < metric.maximum, 'Metric bounds must be ordered') + .refine(metric => !metric.reference || metric.unit === (metric.reference.metric === 'heart_rate' ? 'bpm' : metric.reference.metric === 'breathing_rate' ? 'breaths/min' : 'ms'), 'Reference metric requires canonical units') + +/** Inert proposal metadata. Passing this schema grants no execution or installation rights. */ +export const algorithmPackageManifestSchema = z.object({ + schema: z.literal('biometric-algorithm-package-v1'), + canonicalization: z.literal('finite-binary64-json-v1'), + engine: z.object({ id: identifier, version: z.string().regex(/^\d+\.\d+\.\d+$/) }).strict(), + runtime: z.literal('python-frame-v1'), entrypoint: relativePath, + inputSchema: z.literal('biometric-signal-frame-v1'), outputSchema: z.literal('biometric-metric-frame-v1'), + metrics: z.array(metricSchema).min(1).max(16), + hardware: z.array(z.object({ id: identifier, sampleRatesHz: z.array(positiveInteger).min(1).max(16) }).strict()).min(1).max(16), + temporal: z.discriminatedUnion('mode', [ + z.object({ mode: z.literal('causal'), lookaheadMs: z.literal(0) }).strict(), + z.object({ mode: z.literal('retrospective'), lookaheadMs: positiveInteger }).strict(), + ]), + state: z.object({ + schemaId: identifier, initialization: z.literal('explicit_seed_and_config'), + reset: z.literal('reset_before_each_independent_run'), serialization: z.literal('json'), maximumBytes: positiveInteger, + }).strict(), + resources: z.object({ + cpuMs: positiveInteger, wallMs: positiveInteger, memoryBytes: positiveInteger, + processes: positiveInteger, outputBytes: positiveInteger, + }).strict(), + dependencyLock: relativePath, + artifacts: z.array(artifactSchema).min(2).max(128), +}).strict().superRefine((value, context) => { + const unique = (values: (string | number)[], label: string) => { + if (new Set(values).size !== values.length) context.addIssue({ code: 'custom', message: `Duplicate ${label}` }) + } + unique(value.metrics.map(metric => metric.id), 'metric') + unique(value.hardware.map(hardware => hardware.id), 'hardware') + for (const hardware of value.hardware) unique(hardware.sampleRatesHz, 'sample rate') + unique(value.artifacts.map(artifact => artifact.path), 'artifact path') + for (const path of [value.entrypoint, value.dependencyLock]) { + if (!value.artifacts.some(artifact => artifact.path === path)) context.addIssue({ code: 'custom', message: `Unhashed package file: ${path}` }) + } + if (value.entrypoint === value.dependencyLock) context.addIssue({ code: 'custom', message: 'Entrypoint and dependency lock must be separate files' }) +}) +export type AlgorithmPackageManifest = z.infer + +/** Bounded byte inputs, deliberately with no filesystem paths, imports, or subprocesses. + * Callers must bound transport size before buffering. These limits are proposal intake + * limits, not evidence that an algorithm fits the pod's runtime budget. */ +export const maximumPackageBytes = 32 * 1024 * 1024 +export function verifyAlgorithmPackage(manifestInput: unknown, files: ReadonlyMap) { + const manifest = algorithmPackageManifestSchema.parse(manifestInput) + if (files.size !== manifest.artifacts.length) throw new Error('Package file set differs from manifest') + const totalBytes = manifest.artifacts.reduce((sum, artifact) => sum + artifact.bytes, 0) + if (!Number.isSafeInteger(totalBytes) || totalBytes > maximumPackageBytes) throw new Error('Package exceeds intake byte limit') + for (const artifact of manifest.artifacts) { + const bytes = files.get(artifact.path) + if (!bytes || bytes.byteLength !== artifact.bytes) throw new Error(`Package file size mismatch: ${artifact.path}`) + if (createHash('sha256').update(bytes).digest('hex') !== artifact.sha256) throw new Error(`Package file hash mismatch: ${artifact.path}`) + } + return { manifest, packageHash: configHash(manifest), totalBytes, status: 'inert_proposal' as const } +} diff --git a/src/biometrics/packages/protocol.ts b/src/biometrics/packages/protocol.ts new file mode 100644 index 00000000..13ac10e6 --- /dev/null +++ b/src/biometrics/packages/protocol.ts @@ -0,0 +1,92 @@ +import { z } from 'zod' +import { canonicalJson, configHash, dataHash } from '../algorithms/canonical' +import { algorithmPackageManifestSchema } from './manifest' + +const hash = z.string().regex(/^[a-f0-9]{64}$/) +const time = z.number().finite().nonnegative().max(Number.MAX_SAFE_INTEGER) +const integer = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) +const side = z.enum(['left', 'right']) +const samples = z.array(z.number().finite()).min(1).max(4096) +export const packageRunRequestSchema = z.object({ + schema: z.literal('biometric-package-run-v1'), packageHash: hash, inputHash: hash, + hardwareId: z.string().min(1).max(100), sampleRateHz: z.number().int().positive(), side, + seed: integer, config: z.record(z.string(), z.unknown()), +}).strict() +export const signalFrameSchema = z.object({ + schema: z.literal('biometric-signal-frame-v1'), sequence: integer, timestampMs: time, + samples: z.object({ left: samples.nullable(), right: samples.nullable() }).strict(), + context: z.object({ occupied: z.boolean().nullable(), pumpActive: z.boolean().nullable() }).strict(), +}).strict().refine(frame => frame.samples.left !== null || frame.samples.right !== null, 'Frame must contain samples') + .refine(frame => !frame.samples.left || !frame.samples.right || frame.samples.left.length === frame.samples.right.length, 'Channel lengths differ') +export const metricFrameSchema = z.object({ + schema: z.literal('biometric-metric-frame-v1'), runHash: hash, frameHash: hash, sequence: integer, + readings: z.array(z.object({ + timestampMs: time, side, metric: z.string().min(1).max(100), value: z.number().finite().nullable(), + quality: z.number().min(0).max(1), flags: z.array(z.string().min(1).max(100)).max(32), + }).strict()).max(256), + state: z.unknown(), +}).strict() + +/** Host-side validation of a streaming exchange. This class never runs a package. + * One validator belongs to one independent run and starts at sequence zero. + * The executor must send only the current frame, after resetting package state. */ +export class PackageFrameValidator { + readonly manifest + readonly request + readonly runHash: string + private nextSequence = 0 + private outputBytes = 0 + private nextInputTime = 0 + private firstInputTime: number | null = null + private lastReadingTimes = new Map() + + constructor(manifestInput: unknown, requestInput: unknown) { + this.manifest = algorithmPackageManifestSchema.parse(manifestInput) + this.request = packageRunRequestSchema.parse(requestInput) + if (this.request.packageHash !== configHash(this.manifest)) throw new Error('Package identity mismatch') + const hardware = this.manifest.hardware.find(item => item.id === this.request.hardwareId) + if (!hardware?.sampleRatesHz.includes(this.request.sampleRateHz)) throw new Error('Unsupported hardware or sample rate') + // Canonical hashing also rejects non-JSON config values and bounds complexity. + this.runHash = configHash(this.request) + } + + validateFrame(frameInput: unknown) { + const frame = signalFrameSchema.parse(frameInput) + if (frame.sequence !== this.nextSequence || frame.timestampMs < this.nextInputTime) throw new Error('Out-of-order or overlapping frame') + const channel = frame.samples[this.request.side] + if (!channel) throw new Error('Requested side has no samples') + const endMs = frame.timestampMs + channel.length * 1000 / this.request.sampleRateHz + if (!Number.isFinite(endMs) || endMs > Number.MAX_SAFE_INTEGER) throw new Error('Frame end exceeds timestamp range') + return { frame, endMs } + } + + validateStep(frameInput: unknown, resultInput: unknown) { + const { frame, endMs } = this.validateFrame(frameInput) + const result = metricFrameSchema.parse(resultInput) + if (result.runHash !== this.runHash || result.frameHash !== dataHash(frame) || result.sequence !== frame.sequence) throw new Error('Output identity mismatch') + // A transport must additionally cap stdout before buffering/parsing it. + const outputBytes = this.outputBytes + Buffer.byteLength(canonicalJson(result)) + if (outputBytes > this.manifest.resources.outputBytes) throw new Error('Output byte limit exceeded') + if (Buffer.byteLength(canonicalJson(result.state)) > this.manifest.state.maximumBytes) throw new Error('State byte limit exceeded') + const firstInputTime = this.firstInputTime ?? frame.timestampMs + const lastTimes = new Map(this.lastReadingTimes) + for (const reading of result.readings) { + const metric = this.manifest.metrics.find(item => item.id === reading.metric) + if (!metric || reading.side !== this.request.side) throw new Error('Undeclared metric or side') + if (reading.value !== null && (reading.value < metric.minimum || reading.value > metric.maximum)) throw new Error('Metric outside declared range') + // A retrospective reading may lag the latest available sample by its + // declared lookahead. No output may refer to unavailable future samples. + const latestSampleMs = endMs - 1000 / this.request.sampleRateHz + if (reading.timestampMs < firstInputTime || reading.timestampMs + this.manifest.temporal.lookaheadMs > latestSampleMs) throw new Error('Reading outside observed time range') + if (reading.timestampMs <= (lastTimes.get(reading.metric) ?? -1)) throw new Error('Duplicate or out-of-order reading') + lastTimes.set(reading.metric, reading.timestampMs) + } + // A rejected response leaves the sequence and watermarks unchanged. + this.outputBytes = outputBytes + this.nextSequence++ + this.nextInputTime = endMs + this.firstInputTime = firstInputTime + this.lastReadingTimes = lastTimes + return { frame, result, endMs } + } +} diff --git a/src/biometrics/packages/tests/fixture.ts b/src/biometrics/packages/tests/fixture.ts new file mode 100644 index 00000000..41affcbe --- /dev/null +++ b/src/biometrics/packages/tests/fixture.ts @@ -0,0 +1,17 @@ +import { createHash } from 'node:crypto' + +export function packageFixture() { + const files = new Map([['engine.py', Buffer.from('raise RuntimeError("must never execute during intake")')], ['uv.lock', Buffer.from('version = 1')]]) + const manifest = { + schema: 'biometric-algorithm-package-v1', canonicalization: 'finite-binary64-json-v1', engine: { id: 'test.hr', version: '1.0.0' }, + runtime: 'python-frame-v1', entrypoint: 'engine.py', dependencyLock: 'uv.lock', + inputSchema: 'biometric-signal-frame-v1', outputSchema: 'biometric-metric-frame-v1', + metrics: [{ id: 'heart_rate', definition: 'Heart beats per minute', unit: 'bpm', minimum: 20, maximum: 240 }], + hardware: [{ id: 'pod4', sampleRatesHz: [500] }], temporal: { mode: 'causal', lookaheadMs: 0 }, + state: { schemaId: 'test.hr.state.v1', initialization: 'explicit_seed_and_config', reset: 'reset_before_each_independent_run', serialization: 'json', maximumBytes: 10000 }, + resources: { cpuMs: 1000, wallMs: 2000, memoryBytes: 100000000, processes: 1, outputBytes: 10000 }, + artifacts: [...files].map(([path, bytes]) => ({ path, bytes: bytes.byteLength, sha256: createHash('sha256').update(bytes).digest('hex') })), + } + return { files, manifest } +} + diff --git a/src/biometrics/packages/tests/manifest.test.ts b/src/biometrics/packages/tests/manifest.test.ts new file mode 100644 index 00000000..2b39ec74 --- /dev/null +++ b/src/biometrics/packages/tests/manifest.test.ts @@ -0,0 +1,66 @@ +import { packageFixture as fixture } from './fixture' +import { describe, expect, it } from 'vitest' +import { algorithmPackageManifestSchema, maximumPackageBytes, verifyAlgorithmPackage } from '../manifest' + +describe('inert algorithm package intake', () => { + it('verifies executable bytes without executing them and binds all declarations to the package identity', () => { + const { files, manifest } = fixture() + const original = verifyAlgorithmPackage(manifest, files) + expect(original.status).toBe('inert_proposal') + expect(verifyAlgorithmPackage(JSON.parse(JSON.stringify(manifest)), files).packageHash).toBe(original.packageHash) + manifest.resources.cpuMs++ + expect(verifyAlgorithmPackage(manifest, files).packageHash).not.toBe(original.packageHash) + }) + it('rejects substituted code and locks even when the file size is unchanged', () => { + for (const path of ['engine.py', 'uv.lock']) { + const { files, manifest } = fixture() + const original = files.get(path) + if (!original) throw new Error('Missing fixture file') + files.set(path, Buffer.alloc(original.length, 65)) + expect(() => verifyAlgorithmPackage(manifest, files)).toThrow('hash mismatch') + } + }) + it('rejects missing, extra, truncated and oversized files', () => { + const { files, manifest } = fixture() + files.set('extra.py', Buffer.from('pass')) + expect(() => verifyAlgorithmPackage(manifest, files)).toThrow('file set') + files.delete('extra.py') + files.set('engine.py', Buffer.from('pass')) + expect(() => verifyAlgorithmPackage(manifest, files)).toThrow('size mismatch') + files.delete('engine.py') + expect(() => verifyAlgorithmPackage(manifest, files)).toThrow('file set') + manifest.artifacts[0].bytes = maximumPackageBytes + expect(() => verifyAlgorithmPackage(manifest, fixture().files)).toThrow('byte limit') + }) + it.each(['../engine.py', '/engine.py', 'a/../../engine.py', 'a\\engine.py', 'a//engine.py', './engine.py', 'a/engine.py\u0000'])('rejects unsafe artifact path %s', (path) => { + const { manifest } = fixture() + manifest.artifacts[0].path = path + manifest.entrypoint = path + expect(algorithmPackageManifestSchema.safeParse(manifest).success).toBe(false) + }) + it('requires canonical units and explicit HRV windows for comparable metrics', () => { + const { manifest } = fixture() + const metric = manifest.metrics[0] + const parse = (unit: string, reference: unknown) => algorithmPackageManifestSchema.safeParse({ ...manifest, metrics: [{ ...metric, unit, reference }] }).success + expect(parse('bpm', { metric: 'heart_rate', windowMs: null })).toBe(true) + expect(parse('ms', { metric: 'hrv_rmssd', windowMs: 300000 })).toBe(true) + expect(parse('breaths/min', { metric: 'breathing_rate', windowMs: null })).toBe(true) + expect(parse('bpm', { metric: 'breathing_rate', windowMs: null })).toBe(false) + expect(parse('bpm', { metric: 'hrv_rmssd', windowMs: 300000 })).toBe(false) + expect(parse('ms', { metric: 'hrv_rmssd', windowMs: null })).toBe(false) + expect(parse('ms', { metric: 'hrv_unknown', windowMs: 300000 })).toBe(false) + }) + it('rejects ambiguous identity, unhashed entrypoints, missing locks and unknown privileges', () => { + const { manifest } = fixture() + for (const altered of [ + { ...manifest, metrics: [...manifest.metrics, manifest.metrics[0]] }, + { ...manifest, artifacts: [...manifest.artifacts, manifest.artifacts[0]] }, + { ...manifest, hardware: [...manifest.hardware, manifest.hardware[0]] }, + { ...manifest, hardware: [{ id: 'pod4', sampleRatesHz: [500, 500] }] }, + { ...manifest, entrypoint: 'other.py' }, { ...manifest, dependencyLock: 'absent.lock' }, + { ...manifest, dependencyLock: manifest.entrypoint }, { ...manifest, network: true }, + { ...manifest, resources: { ...manifest.resources, processes: 0 } }, + { ...manifest, temporal: { mode: 'causal', lookaheadMs: 1000 } }, + ]) expect(algorithmPackageManifestSchema.safeParse(altered).success).toBe(false) + }) +}) diff --git a/src/biometrics/packages/tests/protocol.test.ts b/src/biometrics/packages/tests/protocol.test.ts new file mode 100644 index 00000000..85870d07 --- /dev/null +++ b/src/biometrics/packages/tests/protocol.test.ts @@ -0,0 +1,83 @@ +import { describe, expect, it } from 'vitest' +import { canonicalJson, configHash, dataHash } from '../../algorithms/canonical' +import { PackageFrameValidator } from '../protocol' +import { packageFixture } from './fixture' + +function fixture() { + const { manifest } = packageFixture() + const request = { schema: 'biometric-package-run-v1', packageHash: configHash(manifest), inputHash: 'a'.repeat(64), hardwareId: 'pod4', sampleRateHz: 500, side: 'left', seed: 123, config: {} } + const frame = { schema: 'biometric-signal-frame-v1', sequence: 0, timestampMs: 1000, samples: { left: [1, 2, 3], right: null }, context: { occupied: null, pumpActive: null } } + const validator = new PackageFrameValidator(manifest, request) + const result = { schema: 'biometric-metric-frame-v1', runHash: validator.runHash, frameHash: dataHash(frame), sequence: 0, + readings: [{ timestampMs: 1004, side: 'left', metric: 'heart_rate', value: 60, quality: 0.9, flags: [] }], state: { count: 3 } } + return { manifest, request, frame, result, validator } +} + +describe('algorithm frame exchange', () => { + it('accepts deterministic identities and ordered frames with explicit unknown context', () => { + const { frame, result, validator } = fixture() + expect(validator.validateStep(frame, result).endMs).toBe(1006) + const next = { ...frame, sequence: 1, timestampMs: 1006 } + expect(validator.validateStep(next, { ...result, frameHash: dataHash(next), sequence: 1, readings: [] }).endMs).toBe(1012) + }) + it('binds the run to package, input, configuration and deterministic seed', () => { + const { manifest, request, validator } = fixture() + for (const changed of [{ ...request, seed: 124 }, { ...request, config: { threshold: 10 } }, { ...request, inputHash: 'b'.repeat(64) }]) { + expect(new PackageFrameValidator(manifest, changed).runHash).not.toBe(validator.runHash) + } + for (const changed of [{ ...request, packageHash: 'b'.repeat(64) }, { ...request, hardwareId: 'other' }, { ...request, sampleRateHz: 100 }]) { + expect(() => new PackageFrameValidator(manifest, changed)).toThrow() + } + }) + it('rejects output substitution and leaves the run retryable', () => { + const { frame, result, validator } = fixture() + for (const changed of [{ ...result, runHash: 'b'.repeat(64) }, { ...result, frameHash: 'b'.repeat(64) }, { ...result, sequence: 1 }]) { + expect(() => validator.validateStep(frame, changed)).toThrow('identity') + } + expect(validator.validateStep(frame, result).endMs).toBe(1006) + }) + it('rejects future, undeclared, out-of-range and duplicated readings', () => { + const { frame, result } = fixture() + const reading = result.readings[0] + for (const readings of [[{ ...reading, timestampMs: 1006 }], [{ ...reading, timestampMs: 999 }], [{ ...reading, side: 'right' }], + [{ ...reading, metric: 'hrv' }], [{ ...reading, value: 300 }], [reading, reading]]) { + expect(() => fixture().validator.validateStep(frame, { ...result, readings })).toThrow() + } + }) + it('rejects overlapping frames, missing channels and inconsistent channel lengths', () => { + const { frame, result, validator } = fixture() + for (const changed of [{ ...frame, sequence: 1 }, { ...frame, samples: { left: null, right: [1] } }, { ...frame, samples: { left: [1], right: [1, 2] } }]) { + expect(() => validator.validateStep(changed, result)).toThrow() + } + expect(() => validator.validateFrame({ ...frame, timestampMs: Number.MAX_SAFE_INTEGER })).toThrow('timestamp range') + validator.validateStep(frame, result) + expect(() => validator.validateStep({ ...frame, sequence: 1, timestampMs: 1005 }, result)).toThrow('overlapping') + const next = { ...frame, sequence: 1, timestampMs: 1006 } + expect(() => validator.validateStep(next, { ...result, sequence: 1, frameHash: dataHash(next) })).toThrow('Duplicate') + }) + it('enforces declared lookahead without accepting future readings', () => { + const { manifest, request, frame, result } = fixture() + const retrospective = { ...manifest, temporal: { mode: 'retrospective', lookaheadMs: 4 } } + const validator = new PackageFrameValidator(retrospective, { ...request, packageHash: configHash(retrospective) }) + const output = { ...result, runHash: validator.runHash } + expect(() => validator.validateStep(frame, output)).toThrow('time range') + expect(validator.validateStep(frame, { ...output, readings: [{ ...output.readings[0], timestampMs: 1000 }] })).toBeDefined() + }) + it('enforces the cumulative output budget across individually small responses', () => { + const { manifest, request, frame, result } = fixture() + manifest.resources.outputBytes = Buffer.byteLength(canonicalJson(result)) + 10 + const validator = new PackageFrameValidator(manifest, { ...request, packageHash: configHash(manifest) }) + validator.validateStep(frame, { ...result, runHash: validator.runHash }) + const next = { ...frame, sequence: 1, timestampMs: 1006 } + expect(() => validator.validateStep(next, { ...result, runHash: validator.runHash, frameHash: dataHash(next), sequence: 1, readings: [] })).toThrow('Output byte limit') + }) + it('bounds state and response sizes and rejects non-JSON state', () => { + for (const kind of ['state', 'output', 'non-json']) { + const { manifest, request, frame, result } = fixture() + if (kind === 'state') manifest.state.maximumBytes = 2 + if (kind === 'output') manifest.resources.outputBytes = 2 + const validator = new PackageFrameValidator(manifest, { ...request, packageHash: configHash(manifest) }) + expect(() => validator.validateStep(frame, { ...result, runHash: validator.runHash, state: kind === 'non-json' ? new Date() : result.state })).toThrow() + } + }) +})