CR: feat/bio-cal-2-backtest

package.json

@@ -4,7 +4,7 @@
   "private": true,
   "scripts": {
     "prebuild": "node scripts/generate-git-info.mjs && lingui compile",
-    "build": "next build && node scripts/strip-turbopack-external-hashes.mjs && cp -r .next/static .next/standalone/.next/static && { [ ! -d public ] || cp -r public .next/standalone/public || true; }",
+    "build": "next build && node scripts/strip-turbopack-external-hashes.mjs && cp -r .next/static .next/standalone/.next/static && { [ ! -d public ] || cp -r public .next/standalone/public || true; } && node scripts/build/biometrics-worker.mjs",
     "dev": "next dev",
     "db:generate": "drizzle-kit generate",
     "db:migrate": "drizzle-kit migrate",
@@ -27,7 +27,9 @@
     "test:mutation": "stryker run",
     "test:mutation:incremental": "stryker run --incremental",
     "tsc": "tsc --noEmit",
-    "prepare": "git rev-parse --git-dir >/dev/null 2>&1 && git config core.hooksPath .githooks || true"
+    "prepare": "git rev-parse --git-dir >/dev/null 2>&1 && git config core.hooksPath .githooks || true",
+    "build:biometrics-worker": "node scripts/build/biometrics-worker.mjs",
+    "biometrics:experiment": "tsx scripts/algorithm-experiments/cli.ts"
   },
   "lint-staged": {
     "*.{ts,tsx}": "eslint --fix",

scripts/algorithm-experiments/accuracy-policy.ts

@@ -0,0 +1,15 @@
+import { z } from 'zod'
+
+/** Operator-reviewed criteria, frozen into the protocol before holdout access.
+ * There are deliberately no production defaults or clinical accuracy claims. */
+export const packageAccuracyPolicySchema = z.object({
+  schema: z.literal('biometric-package-accuracy-policy-v1'),
+  minimumMaeImprovement: z.number().finite().positive(),
+  maximumNightMaeRegression: z.number().finite().nonnegative(),
+  maximumNightP95Regression: z.number().finite().nonnegative(),
+  maximumNightAbsoluteBiasIncrease: z.number().finite().nonnegative(),
+  minimumCommonPairsPerNight: z.number().int().positive().max(100000),
+  minimumReferenceCoverage: z.number().positive().max(1),
+  maximumReferenceCoverageLoss: z.number().min(0).max(1),
+  maximumLostTimestampFraction: z.number().min(0).max(1),
+}).strict()

scripts/algorithm-experiments/accuracy.test.ts

@@ -0,0 +1,82 @@
+// @vitest-environment node
+import { createHash } from 'node:crypto'
+import { describe, expect, it } from 'vitest'
+import { compareNumericConfigurations } from '../../src/biometrics/backtest/numeric-comparison'
+import { createNightUncertainty } from '../../src/biometrics/backtest/uncertainty-core'
+import { normalizeReferenceImport } from '../../src/biometrics/references/normalize-core'
+import { assessPackageAccuracy } from './accuracy'
+import { managedProtocolSchema } from './managed-protocol'
+import { fixtureResourceTestPolicy } from './resource-test-policy'
+import { configHash } from '../../src/biometrics/algorithms/canonical'
+
+type Input = Parameters<typeof assessPackageAccuracy>[0]
+const policy = { schema: 'biometric-package-accuracy-policy-v1', minimumMaeImprovement: 0.5, maximumNightMaeRegression: 0,
+  maximumNightP95Regression: 0, maximumNightAbsoluteBiasIncrease: 0, minimumCommonPairsPerNight: 2,
+  minimumReferenceCoverage: 1, maximumReferenceCoverageLoss: 0, maximumLostTimestampFraction: 0 }
+function fixture(): Input {
+  const protocol = managedProtocolSchema.parse({ schema: 'biometric-managed-protocol-v1', hardwareId: 'pod4', sampleRateHz: 500, metricId: 'heart_rate', minimumQuality: 0.8,
+    alignment: { version: 'reference-alignment-v1', pointToleranceMs: 0, clockOffsetMs: 0, offsetSource: 'fixed' },
+    uncertainty: { version: 'paired-night-percentile-v1', seed: 'accuracy-test', resamples: 200, minimumNights: 3 },
+    resourcePolicy: fixtureResourceTestPolicy, accuracyPolicy: policy })
+  const reference = normalizeReferenceImport({ subjectId: 'fixture', side: 'left', source: 'synthetic',
+    rows: [0, 1].map(index => ({ recordKey: String(index), metric: 'heart_rate', value: 60, unit: 'bpm', timestampMs: 1700000000000 + index * 1000 })) })
+  const nights = [0, 1, 2].map((index) => {
+    const outputs = (['default', 'current', 'candidate'] as const).map((label, variant) => ({ label, configHash: String(variant),
+      observations: [0, 1].map(point => ({ timestampMs: 1700000000000 + point * 1000, value: variant === 2 ? 60 : 64 + index + (1 - variant) })) }))
+    const comparison = { schema: 'biometric-package-comparison-report-v1' as const, inputHash: 'a', referenceHash: 'b', metricId: 'heart_rate',
+      contract: { metric: 'heart_rate' as const, windowMs: null }, alignment: protocol.alignment, startMs: 1700000000000, endMs: 1700000060000,
+      minimumQuality: 0.8, excludedBoundaryReferences: 0, compatibilityExclusions: {}, referenceSummary: reference.summary,
+      comparisons: compareNumericConfigurations('heart_rate', null, outputs, reference.measurements, protocol.alignment),
+      interpretation: 'descriptive_reference_agreement_not_independent_holdout_or_promotion_evidence' as const }
+    return { nightId: String(index), reportHash: 'c', comparison, resources: [] }
+  })
+  const uncertainty = createNightUncertainty(text => createHash('sha256').update(text).digest(), 'equal_night_mean_common_mae_difference')
+  return { protocol, nights, uncertainty: (['default', 'current'] as const).map(baseline => ({ baseline,
+    report: uncertainty(nights.map(night => ({ cohortId: night.nightId, difference: night.comparison.comparisons.find(row => row.baseline === baseline && row.candidate === 'candidate')?.differences.commonMae ?? null })), protocol.uncertainty) })),
+  currentEvidence: { trialId: 'fixture', state: 'succeeded', reportHash: 'hash', otherExposures: [], uniqueHoldoutAttempt: true } }
+}
+describe('frozen package accuracy criteria', () => {
+  it('requires supported improvement against both baselines without authorizing installation', () => {
+    expect(assessPackageAccuracy(fixture())).toMatchObject({ status: 'criteria_met', failures: [], missing: [], interpretation: expect.stringContaining('requires_full_package') })
+    const input = fixture()
+    const report = input.uncertainty[1].report
+    if (report.interval) report.interval.upper = 0
+    expect(assessPackageAccuracy(input)).toMatchObject({ status: 'criteria_not_met', failures: ['current:improvement_not_supported'] })
+  })
+  it('cannot backfill criteria into an already frozen protocol or reuse exposed holdout', () => {
+    const input = fixture(), frozen = configHash(input.protocol)
+    delete input.protocol.accuracyPolicy
+    expect(configHash(input.protocol)).not.toBe(frozen)
+    expect(assessPackageAccuracy(input).missing).toContain('no_frozen_accuracy_policy')
+    input.currentEvidence.uniqueHoldoutAttempt = false
+    expect(assessPackageAccuracy(input).missing).toContain('no_unique_current_holdout')
+  })
+  it('rejects coverage loss and individual-night regressions even with favorable aggregate intervals', () => {
+    const input = fixture()
+    const row = input.nights[0].comparison.comparisons.find(row => row.candidate === 'candidate' && row.baseline === 'default')
+    if (!row) throw new Error('Missing fixture comparison')
+    row.ownSupport.candidate.referenceCoverage = 0.5
+    row.differences.referenceCoverage = -0.5
+    row.lostObservationTimestamps = 1
+    row.differences.commonMae = 1
+    row.differences.commonP95AbsoluteError = 2
+    if (row.commonSupport.candidate.metrics) row.commonSupport.candidate.metrics.signedBias = 99
+    const result = assessPackageAccuracy(input)
+    expect(result.status).toBe('criteria_not_met')
+    expect(result.failures).toEqual(expect.arrayContaining(['0:default:reference_coverage_below_minimum', '0:default:reference_coverage_loss',
+      '0:default:observation_support_loss', '0:default:night_mae_regression', '0:default:night_tail_regression', '0:default:night_bias_regression']))
+  })
+  it('does not treat omitted nights, missing pairs or degenerate intervals as successful evidence', () => {
+    const input = fixture()
+    input.uncertainty[0].report.omittedNights = 1
+    input.uncertainty[1].report = { ...input.uncertainty[1].report, status: 'degenerate', interval: null }
+    const row = input.nights[0].comparison.comparisons.find(row => row.candidate === 'candidate' && row.baseline === 'default')
+    if (!row) throw new Error('Missing fixture comparison')
+    row.commonSupport.candidate.pairedCount = 0
+    row.ambiguousObservationTimestamps.candidate = 1
+    expect(assessPackageAccuracy(input)).toMatchObject({ status: 'insufficient_evidence', missing: expect.arrayContaining([
+      'default:insufficient_whole_night_uncertainty', 'current:insufficient_whole_night_uncertainty',
+      '0:default:insufficient_common_pairs', '0:default:ambiguous_observation_support',
+    ]) })
+  })
+})

scripts/algorithm-experiments/accuracy.ts

@@ -0,0 +1,66 @@
+import { configHash } from '../../src/biometrics/algorithms/canonical'
+import { packageAccuracyPolicySchema } from './accuracy-policy'
+import { readManagedEvidence } from './managed-evidence'
+
+type Evidence = ReturnType<typeof readManagedEvidence>
+type AccuracyInput = Pick<Evidence, 'protocol' | 'nights' | 'uncertainty' | 'currentEvidence'>
+
+/** Pure assessment of already verified evidence. The authoritative entry point
+ * below reads the retained graph and fresh ledger state first. */
+export function assessPackageAccuracy(evidence: AccuracyInput) {
+  const failures: string[] = [], missing: string[] = []
+  const rawPolicy = evidence.protocol.accuracyPolicy
+  const policy = rawPolicy === undefined ? null : packageAccuracyPolicySchema.parse(rawPolicy)
+  if (!policy) missing.push('no_frozen_accuracy_policy')
+  if (!evidence.currentEvidence.uniqueHoldoutAttempt) missing.push('no_unique_current_holdout')
+  if (!evidence.nights.length) missing.push('no_nights')
+  for (const night of evidence.nights) {
+    for (const baseline of ['default', 'current'] as const) {
+      const key = `${night.nightId}:${baseline}`
+      const comparison = night.comparison.comparisons.find(row => row.candidate === 'candidate' && row.baseline === baseline)
+      if (!comparison) {
+        missing.push(`${key}:missing_comparison`)
+        continue
+      }
+      if (comparison.commonSupport.candidate.status === 'incompatible' || comparison.commonSupport.baseline.status === 'incompatible') {
+        failures.push(`${key}:incompatible_metric`)
+        continue
+      }
+      if (comparison.ambiguousObservationTimestamps.baseline || comparison.ambiguousObservationTimestamps.candidate) missing.push(`${key}:ambiguous_observation_support`)
+      if (!policy) continue
+      const candidate = comparison.commonSupport.candidate, previous = comparison.commonSupport.baseline
+      if (!candidate.metrics || !previous.metrics || candidate.pairedCount < policy.minimumCommonPairsPerNight
+        || previous.pairedCount < policy.minimumCommonPairsPerNight) missing.push(`${key}:insufficient_common_pairs`)
+      const own = comparison.ownSupport.candidate
+      if (own.referenceCoverage === null) missing.push(`${key}:missing_reference_coverage`)
+      else if (own.referenceCoverage < policy.minimumReferenceCoverage) failures.push(`${key}:reference_coverage_below_minimum`)
+      const coverageChange = comparison.differences.referenceCoverage
+      if (coverageChange === null) missing.push(`${key}:missing_coverage_difference`)
+      else if (coverageChange < -policy.maximumReferenceCoverageLoss) failures.push(`${key}:reference_coverage_loss`)
+      const baselineCount = comparison.ownSupport.baseline.validObservationCount
+      if (!baselineCount) missing.push(`${key}:missing_baseline_observations`)
+      else if (comparison.lostObservationTimestamps / baselineCount > policy.maximumLostTimestampFraction) failures.push(`${key}:observation_support_loss`)
+      if (comparison.differences.commonMae !== null && comparison.differences.commonMae > policy.maximumNightMaeRegression) failures.push(`${key}:night_mae_regression`)
+      if (comparison.differences.commonP95AbsoluteError !== null && comparison.differences.commonP95AbsoluteError > policy.maximumNightP95Regression) failures.push(`${key}:night_tail_regression`)
+      if (candidate.metrics && previous.metrics
+        && Math.abs(candidate.metrics.signedBias) - Math.abs(previous.metrics.signedBias) > policy.maximumNightAbsoluteBiasIncrease) failures.push(`${key}:night_bias_regression`)
+    }
+  }
+  for (const baseline of ['default', 'current'] as const) {
+    const report = evidence.uncertainty.find(row => row.baseline === baseline)?.report
+    if (!report || report.status !== 'estimated' || !report.interval || report.omittedNights > 0
+      || report.scoredNights !== evidence.nights.length) missing.push(`${baseline}:insufficient_whole_night_uncertainty`)
+    else if (policy && (report.estimate === null || report.estimate > -policy.minimumMaeImprovement
+      || report.interval.upper > -policy.minimumMaeImprovement)) failures.push(`${baseline}:improvement_not_supported`)
+  }
+  return { schema: 'biometric-package-accuracy-assessment-v1' as const,
+    status: failures.length ? 'criteria_not_met' as const : missing.length ? 'insufficient_evidence' as const : 'criteria_met' as const,
+    policyHash: policy ? configHash(policy) : null, failures, missing,
+    interpretation: 'reference_agreement_criteria_only_requires_full_package_qualification_and_review' as const }
+}
+
+export function assessManagedAccuracy(...args: Parameters<typeof readManagedEvidence>) {
+  const evidence = readManagedEvidence(...args)
+  return { trialId: evidence.trialId, reportHash: evidence.reportHash, currentEvidence: evidence.currentEvidence,
+    assessment: assessPackageAccuracy(evidence), preflight: evidence.preflight }
+}

scripts/algorithm-experiments/cli.ts

@@ -0,0 +1,87 @@
+import { recoverExperimentStaging } from './staging-recovery'
+import { runPackagePreflight } from './preflight'
+import { recoverExpiredContainers } from './recovery'
+import { pathToFileURL } from 'node:url'
+import { z } from 'zod'
+import { runManagedFiles } from './managed-files'
+import { experimentPolicySchema as policySchema } from './policy'
+import { comparePackageOutputs } from './comparison'
+import { runPackageFixtures } from './fixtures'
+import { packageInputHash } from './input'
+import { inspectPythonEnvironment } from './environment'
+import { executePackageExperiment } from './container'
+import { checkPackageDeterminism } from './determinism'
+import { publishJson, readJsonFile, readJsonLines, readPackageProposal } from './files'
+
+const positive = z.number().int().positive().max(Number.MAX_SAFE_INTEGER)
+const usage = 'Usage: preflight PROPOSAL PLAN SUITE POLICY OUTPUT | recover OUTPUT | managed WORKSPACE DESCRIPTOR RECEIPT [STORAGE_POLICY] | compare INPUT OUTPUT | fixtures PROPOSAL SUITE POLICY OUTPUT | lock IMAGE OUTPUT | input-hash FRAMES MAX_BYTES OUTPUT | inspect PROPOSAL OUTPUT | run PROPOSAL REQUEST FRAMES POLICY OUTPUT | determinism PROPOSAL REQUEST FRAMES POLICY REPETITIONS OUTPUT'
+
+export async function runExperimentCli(args: string[]) {
+  const [command, ...paths] = args
+  if (command === 'preflight' && paths.length === 5) {
+    const proposal = readPackageProposal(paths[0])
+    const plan = readJsonFile(paths[1], 16384)
+    const suite = readJsonFile(paths[2], 32 * 1024 * 1024)
+    const policy = policySchema.parse(readJsonFile(paths[3], 16384))
+    await publishJson(paths[4], await runPackagePreflight({ ...proposal, policy }, plan, suite))
+    return
+  }
+  if (command === 'recover' && paths.length === 1) {
+    const containers = await recoverExpiredContainers()
+    const staging = await recoverExperimentStaging()
+    await publishJson(paths[0], { schema: 'biometric-experiment-recovery-v1', containers, staging })
+    return
+  }
+  if (command === 'managed' && (paths.length === 3 || paths.length === 4)) {
+    await publishJson(paths[2], await runManagedFiles(paths[0], paths[1], paths[3] ? readJsonFile(paths[3], 16384) : undefined))
+    return
+  }
+  if (command === 'compare' && paths.length === 2) {
+    await publishJson(paths[1], comparePackageOutputs(readJsonFile(paths[0], 32 * 1024 * 1024)))
+    return
+  }
+  if (command === 'fixtures' && paths.length === 4) {
+    const proposal = readPackageProposal(paths[0])
+    const suite = readJsonFile(paths[1], 32 * 1024 * 1024)
+    const policy = policySchema.parse(readJsonFile(paths[2], 16384))
+    const report = await runPackageFixtures({ manifest: proposal.manifest, files: proposal.files, policy }, suite)
+    await publishJson(paths[3], report)
+    return
+  }
+  if (command === 'lock' && paths.length === 2) {
+    await publishJson(paths[1], await inspectPythonEnvironment(paths[0]))
+    return
+  }
+  if (command === 'input-hash' && paths.length === 3) {
+    const maximumBytes = positive.parse(Number(paths[1]))
+    const hash = packageInputHash(readJsonLines(paths[0], maximumBytes))
+    await publishJson(paths[2], { schema: 'biometric-package-input-v1', hash })
+    return
+  }
+  if (command === 'inspect' && paths.length === 2) {
+    const proposal = readPackageProposal(paths[0])
+    await publishJson(paths[1], { schema: 'biometric-package-intake-v1', packageHash: proposal.packageHash,
+      status: proposal.status, manifest: proposal.manifest, totalBytes: proposal.totalBytes })
+    return
+  }
+  if ((command === 'run' && paths.length === 5) || (command === 'determinism' && paths.length === 6)) {
+    const policy = policySchema.parse(readJsonFile(paths[3], 16384))
+    const proposal = readPackageProposal(paths[0])
+    const request = readJsonFile(paths[1], 65536)
+    const frames = () => readJsonLines(paths[2], policy.maximumInputBytes)
+    const options = { manifest: proposal.manifest, files: proposal.files, request, policy }
+    const report = command === 'run'
+      ? { schema: 'biometric-package-experiment-v1', status: 'experiment_only', ...await executePackageExperiment({ ...options, frames: frames() }) }
+      : await checkPackageDeterminism({ ...options, frames }, Number(paths[4]))
+    await publishJson(paths[paths.length - 1], report)
+    return
+  }
+  throw new Error(usage)
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+  runExperimentCli(process.argv.slice(2)).catch((error: unknown) => {
+    process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
+    process.exitCode = 1
+  })
+}