diff --git a/docs/lexicon.md b/docs/lexicon.md index e5e5ededc..f3f6864e0 100644 --- a/docs/lexicon.md +++ b/docs/lexicon.md @@ -7,3 +7,7 @@ - Layout sibling: the previous or next unit in the target's enclosing vertical layout. Transforms and viewport clipping preserve this relation. - Placeholder: replacement text bars, an empty ring track or a muted plot sine, painted over the existing surface. Icons, images and card titles keep their source paint. - Unscoped guide content: a page-wide selector, such as every pressable row. It does not identify a particular control on a neighbouring step. + +## Sleep staging + +- Staging provenance: a night's requested method, effective method and observed seconds that retained the original labels after an HMM abstention. `hmm_cardio` means HMM and cardio both contributed; `hmm_vendor_staged` means HMM and a paired device both contributed. An absent field means unknown; it does not mean cardio or zero fallback. Avoid calling the requested method the effective method. diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index d8af739ec..2bde47b07 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -1893,7 +1893,10 @@ import 'vendor_sleep.dart'; // no window does, instead of the longest of the search window's sessions. // HrvFreqStream v2, CardioRowCache v3 and policy-bound PRSA states require cold // rebuilds of old checkpoints, isolated by this algorithm version. -const int kAlgoVersion = 124; +// 124 → 125: HMM nights persist the effective method and observed cardio +// fallback seconds. Capture sleep_hmm_stages in prepared worker inputs and +// staging cache identity; cardio payloads are unchanged (hmm_staging_provenance_test). +const int kAlgoVersion = 125; /// The sibling SHAs this version was derived against, asserted against /// pubspec.yaml in test/db_serve_version_and_reads_test.dart. /// @@ -2680,6 +2683,7 @@ class _BaselineHistoryCache { 'rhr', 'resp_rate', 'skin_temp_adc', + 'enmo_median_night', 'readiness', // The readiness composite z — later days' scores are calibrated against // its spread (`calibratedReadinessScore`). @@ -4451,7 +4455,9 @@ class DerivationEngine { // rows, which the signature check rejects. Read after, it would attach the // newer revision to rows that predate it and certify a stale ceiling. final ceilingInput = await _ceilingInputRevision(dayId); - final candidate = await _sleepCandidateForDay(dayId, stats: stats, union: union); + final useHmmStages = await _hmmStagesOn(); + final candidate = await _sleepCandidateForDay(dayId, stats: stats, union: union, + useHmmStages: useHmmStages); final dayStart = _localDayLabelToSec(dayId); final dayEnd = _localNextDayLabelToSec(dayId); @@ -4568,6 +4574,7 @@ class DerivationEngine { sleepOffsetSec: candidate.present ? candidate.sleepOffsetSec : 0, ); final prepared = candidate.toPreparedDay( + useHmmStages: useHmmStages, daySub: daySub, napSub: napSub, sleepSub: sleepSub, @@ -4725,7 +4732,9 @@ class DerivationEngine { String dayId, { _PrepareStats? stats, _PrepareUnion? union, + bool? useHmmStages, }) async { + final hmmStages = useHmmStages ?? await _hmmStagesOn(); // A user sleep override is the source of truth — never serve the cached auto // candidate, and don't cache the override result (so a later edit / clear is // not shadowed by a stale artifact). The auto path keeps its finalized cache. @@ -4931,6 +4940,7 @@ class DerivationEngine { vendorNights: vendorNights, activeWearable: stagingWearable, prefixContext: prefixContext, + useHmmStages: hmmStages, ); if (revisions != null && !debugDisableStagedSleepCache) stagingKey = inputKey(false); if (checkpointPlan != null) { @@ -6777,6 +6787,7 @@ class DerivationEngine { ); await _derivePreparedDay( PreparedDerivationDay( + useHmmStages: await _hmmStagesOn(), date: day.date, endSec: day.endSec, confidence: day.confidence, @@ -6889,6 +6900,19 @@ class DerivationEngine { sleepRrTsMs: sleepSub.rrTsMs, sleepRrMs: sleepSub.rrMs, sleepSkinTemp: sleepSub.skinTemp, + // Per-second ENMO over the sleep window (gravity vector magnitude − 1 g), + // for the hybrid HMM input. Absent accel (0,0,0 sentinel) -> 0.0, which + // the classifier reads as "no motion evidence", never "still". + sleepEnmo: [ + for (var i = 0; i < sleepSub.tsSec.length; i++) + sleepSub.accelPresentAt(i) + ? math.sqrt( + sleepSub.ax[i] * sleepSub.ax[i] + + sleepSub.ay[i] * sleepSub.ay[i] + + sleepSub.az[i] * sleepSub.az[i], + ) - 1.0 + : -1.0, // absent sentinel (CodeRabbit #756): measured zero motion is >= 0 + ], sleepJson: day.sleepJson, hypnoStages: day.hypnoStages, sleepOnsetSec: day.sleepOnsetSec, @@ -6905,7 +6929,7 @@ class DerivationEngine { // guess — sleep-onset latency only means what people think it means on a // forced window. sleepSource: day.sleepSource, - useHmmStages: await _hmmStagesOn(), + useHmmStages: day.useHmmStages, ); final withHistory = _attachHistory(input, history); @@ -7759,6 +7783,10 @@ class DerivationEngine { // EVERY day (even during the bootstrap window where skin_temp_z is null) // so the baseline fills and z begins computing from ~day 4. 'skin_temp_adc': sc('skin_temp_adc'), + // The night's stable-core ENMO median, only for HMM-restaged nights + // (null otherwise -> no series row, history fills from HMM nights). + if (((bundle['sleep'] as Map?)?['stager_enmo_median'] as num?)?.toDouble() case final v?) + if (v > 0) 'enmo_median_night': v, 'dip_pct': sc('dip_pct'), // Headline 0–21 strain (for trend/sparkline); raw TRIMP kept too. // Written even when NULL: an abstaining day must overwrite a value a @@ -8383,6 +8411,8 @@ class DerivationEngine { // mismatch that left z permanently null. The raw mean is stored every day so // this series fills and z starts computing once ≥3 days exist. m['skin_temp_adc_history'] = before('skin_temp_adc'); + // Per-night stable-core ENMO medians (written only for HMM-staged nights). + m['enmo_history'] = before('enmo_median_night'); // TS-03/TS-04 — the observed ceiling this day's ZONES are banded on. A max, // not a window (see [maxBefore]), and strictly before today so a day is // never banded on a ceiling its own session set. diff --git a/lib/compute/derive_prepare.dart b/lib/compute/derive_prepare.dart index 12845ca04..bbd845d85 100644 --- a/lib/compute/derive_prepare.dart +++ b/lib/compute/derive_prepare.dart @@ -20,6 +20,9 @@ class PreparedDerivationDay { final int sleepOnsetSec; final int sleepOffsetSec; final String sleepSource; + + /// Captured before staging so the cache and bundle worker use one choice. + final bool useHmmStages; final Substrate daySub; final Substrate sleepSub; @@ -78,6 +81,7 @@ class PreparedDerivationDay { required this.sleepSub, Substrate? napSub, this.sleepSource = 'auto', + this.useHmmStages = false, this.tonightSleepOnsetSec = 0, this.tonightSleepOffsetSec = 0, this.tonightPending = false, @@ -95,6 +99,7 @@ class PreparedDerivationDay { 'sleep_onset_sec': sleepOnsetSec, 'sleep_offset_sec': sleepOffsetSec, 'sleep_source': sleepSource, + if (useHmmStages) 'use_hmm_stages': true, 'day_sub': daySub.toJson(), 'sleep_sub': sleepSub.toJson(), 'nap_sub': napSub.toJson(), @@ -124,6 +129,7 @@ class PreparedDerivationDay { hypnoStages: strs('hypno_stages'), sleepOnsetSec: (m['sleep_onset_sec'] as num?)?.toInt() ?? 0, sleepOffsetSec: (m['sleep_offset_sec'] as num?)?.toInt() ?? 0, + useHmmStages: m['use_hmm_stages'] == true, sleepSource: m['sleep_source'] as String? ?? 'auto', tonightSleepOnsetSec: (m['tonight_sleep_onset_sec'] as num?)?.toInt() ?? 0, @@ -271,6 +277,7 @@ class SleepSessionCandidate { int tonightSleepOnsetSec = 0, int tonightSleepOffsetSec = 0, bool tonightPending = false, + bool useHmmStages = false, Map> ownership = const {}, Map> priority = const {}, }) => PreparedDerivationDay( @@ -292,6 +299,7 @@ class SleepSessionCandidate { sleepOnsetSec: sleepOnsetSec, sleepOffsetSec: sleepOffsetSec, sleepSource: sleepSource, + useHmmStages: useHmmStages, daySub: daySub, napSub: napSub, sleepSub: sleepSub, diff --git a/lib/compute/onehz_pipeline.dart b/lib/compute/onehz_pipeline.dart index 71d8545b7..bf0ab5073 100644 --- a/lib/compute/onehz_pipeline.dart +++ b/lib/compute/onehz_pipeline.dart @@ -337,6 +337,13 @@ class DayBundleInput { final List sleepRrTsMs; final List sleepRrMs; final List sleepSkinTemp; + /// Per-second ENMO (g) over the sleep window — HYBRID HMM input. -1 means + /// ABSENT (accel sentinel (0,0,0)), never a still wrist; the classifier + /// treats absent as "no motion evidence" rather than "perfectly still". + final List sleepEnmo; + /// Trailing per-night ENMO medians (stable core), oldest first — the + /// persisted motion history for multi-night self-calibration. + final List enmoHistory; // ── the SINGLE-SOURCE sleep segmentation (JSON of SleepSegmentation) ────── final Map sleepJson; // {window,tst_sec,…,confidence} @@ -436,6 +443,8 @@ class DayBundleInput { required this.sleepRrTsMs, required this.sleepRrMs, required this.sleepSkinTemp, + this.sleepEnmo = const [], + this.enmoHistory = const [], required this.sleepJson, required this.hypnoStages, required this.sleepOnsetSec, @@ -471,6 +480,8 @@ class DayBundleInput { 'sleep_rr_ts_ms': sleepRrTsMs, 'sleep_rr_ms': sleepRrMs, 'sleep_skin_temp': sleepSkinTemp, + 'sleep_enmo': sleepEnmo, + 'enmo_history': enmoHistory, 'sleep_json': sleepJson, 'hypno_stages': hypnoStages, 'sleep_onset_sec': sleepOnsetSec, @@ -523,6 +534,8 @@ class DayBundleInput { sleepRrTsMs: dbls('sleep_rr_ts_ms'), sleepRrMs: dbls('sleep_rr_ms'), sleepSkinTemp: ints('sleep_skin_temp'), + sleepEnmo: dbls('sleep_enmo'), + enmoHistory: dbls('enmo_history'), sleepJson: ((m['sleep_json'] as Map?) ?? const {}) .cast(), hypnoStages: strs('hypno_stages'), @@ -561,7 +574,7 @@ class DayBundleInput { /// efficiency. Figures `segmentSleep` derives that edge does not recompute /// here are dropped rather than left describing the old labels. Null when /// the labels hold no sleep at all: the caller keeps the original night. - DayBundleInput? withHypnoStages(List stages) { + DayBundleInput? withHypnoStages(List stages, {bool atypical = false, double? enmoMedian}) { var light = 0, deep = 0, rem = 0, wake = 0, unobserved = 0; var firstSleep = -1, lastSleep = -1; for (var i = 0; i < stages.length; i++) { @@ -606,7 +619,18 @@ class DayBundleInput { ..['rem_sec'] = rem ..['wake_sec'] = wake ..['efficiency_pct'] = observedSec > 0 ? 100.0 * tst / observedSec : null - ..['stager_source'] = 'hmm'; + ..['stager_source'] = 'hmm' + // Multi-night calibration flag: tonight's baseline deviated >2 robust z + // from the user's long-term baseline (illness, alcohol, travel...) — + // the staging is still published, but consumers can treat it as lower + // confidence. + ..['stager_atypical_night'] = atypical; + if (enmoMedian != null && enmoMedian > 0) { + // Persisted as the night's motion baseline -> metric_series + // 'enmo_median_night', feeding later nights' 7d/long-term motion + // calibration (enmo_history). + json['stager_enmo_median'] = enmoMedian; + } return DayBundleInput( date: date, dayTsSec: dayTsSec, @@ -618,6 +642,8 @@ class DayBundleInput { sleepRrTsMs: sleepRrTsMs, sleepRrMs: sleepRrMs, sleepSkinTemp: sleepSkinTemp, + sleepEnmo: sleepEnmo, + enmoHistory: enmoHistory, sleepJson: json, hypnoStages: stages, sleepOnsetSec: sleepOnsetSec, @@ -676,9 +702,38 @@ Map _deriveDayBundle(Map inputJson, // each 30 s epoch it has no heart rate or RMSSD for; a second the band did // not watch stays unobserved. The night's accounting is then recomputed // from the final labels and the night is tagged `stager_source: hmm`. + var stagingProvenance = const {}; if (d.useHmmStages && d.hypnoStages.isNotEmpty) { + final observedSec = d.hypnoStages.where((s) => + s == 'light' || s == 'nrem' || s == 'deep' || s == 'rem' || + s == 'wake').length; + final fallbackMethod = d.sleepSource == 'vendor_staged' + ? 'vendor_staged' : 'cardio'; final restaged = _hmmRestage(d); - if (restaged != null) d = d.withHypnoStages(restaged) ?? d; + final replaced = restaged == null ? null : d.withHypnoStages( + restaged.labels, atypical: restaged.atypicalNight, + enmoMedian: restaged.enmoMedian); + if (replaced == null) { + stagingProvenance = { + 'staging_method': 'hmm', + 'staging_effective_method': fallbackMethod, + 'staging_fallback': restaged == null + ? 'hmm_unavailable' : 'hmm_no_sleep', + 'staging_fallback_sec': observedSec, + }; + } else { + final fallbackSec = restaged!.fallbackSec; + stagingProvenance = { + 'staging_method': 'hmm', + 'staging_effective_method': fallbackSec == 0 ? 'hmm' : + fallbackSec == observedSec ? fallbackMethod : 'hmm_$fallbackMethod', + if (fallbackSec > 0) ...{ + 'staging_fallback': 'hmm_epoch_unavailable', + 'staging_fallback_sec': fallbackSec, + }, + }; + d = replaced; + } } // ── HR over the DAY (for the curve, strain zones, dip day-side) ─────────── @@ -1564,9 +1619,20 @@ Map _deriveDayBundle(Map inputJson, // Why this night produced nothing, when the segmenter knows. Null on a // normal night; never render a bare dash for an absence that carries one. 'absence_reason': absenceReason, + ...stagingProvenance, // Present only when the experimental re-staging labelled this night (PR // #628), so a night of the shipped stager is stored exactly as before. - if (d.sleepJson['stager_source'] == 'hmm') 'stager_source': 'hmm', + if (d.sleepJson['stager_source'] == 'hmm') ...{ + 'stager_source': 'hmm', + // Forward the atypical-night (outlier) flag alongside, so the persisted + // night carries the calibration confidence, not just the source tag. + if (d.sleepJson['stager_atypical_night'] == true) + 'stager_atypical_night': true, + // The night's stable-core ENMO median, persisted to metric_series as + // 'enmo_median_night' for multi-night motion calibration. + if ((d.sleepJson['stager_enmo_median'] as num?)?.toDouble() case final v?) + if (v > 0) 'stager_enmo_median': v, + }, // Seconds the band's own last SLEEP cut off the end of the auto window; null = rule did not apply. Provenance, never a metric. 'band_offset_trim_sec': bandOffsetTrimSec, 'stager': _envelope( @@ -2591,13 +2657,32 @@ List _rmssdPerSecond( /// later label. Within the night the decoder abstains per 30 s epoch: an /// epoch with no plausible heart rate or no RMSSD keeps its shipped label, /// and an unobserved second stays unobserved. -List? _hmmRestage(DayBundleInput d) { +/// Result of a restage: the merged labels plus whether the night's baseline +/// deviated strongly from the user's long-term baseline (atypical night). +class HmmRestageResult { + final List labels; + final bool atypicalNight; + final double? enmoMedian; + final int fallbackSec; + const HmmRestageResult(this.labels, this.atypicalNight, + [this.enmoMedian, this.fallbackSec = 0]); +} + +HmmRestageResult? _hmmRestage(DayBundleInput d) { final n = d.hypnoStages.length; final t0 = d.sleepOnsetSec; + var atypical = false; + double? enmoMedian; final hrAt = List.filled(n, 0.0); + final enmoAt = List.filled(n, 0.0); + final tempAt = List.filled(n, 0.0); for (var i = 0; i < d.sleepTsSec.length && i < d.sleepHr.length; i++) { final k = d.sleepTsSec[i] - t0; - if (k >= 0 && k < n) hrAt[k] = d.sleepHr[i].toDouble(); + if (k >= 0 && k < n) { + hrAt[k] = d.sleepHr[i].toDouble(); + if (i < d.sleepEnmo.length) enmoAt[k] = d.sleepEnmo[i]; + if (i < d.sleepSkinTemp.length) tempAt[k] = d.sleepSkinTemp[i].toDouble(); + } } final corrected = correctRr(d.sleepRrMs, rrTsMs: d.sleepRrTsMs); // nnTimesMs is relative to the start of the first input beat; put it back @@ -2612,14 +2697,30 @@ List? _hmmRestage(DayBundleInput d) { [for (var i = 0; i < n; i++) t0 + i], windowSec: 60, ); - final labels = _classifySleepStagesHMM([ - for (var i = 0; i < n; i++) - { - 'ts': t0 + i, - 'hr': d.hypnoStages[i] == 'unobserved' ? 0.0 : hrAt[i], - 'rmssd': d.hypnoStages[i] == 'unobserved' ? 0.0 : rmssdAt[i], - }, - ]); + final labels = _classifySleepStagesHMM( + [ + for (var i = 0; i < n; i++) + { + 'ts': t0 + i, + 'hr': d.hypnoStages[i] == 'unobserved' ? 0.0 : hrAt[i], + 'rmssd': d.hypnoStages[i] == 'unobserved' ? 0.0 : rmssdAt[i], + // HYBRID: motion + skin temp per second (0 = absent sentinel). + 'enmo': enmoAt[i], + 'skin_temp': tempAt[i], + }, + ], + // Multi-night self-calibration: 7d rolling + long-term baselines from the + // persisted nightly histories (HR, RMSSD, skin temp) feed the blended + // emission baseline and the atypical-night (outlier) flag. + rhrHistory: d.rhrHistory, + rmssdHistory: d.rmssdHistory, + skinTempHistory: d.skinTempAdcHistory, + enmoHistory: d.enmoHistory, + onAnalyzer: (a) { + atypical = a.atypicalNight; + enmoMedian = a.enmoMedianNight; + }, + ); if (labels == null) return null; final merged = [ for (var i = 0; i < n; i++) @@ -2638,14 +2739,52 @@ List? _hmmRestage(DayBundleInput d) { } prev = merged[i]; } - return merged; + // AASM-aligned wake post-processing on the merged per-second labels: + // - a wake run shorter than 15 s is an AROUSAL, not wake (AASM: arousal + // >= 3 s, wake epoch needs > 50% of the 30 s epoch awake) -> the seconds + // revert to the surrounding sleep stage; + // - after a REAL (>= 15 s) wake phase the following sleep does NOT restart + // from scratch — staging continues where it left off (AASM scores + // sequentially, no reset), which the Viterbi path already respects; long + // wake runs simply stay wake and count towards WASO. + const minWakeRunSec = 15; + var i = 0; + while (i < merged.length) { + if (merged[i] != 'wake' || merged[i] == 'unobserved') { + i++; + continue; + } + var j = i; + while (j < merged.length && merged[j] == 'wake') { + j++; + } + if (j - i < minWakeRunSec) { + final surrounding = i > 0 ? merged[i - 1] : 'light'; + for (var k = i; k < j; k++) { + merged[k] = surrounding == 'unobserved' ? 'light' : surrounding; + } + } + i = j; + } + var fallbackSec = 0; + for (var i = 0; i < n; i++) { + if (d.hypnoStages[i] != 'unobserved' && labels[i] == null) fallbackSec++; + } + return HmmRestageResult(merged, atypical, enmoMedian, fallbackSec); } /// The Viterbi decode over 30 s epochs of [epochData] (per-second records /// with `ts`, `hr`, `rmssd`). Null when the whole night abstains (see /// [_hmmRestage]); otherwise one label per record, null where its epoch had /// no usable heart rate or RMSSD. -List? _classifySleepStagesHMM(List> epochData) { +List? _classifySleepStagesHMM( + List> epochData, { + List rhrHistory = const [], + List rmssdHistory = const [], + List skinTempHistory = const [], + List enmoHistory = const [], + void Function(SleepStageAnalyzer)? onAnalyzer, +}) { if (epochData.isEmpty) return null; final analyzer = SleepStageAnalyzer(); // Baseline from the first 120 minutes BY ELAPSED TIME, not the first 240 @@ -2655,6 +2794,19 @@ List? _classifySleepStagesHMM(List> epochData) { .where((e) => ((e['ts'] as num?)?.toDouble() ?? 0.0) - t0 <= 120 * 60) .toList(); analyzer.computeBaseline(first120m); + // Motion baseline from the stable core of the night (middle 50 %), not the + // opening window — settling-in movement would inflate the person's normal + // sleep-motion level and push the wake threshold too high. + analyzer.computeMotionBaseline(epochData); + // THREE-LEVEL calibration: blend in the 7-day rolling and long-term + // baselines from the persisted nightly histories. + analyzer.computeMultiNightBaselines( + rhrHistory: rhrHistory, + rmssdHistory: rmssdHistory, + skinTempHistory: skinTempHistory, + enmoHistory: enmoHistory, + ); + onAnalyzer?.call(analyzer); // No baseline (RR that only starts after the first 120 minutes, say) makes // every emission neutral, and wake's self-transition then labels a measured // night as wake. Keep the shipped labels instead. @@ -2680,11 +2832,15 @@ List? _classifySleepStagesHMM(List> epochData) { final idx = {}; for (var i = 0; i < epochData.length; i++) { final k = idx.putIfAbsent(epochOf[i], () { - epochs.add({'hr': 0.0, 'rmssd': 0.0, 'nh': 0, 'nr': 0}); + // HYBRID: epochs also average ENMO and skin temp (0 sentinel = absent, + // excluded from the mean, never counted as "still"/"cold"). + epochs.add({'hr': 0.0, 'rmssd': 0.0, 'enmo': 0.0, 'skin_temp': 0.0, 'nh': 0, 'nr': 0, 'ne': 0, 'nt': 0}); return epochs.length - 1; }); final hr = (epochData[i]['hr'] as num?)?.toDouble() ?? 0.0; final rmssd = (epochData[i]['rmssd'] as num?)?.toDouble() ?? 0.0; + final enmo = (epochData[i]['enmo'] as num?)?.toDouble() ?? 0.0; + final temp = (epochData[i]['skin_temp'] as num?)?.toDouble() ?? 0.0; final ep = epochs[k]; if (hr > 30.0 && hr < 200.0) { ep['hr'] = (ep['hr'] as double) + hr; @@ -2694,11 +2850,22 @@ List? _classifySleepStagesHMM(List> epochData) { ep['rmssd'] = (ep['rmssd'] as double) + rmssd; ep['nr'] = (ep['nr'] as int) + 1; } + if (enmo >= 0.0) { // measured (incl. real zero motion); -1 = absent + ep['enmo'] = (ep['enmo'] as double) + enmo; + ep['ne'] = (ep['ne'] as int) + 1; + } + if (temp > 0.0) { + ep['skin_temp'] = (ep['skin_temp'] as double) + temp; + ep['nt'] = (ep['nt'] as int) + 1; + } } for (final ep in epochs) { final nh = ep['nh'] as int, nr = ep['nr'] as int; + final ne = ep['ne'] as int, nt = ep['nt'] as int; ep['hr'] = nh == 0 ? 0.0 : (ep['hr'] as double) / nh; ep['rmssd'] = nr == 0 ? 0.0 : (ep['rmssd'] as double) / nr; + ep['enmo'] = ne == 0 ? 0.0 : (ep['enmo'] as double) / ne; + ep['skin_temp'] = nt == 0 ? 0.0 : (ep['skin_temp'] as double) / nt; } final states = analyzer.viterbiDecode(epochs); if (states.length != epochs.length) return null; diff --git a/lib/compute/sleep_pipeline_hmm.dart b/lib/compute/sleep_pipeline_hmm.dart index c09826f8d..b7d1e1799 100644 --- a/lib/compute/sleep_pipeline_hmm.dart +++ b/lib/compute/sleep_pipeline_hmm.dart @@ -34,26 +34,26 @@ class HMMConfig { SleepState.rem: log(0.005) }, SleepState.nremLight: { - SleepState.wake: log(0.05), - SleepState.nremLight: log(0.80), + SleepState.wake: log(0.12), + SleepState.nremLight: log(0.73), SleepState.nremDeep: log(0.10), SleepState.rem: log(0.05) }, SleepState.nremDeep: { - SleepState.wake: log(0.02), - SleepState.nremLight: log(0.15), + SleepState.wake: log(0.06), + SleepState.nremLight: log(0.11), // rows must sum to 1.0 (CodeRabbit #756) SleepState.nremDeep: log(0.80), SleepState.rem: log(0.03) }, SleepState.rem: { - SleepState.wake: log(0.05), + SleepState.wake: log(0.10), SleepState.nremLight: log(0.10), // INTENTIONAL HARD FORBID: REM never goes straight to deep (it passes // through light). log(0) is -infinity; the decoder only ever ADDS it to // finite emissions and compares, so it rules the move out without ever // producing NaN. SleepState.nremDeep: double.negativeInfinity, - SleepState.rem: log(0.85) + SleepState.rem: log(0.80) // rows must sum to 1.0 (CodeRabbit #756) }, }; } @@ -61,6 +61,100 @@ class HMMConfig { class SleepStageAnalyzer { double? hrMinNight; double? rmssdMeanNight; + // HYBRID baselines: motion (ENMO, g) and skin temp (centi-°C) medians from + // the opening window. 0 = absent sentinel and is EXCLUDED — a missing sensor + // must never set the baseline to "perfectly still" / "cold". + double? enmoMedianNight; + double? skinTempMedianNight; + + // THREE-LEVEL self-calibration (per user, across nights): + // - nightly baseline (computeBaseline, first 120 min) — primary, always there + // - 7-day rolling baseline — smooths single restless nights out + // - long-term baseline — the user's personal reference; a nightly baseline + // that deviates strongly from it marks the night as ATYPICAL (illness, + // alcohol, travel) instead of silently bending the thresholds. + double? hrMin7d; + double? rmssdMean7d; + double? hrMinLong; + double? rmssdMeanLong; + /// True when tonight's baseline deviates from the long-term baseline by more + /// than [atypicalZThreshold] robust z — an outlier night, lower confidence. + bool atypicalNight = false; + static const double atypicalZThreshold = 2.0; + + /// Feed the multi-night histories (trailing nightly series, oldest first). + /// [rhrHistory]/[rmssdHistory] are the persisted per-night resting-HR / + /// nocturnal-RMSSD series; the last 7 form the rolling baseline, all of them + /// the long-term one. Missing/empty history simply leaves the level unset. + void computeMultiNightBaselines({ + List rhrHistory = const [], + List rmssdHistory = const [], + List skinTempHistory = const [], + List enmoHistory = const [], + }) { + double? medianOf(List xs) { + final v = xs.where((x) => x.isFinite && x > 0).toList()..sort(); + if (v.isEmpty) return null; + return v[v.length ~/ 2]; + } + double? madOf(List xs, double? med) { + if (med == null) return null; + final dev = xs.where((x) => x.isFinite).map((x) => (x - med).abs()).toList()..sort(); + if (dev.isEmpty) return null; + return dev[dev.length ~/ 2]; + } + List last7(List xs) => xs.length > 7 ? xs.sublist(xs.length - 7) : xs; + hrMin7d = medianOf(last7(rhrHistory)); + rmssdMean7d = medianOf(last7(rmssdHistory)); + hrMinLong = medianOf(rhrHistory); + rmssdMeanLong = medianOf(rmssdHistory); + // Motion + thermal channels self-calibrate too when a nightly history + // exists (skinTempAdcHistory); without one they stay nightly-only. + skinTempMedian7d = medianOf(last7(skinTempHistory)); + skinTempMedianLong = medianOf(skinTempHistory); + // Motion history: persisted per-night ENMO medians (stable-core), so the + // motion channel self-calibrates across nights like the others. Absent + // history (pre-feature nights) simply leaves the levels unset. + enmoMedian7d = medianOfEnmo(last7(enmoHistory)); + enmoMedianLong = medianOfEnmo(enmoHistory); + // Outlier check: nightly vs long-term, robust z via MAD (1.4826*MAD ~ sd). + if (hrMinNight != null && hrMinLong != null) { + final mad = madOf(rhrHistory, hrMinLong); + final sd = mad != null && mad > 0 ? 1.4826 * mad : 3.0; // floor: 3 bpm + final z = (hrMinNight! - hrMinLong!) / sd; + if (z.abs() > atypicalZThreshold) atypicalNight = true; + } + } + + double? skinTempMedian7d; + double? skinTempMedianLong; + double? enmoMedian7d; + double? enmoMedianLong; + // ENMO medians use >= 0 as valid (a measured still night is a real 0); + // the -1 absent sentinel is excluded. + double? medianOfEnmo(List xs) { + final v = xs.where((x) => x.isFinite && x >= 0).toList()..sort(); + if (v.isEmpty) return null; + return v[v.length ~/ 2]; + } + /// Effective thermal baseline: blend nightly with 7d rolling when both exist. + double? get effectiveSkinTempMedian => + skinTempMedian7d != null && skinTempMedianNight != null + ? (skinTempMedianNight! + skinTempMedian7d!) / 2 + : (skinTempMedianNight ?? skinTempMedian7d); + /// Effective motion baseline: blend nightly (stable core) with 7d rolling. + double? get effectiveEnmoMedian => + enmoMedian7d != null && enmoMedianNight != null + ? (enmoMedianNight! + enmoMedian7d!) / 2 + : (enmoMedianNight ?? enmoMedian7d); + + /// Effective HR baseline for the emission: blend nightly with the 7-day + /// rolling level when both exist (half/half) — the night's own physiology + /// anchors it, the rolling window keeps single odd nights from dominating. + double? get effectiveHrMin => + hrMin7d != null && hrMinNight != null ? (hrMinNight! + hrMin7d!) / 2 : (hrMinNight ?? hrMin7d); + double? get effectiveRmssdMean => + rmssdMean7d != null && rmssdMeanNight != null ? (rmssdMeanNight! + rmssdMean7d!) / 2 : (rmssdMeanNight ?? rmssdMean7d); // The same plausibility window decides which samples form the baseline and // which observations get scored. A value outside it (a 250 bpm spike, a NaN @@ -68,6 +162,8 @@ class SleepStageAnalyzer { // would push the decoder towards wake. static bool _plausibleHr(double? h) => h != null && h.isFinite && h > 30.0 && h < 200.0; static bool _plausibleRmssd(double? r) => r != null && r.isFinite && r > 0.0 && r < 500.0; + static double _sigmoid(double x, double center, double scale) => + 1.0 / (1.0 + exp(-(x - center) / scale)); void computeBaseline(List> first120minData) { if (first120minData.isEmpty) return; @@ -81,41 +177,141 @@ class SleepStageAnalyzer { .where(_plausibleRmssd) .cast() .toList(); - + final enmos = first120minData + .map((d) => (d['enmo'] as num?)?.toDouble() ?? -1.0) + .where((e) => e.isFinite && e >= 0.0) + .toList(); + final temps = first120minData + .map((d) => (d['skin_temp'] as num?)?.toDouble() ?? 0.0) + .where((t) => t.isFinite && t > 0.0) + .toList(); + if (hrs.isNotEmpty) { hrs.sort(); hrMinNight = hrs[(hrs.length * 0.1).floor()]; } if (rmssds.isNotEmpty) rmssdMeanNight = rmssds.reduce((a, b) => a + b) / rmssds.length; + if (enmos.isNotEmpty) { + enmos.sort(); + enmoMedianNight = enmos[enmos.length ~/ 2]; + } + if (temps.isNotEmpty) { + temps.sort(); + skinTempMedianNight = temps[temps.length ~/ 2]; + } + } + + /// Motion (and thermal) baseline from the STABLE CORE of the night — the + /// middle 50 % by elapsed time — NOT the first 120 min. The opening window + /// contains settling-in / turning over and is systematically more active, so + /// a median taken from it would overstate the person's normal sleep motion + /// and the wake-vs-arousal threshold would sit too high. HR/RMSSD keep the + /// opening-window baseline (autonomic settling is exactly what we want there). + /// Missing ENMO (0 sentinel) is excluded — never "still". + void computeMotionBaseline(List> nightData) { + if (nightData.length < 10) return; + final t0 = (nightData.first['ts'] as num?)?.toDouble() ?? 0.0; + final t1 = (nightData.last['ts'] as num?)?.toDouble() ?? t0; + final span = t1 - t0; + if (span <= 0) return; + final core = nightData + .where((e) => ((e['ts'] as num?)?.toDouble() ?? 0.0) - t0 > span * 0.25 && + ((e['ts'] as num?)?.toDouble() ?? 0.0) - t0 < span * 0.75) + .toList(); + if (core.isEmpty) return; + final enmos = core + .map((d) => (d['enmo'] as num?)?.toDouble() ?? -1.0) + .where((e) => e.isFinite && e >= 0.0) + .toList(); + if (enmos.isNotEmpty) { + enmos.sort(); + enmoMedianNight = enmos[enmos.length ~/ 2]; + } } - double _getEmissionLogProb(SleepState state, double hr, double rmssd, {bool isContactLost = false}) { + /// HYBRID emission over ALL available channels: HR, RMSSD, ENMO (motion), + /// skin temp. Absent channels (0 sentinel) contribute a NEUTRAL factor — + /// never evidence. Wake is boosted by ANY of: elevated HR + low RMSSD + /// (autonomic arousal), high movement, warm skin (peripheral vasodilation + /// on arousal); sleep states are penalised by movement. + double _getEmissionLogProb(SleepState state, double hr, double rmssd, { + double enmo = 0.0, + double skinTemp = 0.0, + bool isContactLost = false, + }) { if (isContactLost || - !_plausibleHr(hr) || - !_plausibleRmssd(rmssd) || - hrMinNight == null || - rmssdMeanNight == null) { + effectiveHrMin == null || + effectiveRmssdMean == null) { return log(0.25); } + if (!_plausibleHr(hr) || !_plausibleRmssd(rmssd)) { + // HR/RR unusable: motion-only evidence. A clearly moving wrist cannot + // be deep sleep. If motion is ALSO absent (0 sentinel), the epoch is + // fully unmeasured — return the uniform neutral for every state so the + // gap cannot bias adjacent measured epochs towards wake (CodeRabbit #756). + if (enmo < 0) return log(0.25); // absent sentinel: fully unmeasured epoch + final motionWake = _sigmoid(enmo, 0.08, 0.03); + switch (state) { + case SleepState.wake: + return log(max(0.01, 0.2 + 0.8 * motionWake)); + case SleepState.nremDeep: + return log(max(0.01, 0.1 * (1.0 - motionWake))); + case SleepState.rem: + return log(max(0.01, 0.15 * (1.0 - motionWake))); + case SleepState.nremLight: + return log(max(0.01, 0.3)); + } + } - final hrNorm = hr / hrMinNight!; - final rmssdNorm = rmssd / rmssdMeanNight!; - - double sigmoid(double x, double center, double scale) => 1.0 / (1.0 + exp(-(x - center) / scale)); + final hrNorm = hr / effectiveHrMin!; + final rmssdNorm = rmssd / effectiveRmssdMean!; + // Motion relative to the night's own median (robust for restless + // sleepers). Absent ENMO (0 sentinel) = NO evidence either way: wake + // factor 0 (does not boost wake), sleep factor 1.0 (does not penalise). + final motionMeasured = enmo >= 0 && effectiveEnmoMedian != null && effectiveEnmoMedian! > 0; + final motion = motionMeasured ? enmo / effectiveEnmoMedian! : 1.0; + final motionWake = motionMeasured ? _sigmoid(motion, 3.0, 1.5) : 0.0; + final motionSleep = motionMeasured ? 1.0 - _sigmoid(motion, 2.0, 1.0) : 1.0; + // Skin temp vs the night median, normalised so ~100 centi-°C = 1 °C. + // Absent temp = no thermal evidence (excluded from the noisy-OR, i.e. + // factor 0, never the 0.5 that would silently boost wake) — CodeRabbit #756. + final tempMeasured = skinTemp > 0 && effectiveSkinTempMedian != null; + final tempWake = tempMeasured + ? _sigmoid((skinTemp - effectiveSkinTempMedian!) / 100.0, 1.5, 1.0) + : 0.0; double prob; switch (state) { case SleepState.wake: - prob = sigmoid(hrNorm, 1.2, 0.05) * (1.0 - sigmoid(rmssdNorm, 0.8, 0.05)); + // Noisy-OR over autonomic, motion and thermal arousal cues — any one + // strong cue can carry wake, so brief awakenings with movement but + // barely-elevated HR are still caught. + final autonomic = + _sigmoid(hrNorm, 1.15, 0.05) * (1.0 - _sigmoid(rmssdNorm, 0.8, 0.05)); + // Weak HR-only bonus for "quiet wakefulness" (issue #593 pattern: + // lying still, HR clearly above the personal sleep baseline, RMSSD + // barely moved). Capped at 0.3 and centred high (1.22×) so REM — + // whose HR/HRV is wake-like (Zemaityte et al.) — is not flipped to + // wake; REM's own emission + self-transition guard it instead. + final hrOnlyWake = _sigmoid(hrNorm, 1.22, 0.07) * 0.3; + prob = 1.0 - + (1.0 - autonomic) * + (1.0 - motionWake) * + (1.0 - tempWake) * + (1.0 - hrOnlyWake); break; case SleepState.nremDeep: - prob = (1.0 - sigmoid(hrNorm, 1.0, 0.05)) * sigmoid(rmssdNorm, 1.1, 0.05); + prob = (1.0 - _sigmoid(hrNorm, 1.0, 0.05)) * + _sigmoid(rmssdNorm, 1.1, 0.05) * + motionSleep; break; case SleepState.rem: - prob = sigmoid(hrNorm, 1.05, 0.05) * sigmoid(rmssdNorm, 1.0, 0.05); + prob = _sigmoid(hrNorm, 1.05, 0.05) * + _sigmoid(rmssdNorm, 1.0, 0.05) * + (0.5 + 0.5 * motionSleep); break; case SleepState.nremLight: - prob = 0.4; + prob = 0.4 * (0.5 + 0.5 * motionSleep); break; } return log(max(0.01, prob)); @@ -138,6 +334,8 @@ class SleepStageAnalyzer { final initProb = (state == SleepState.wake || state == SleepState.nremLight) ? log(0.5) : double.negativeInfinity; viterbi[0][i] = initProb + _getEmissionLogProb(state, hr, rmssd, + enmo: (nightData[0]['enmo'] as num?)?.toDouble() ?? 0.0, + skinTemp: (nightData[0]['skin_temp'] as num?)?.toDouble() ?? 0.0, isContactLost: nightData[0]['isContactLost'] == true); } @@ -145,10 +343,13 @@ class SleepStageAnalyzer { final hr = (nightData[t]['hr'] as num?)?.toDouble() ?? 0.0; final rmssd = (nightData[t]['rmssd'] as num?)?.toDouble() ?? 0.0; final contactLost = nightData[t]['isContactLost'] == true; + final enmo = (nightData[t]['enmo'] as num?)?.toDouble() ?? 0.0; + final skinTemp = (nightData[t]['skin_temp'] as num?)?.toDouble() ?? 0.0; for (var j = 0; j < N; j++) { final nextState = states[j]; final emission = - _getEmissionLogProb(nextState, hr, rmssd, isContactLost: contactLost); + _getEmissionLogProb(nextState, hr, rmssd, + enmo: enmo, skinTemp: skinTemp, isContactLost: contactLost); var maxLogProb = double.negativeInfinity; var bestPrevState = 0; for (var i = 0; i < N; i++) { diff --git a/lib/compute/staged_sleep_cache.dart b/lib/compute/staged_sleep_cache.dart index 67174684f..ed1f6899e 100644 --- a/lib/compute/staged_sleep_cache.dart +++ b/lib/compute/staged_sleep_cache.dart @@ -77,6 +77,7 @@ String stagingInputKey({ required List vendorNights, required (String, String)? activeWearable, bool prefixContext = false, + bool useHmmStages = false, }) { // Count local midnights, including DST days, rather than dividing by 86400. var walked = 0; @@ -116,6 +117,7 @@ String stagingInputKey({ final now = DateTime.now(); return jsonEncode([ fromSec, toSec, prefixContext ? const [] : revisions, + if (useHmmStages) 'hmm', [for (final sig in InputSignal.values) [sig.name, [for (final o in ownership[sig] ?? const []) [o.start, o.end, o.deviceId]], priority[sig]]], diff --git a/lib/compute/staging_provenance.dart b/lib/compute/staging_provenance.dart new file mode 100644 index 000000000..6277da556 --- /dev/null +++ b/lib/compute/staging_provenance.dart @@ -0,0 +1,17 @@ +/// Typed read boundary for the optional provenance in a stored sleep section. +/// Unknown method/reason strings survive newer exports; malformed values stay +/// absent. Durations are observed seconds that retained the original labels. +Map stagingProvenanceOf(Map? sleep) { + if (sleep == null) return const {}; + final method = sleep['staging_method']; + final effective = sleep['staging_effective_method']; + final fallback = sleep['staging_fallback']; + final sec = sleep['staging_fallback_sec']; + return { + if (method is String) 'staging_method': method, + if (effective is String) 'staging_effective_method': effective, + if (fallback is String) 'staging_fallback': fallback, + if (fallback is String && sec is num && sec.isFinite && sec >= 0) + 'staging_fallback_sec': sec.round(), + }; +} diff --git a/lib/data/local_repository_impl.dart b/lib/data/local_repository_impl.dart index 4d56f8a52..2cb22602c 100644 --- a/lib/data/local_repository_impl.dart +++ b/lib/data/local_repository_impl.dart @@ -20,6 +20,7 @@ import 'dart:math' as math; import 'package:flutter/foundation.dart' show visibleForTesting; +import '../compute/staging_provenance.dart'; import '../compute/derivation_engine.dart'; import '../compute/findings.dart' show servedReadiness; import '../compute/hr_max.dart'; @@ -1175,6 +1176,7 @@ class LocalRepositoryImpl extends LocalRepository { // Shape matches sleep_detail_screen's contract exactly. 'has_sleep': true, 'sleep_source': sleepSource, + ...stagingProvenanceOf(_sub(b, 'sleep')), 'duration_min': (tst / 60).round(), 'in_bed_min': spt == null ? null : (spt / 60).round(), 'awake_min': waso == null ? null : (waso / 60).round(), diff --git a/test/ed2a_substrate_transfer_test.dart b/test/ed2a_substrate_transfer_test.dart index 2c727d87e..b32de979b 100644 --- a/test/ed2a_substrate_transfer_test.dart +++ b/test/ed2a_substrate_transfer_test.dart @@ -241,7 +241,7 @@ void main() { markTestSkipped('committed golden is UTC; run this test with TZ=UTC'); return; } - expect(kAlgoVersion, 124, reason: 'ED-2b repins analytics; transfer remains exact'); + expect(kAlgoVersion, 125, reason: 'ED-2b repins analytics; transfer remains exact'); final fixture = _fixture(); final baseline = substrateFromDecodedPage(fixture.frames, fixture.rr); final transferred = Substrate.fromTransfer(baseline.toJson()); diff --git a/test/ed2b_algo_version_114_test.dart b/test/ed2b_algo_version_114_test.dart index b3de3bd82..45baf2506 100644 --- a/test/ed2b_algo_version_114_test.dart +++ b/test/ed2b_algo_version_114_test.dart @@ -6,7 +6,7 @@ import 'package:openstrap_edge/compute/derivation_engine.dart'; void main() { // WHY: finalized days need a new version when the analytics pin changes outputs. test('ED-2b adoption and workout rest repin name their output changes', () { - expect(kAlgoVersion, 124); + expect(kAlgoVersion, 125); expect(kAnalyticsPin, 'a438df40b38a4586a52a68ab7e1c100fca111ccd'); final src = File('lib/compute/derivation_engine.dart').readAsStringSync(); final at = src.indexOf('// 121 → 122'); diff --git a/test/hmm_staging_flag_cache_test.dart b/test/hmm_staging_flag_cache_test.dart new file mode 100644 index 000000000..328538b10 --- /dev/null +++ b/test/hmm_staging_flag_cache_test.dart @@ -0,0 +1,96 @@ +import 'dart:convert'; + +import 'package:openstrap_edge/data/db.dart'; +import 'package:openstrap_edge/data/series_codec.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:shared_preferences/shared_preferences.dart'; +import 'package:openstrap_edge/compute/derivation_engine.dart'; +import 'package:openstrap_edge/compute/derive_prepare.dart'; + +import 'support/ed2a_recalc_fixture.dart'; + +void main() { + setUpAll(edInit); + setUp(() async { + await edFresh('hmm_flag_cache.db'); + await edRows(edStart, edStart + 120, hr: 60, az: 1); + }); + tearDown(edDrop); + + // why: the actual engine must persist the captured choice and replace old provenance. + // # catches: wiring the flag into debug preparation but not the bundle worker. + test('toggling re-derives only selected nights and clears stored fallback', () async { + await edRows(edStart, edStart + 600, hr: 55, az: 1); + await LocalDb.putSleepOverride(dayId: edDay, onsetTs: edStart, + offsetTs: edStart + 600, source: 'manual'); + await LocalDb.putDayResult(dayId: '2019-01-01', algoVersion: kAlgoVersion, + payloadJson: '{"date":"2019-01-01","sentinel":42}', windowJson: '{}'); + final e = DerivationEngine(); + final prefs = await SharedPreferences.getInstance(); + Future> sleep() async { + final row = await LocalDb.dayResult(edDay); + final payload = SeriesCodec.decodePayloadJson(row!['payload_json'] as String)!; + return (payload['sleep'] as Map).cast(); + } + expect(await e.runDays(edProfile, {edDay}), 1); + final cardio = await sleep(); + expect(cardio.containsKey('staging_method'), isFalse); + await prefs.setBool(kSleepHmmStagesPref, true); + expect(await e.runDays(edProfile, {edDay}), 1); + final fallback = await sleep(); + expect(fallback['staging_effective_method'], 'cardio'); + expect(fallback['staging_fallback'], 'hmm_unavailable'); + await prefs.setBool(kSleepHmmStagesPref, false); + expect(await e.runDays(edProfile, {edDay}), 1); + final off = await sleep(); + expect(off.containsKey('staging_method'), isFalse); + expect(off.containsKey('staging_fallback'), isFalse); + final other = await LocalDb.dayResult('2019-01-01'); + expect(jsonDecode(other!['payload_json'] as String), + {'date': '2019-01-01', 'sentinel': 42}); + }); + + // why: imported substrate uses a separate preparation path into the same worker. + // # catches: defaulting imports to cardio after moving the flag into prepared input. + test('imported days carry the enabled flag into the bundle worker', () async { + await (await SharedPreferences.getInstance()).setBool(kSleepHmmStagesPref, true); + final choices = []; + final e = DerivationEngine()..debugOnNightBundle = (input, _) { + choices.add(input['use_hmm_stages']); + }; + expect(await e.deriveImportedDays(edSmallSub(count: 120), edProfile, {edDay}), 1); + expect(choices, [true]); + }); + + // why: HMM worker inputs and cached candidates must agree on one flag value. + // # catches: reading the flag only after preparation or omitting it from the key. + test('toggle invalidates requested nights once and preserves unrelated cache hits', () async { + final e = DerivationEngine(); + for (final day in [edDay, '2020-01-03']) { + final prepared = await e.debugPrepareDay(day); + expect(prepared!.toJson()['use_hmm_stages'] ?? false, isFalse); + } + final initial = e.debugStagingComputations; + expect(initial, 2); + final prefs = await SharedPreferences.getInstance(); + await prefs.setBool('unrelated_preference', true); + await e.debugPrepareDay(edDay); + expect(e.debugStagingComputations, initial); + await prefs.setBool(kSleepHmmStagesPref, true); + final prepared = await e.debugPrepareDay(edDay); + expect(prepared!.toJson()['use_hmm_stages'], true); + final restored = PreparedDerivationDay.fromJson(prepared.toJson()); + expect(restored.toJson()['use_hmm_stages'], true); + expect(e.debugStagingComputations, initial + 1); + await e.debugPrepareDay(edDay); + expect(e.debugStagingComputations, initial + 1); + await e.debugPrepareDay('2020-01-03'); + expect(e.debugStagingComputations, initial + 2); + await prefs.setBool(kSleepHmmStagesPref, false); + final off = await e.debugPrepareDay(edDay); + expect(off!.toJson()['use_hmm_stages'] ?? false, isFalse); + expect(e.debugStagingComputations, initial + 3); + await e.debugPrepareDay(edDay); + expect(e.debugStagingComputations, initial + 3); + }); +} diff --git a/test/hmm_staging_provenance_read_test.dart b/test/hmm_staging_provenance_read_test.dart new file mode 100644 index 000000000..19a3a35af --- /dev/null +++ b/test/hmm_staging_provenance_read_test.dart @@ -0,0 +1,242 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:path/path.dart' as p; +import 'package:path_provider_platform_interface/path_provider_platform_interface.dart'; +import 'package:shared_preferences/shared_preferences.dart'; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; + +import 'package:openstrap_edge/compute/derivation_engine.dart' show kAlgoVersion; +import 'package:openstrap_edge/data/db.dart'; +import 'package:openstrap_edge/data/local_repository_impl.dart'; +import 'package:openstrap_edge/data/series_codec.dart'; + +const _day = '2026-05-20'; + +class _FakePathProvider extends PathProviderPlatform { + _FakePathProvider(this.root); + final String root; + @override + Future getTemporaryPath() async => root; + @override + Future getApplicationSupportPath() async => root; + @override + Future getApplicationDocumentsPath() async => root; + @override + Future getApplicationCachePath() async => root; + @override + Future getLibraryPath() async => root; + @override + Future getDownloadsPath() async => root; +} + +Future _put(String day, Map staging) => + LocalDb.putDayResult( + dayId: day, + algoVersion: kAlgoVersion, + payloadJson: jsonEncode({ + 'date': day, + 'sleep_source': 'auto', + 'sleep': { + 'accounting': { + 'value': { + 'tst_sec': 25200, + 'waso_sec': 600, + 'light_sec': 15000, + 'deep_sec': 4800, + 'rem_sec': 5400, + }, + }, + ...staging, + }, + }), + windowJson: '{}', + ); + +const _fallback = { + 'staging_method': 'hmm', + 'staging_effective_method': 'cardio', + 'staging_fallback': 'hmm_unavailable', + 'staging_fallback_sec': 28437, + 'staging_note': 'claim and fallback', +}; + +const _keys = [ + 'staging_method', + 'staging_effective_method', + 'staging_fallback', + 'staging_fallback_sec', +]; + +Map _picked(Map night) => { + for (final k in _keys) + if (night.containsKey(k)) k: night[k], + }; + +void main() { + TestWidgetsFlutterBinding.ensureInitialized(); + late LocalRepositoryImpl repo; + late Directory tmp; + + setUpAll(() async { + tmp = await Directory.systemTemp.createTemp('openstrap_hmm_prov_'); + PathProviderPlatform.instance = _FakePathProvider(tmp.path); + sqfliteFfiInit(); + databaseFactory = databaseFactoryFfi; + LocalDb.dbName = 'openstrap_hmm_provenance_read_test.db'; + await databaseFactory.deleteDatabase( + p.join(await databaseFactory.getDatabasesPath(), LocalDb.dbName), + ); + repo = LocalRepositoryImpl(getProfileMap: () => const {}); + }); + + tearDownAll(() async { + await LocalDb.close(); + await databaseFactory.deleteDatabase( + p.join(await databaseFactory.getDatabasesPath(), LocalDb.dbName), + ); + if (await tmp.exists()) await tmp.delete(recursive: true); + }); + + setUp(() async { + SharedPreferences.setMockInitialValues({}); + final db = await LocalDb.instance; + await db.delete('day_result'); + }); + + // why: stored provenance must survive reads and exports without fabrication. + // # catches: dropping or blindly casting the stored provenance keys. + test('the night map carries the provenance of a fallback night', () async { + await _put(_day, _fallback); + final n = await repo.getDaySleepV2(_day); + expect(_picked(n), { + 'staging_method': 'hmm', + 'staging_effective_method': 'cardio', + 'staging_fallback': 'hmm_unavailable', + 'staging_fallback_sec': 28437, + }); + }); + + // why: stored provenance must survive reads and exports without fabrication. + // # catches: dropping or blindly casting the stored provenance keys. + test('an ordinary experimental night has the method and no fallback', + () async { + await _put(_day, { + 'staging_method': 'hmm', + 'staging_effective_method': 'hmm', + }); + final n = await repo.getDaySleepV2(_day); + expect(_picked(n), { + 'staging_method': 'hmm', + 'staging_effective_method': 'hmm', + }); + }); + + // why: stored provenance must survive reads and exports without fabrication. + // # catches: dropping or blindly casting the stored provenance keys. + test('an older payload without the keys reads as before: nothing invented', + () async { + await _put(_day, {'staging_note': 'old'}); + final n = await repo.getDaySleepV2(_day); + expect(n['has_sleep'], isTrue); + expect(_picked(n), isEmpty); + // An older experimental night: the requested method is all it has. + await _put('2026-05-21', {'staging_method': 'hmm'}); + expect(_picked(await repo.getDaySleepV2('2026-05-21')), + {'staging_method': 'hmm'}); + }); + + // why: stored provenance must survive reads and exports without fabrication. + // # catches: dropping or blindly casting the stored provenance keys. + test('an unknown fallback value is passed on as text, never as a method', + () async { + await _put(_day, { + ..._fallback, + 'staging_fallback': 'hmm_from_the_future', + }); + final n = await repo.getDaySleepV2(_day); + expect(n['staging_fallback'], 'hmm_from_the_future'); + expect(n['has_sleep'], isTrue, reason: 'the estimate is still shown'); + }); + + // why: stored provenance must survive reads and exports without fabrication. + // # catches: dropping or blindly casting the stored provenance keys. + test('a hand-edited row degrades to unknown, never to a wrong value', + () async { + await _put(_day, { + 'staging_method': 'hmm', + 'staging_effective_method': 7, + 'staging_fallback': {'x': 1}, + 'staging_fallback_sec': -5, + }); + final n = await repo.getDaySleepV2(_day); + expect(_picked(n), {'staging_method': 'hmm'}); + await _put('2026-05-21', { + 'staging_method': 'hmm', + 'staging_fallback': 'hmm_unavailable', + 'staging_fallback_sec': 'soon', + }); + final m = await repo.getDaySleepV2('2026-05-21'); + expect(m['staging_fallback'], 'hmm_unavailable'); + expect(m.containsKey('staging_fallback_sec'), isFalse); + }); + + Future> storedSleep(String dbPath, String day) async { + final out = await databaseFactory.openDatabase( + dbPath, + options: OpenDatabaseOptions(readOnly: true, singleInstance: false), + ); + try { + final rows = await out.query('day_result', + where: 'day_id = ?', whereArgs: [day]); + expect(rows, hasLength(1)); + final payload = SeriesCodec.decodePayloadJson( + rows.single['payload_json'] as String)!; + return (payload['sleep'] as Map).cast(); + } finally { + await out.close(); + } + } + + // why: stored provenance must survive reads and exports without fabrication. + // # catches: dropping or blindly casting the stored provenance keys. + test('a selected-day export carries the keys and a restore brings them back', + () async { + await _put(_day, _fallback); + await _put('2026-05-22', {'staging_method': 'cardio'}); + final path = await LocalDb.exportDaysDb({_day}); + addTearDown(() => databaseFactory.deleteDatabase(path)); + final sleep = await storedSleep(path, _day); + for (final k in _keys) { + expect(sleep[k], _fallback[k], reason: k); + } + + final db = await LocalDb.instance; + await db.delete('day_result'); + await LocalDb.importFromDbFile(path); + expect(_picked(await repo.getDaySleepV2(_day)), { + for (final k in _keys) k: _fallback[k], + }); + }); + + // why: stored provenance must survive reads and exports without fabrication. + // # catches: dropping or blindly casting the stored provenance keys. + test('a full export carries the keys and a restore brings them back', + () async { + await _put(_day, _fallback); + final path = await LocalDb.exportCopy(); + addTearDown(() => databaseFactory.deleteDatabase(path)); + final sleep = await storedSleep(path, _day); + for (final k in _keys) { + expect(sleep[k], _fallback[k], reason: k); + } + + final db = await LocalDb.instance; + await db.delete('day_result'); + await LocalDb.importFromDbFile(path); + expect(_picked(await repo.getDaySleepV2(_day)), { + for (final k in _keys) k: _fallback[k], + }); + }); +} diff --git a/test/hmm_staging_provenance_test.dart b/test/hmm_staging_provenance_test.dart new file mode 100644 index 000000000..4a7463fc3 --- /dev/null +++ b/test/hmm_staging_provenance_test.dart @@ -0,0 +1,117 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:crypto/crypto.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/compute/derivation_engine.dart' show kAlgoVersion; +import 'package:openstrap_edge/compute/onehz_pipeline.dart'; +import 'package:openstrap_edge/compute/staging_provenance.dart'; + +import 'support/hmm_night_fixture.dart'; +import 'support/ed2a_recalc_fixture.dart' show edSetZone; + +void main() { + // why: the upstream cardio result must remain byte-identical after this port. + // # catches: unconditional HMM provenance on flag-off nights. + test('cardio payload remains byte-identical to 756 and has no HMM keys', () { + // The full payload includes local clock fields; its baseline is UTC. + edSetZone('UTC'); + addTearDown(() => edSetZone(Platform.environment['TZ'] ?? 'UTC')); + final off = hmmNight(enabled: false); + final implicit = {...off}..remove('use_hmm_stages'); + final stored = jsonEncode(deriveDayBundle(off)); + expect(stored, jsonEncode(deriveDayBundle(implicit))); + // Captured at 90a879b21 with this six-minute fixture, before production edits. + expect(sha256.convert(utf8.encode(stored)).toString(), + '01c1d05988233c2902d97e48a3bde7ca2a9261f1c84ab7d357f658ca3270a1d7'); + final sleep = deriveDayBundle(off)['sleep'] as Map; + expect(sleep.keys.where((k) => k.toString().startsWith('staging_') || + k.toString().startsWith('stager_')), isEmpty); + }); + + // why: immutable day rows need a new version before provenance can be stored. + // # catches: leaving the producing algorithm at 124. + test('provenance has a fresh algorithm version', () { + expect(kAlgoVersion, 125); + }); + + // why: opting in is not evidence that HMM produced the labels. + // # catches: tagging a whole-night abstention as effective HMM. + test('whole-night abstention stores cardio fallback and observed seconds', () { + final sleep = deriveDayBundle(hmmNight(rr: false))['sleep'] as Map; + expect(sleep['staging_method'], 'hmm'); + expect(sleep['staging_effective_method'], 'cardio'); + expect(sleep['staging_fallback'], 'hmm_unavailable'); + expect(sleep['staging_fallback_sec'], 360); + }); + + // why: unobserved gaps cannot be claimed as cardio fallback time. + // # catches: using the window length as the fallback duration. + test('whole-night fallback excludes unobserved seconds', () { + final input = hmmNight(rr: false); + input['hypno_stages'] = [for (var i = 0; i < 360; i++) + i < 30 ? 'unobserved' : 'light']; + final sleep = deriveDayBundle(input)['sleep'] as Map; + expect(sleep['staging_fallback_sec'], 330); + }); + + // why: a mixed night must report only seconds that kept cardio labels. + // # catches: counting every second or none when one epoch abstains. + test('partial fallback counts the two missing-HR epochs', () { + final sleep = deriveDayBundle(hmmNight(gap: true))['sleep'] as Map; + expect(sleep['staging_effective_method'], 'hmm_cardio'); + expect(sleep['staging_fallback'], 'hmm_epoch_unavailable'); + expect(sleep['staging_fallback_sec'], 60); + }); + + // why: an HMM abstention can retain a paired device's own labels. + // # catches: reporting every retained label as cardio regardless of source. + test('fallback retains the vendor method in provenance', () { + final input = hmmNight(rr: false)..['sleep_source'] = 'vendor_staged'; + final sleep = deriveDayBundle(input)['sleep'] as Map; + expect(sleep['staging_effective_method'], 'vendor_staged'); + expect(sleep['staging_fallback_sec'], 360); + }); + + // why: mixed staging must name the method supplying the fallback labels. + // # catches: labelling a mixed vendor/HMM night as hmm_cardio. + test('partial vendor fallback names both methods', () { + final input = hmmNight(gap: true)..['sleep_source'] = 'vendor_staged'; + final sleep = deriveDayBundle(input)['sleep'] as Map; + expect(sleep['staging_effective_method'], 'hmm_vendor_staged'); + expect(sleep['staging_fallback_sec'], 60); + }); + + // why: a successful recalculation cannot inherit an earlier fallback. + // # catches: forwarding stale fallback keys from the incoming sleep map. + test('successful HMM clears stale fallback metadata', () { + final input = hmmNight(); + input['sleep_json'] = { + ...input['sleep_json'] as Map, + 'staging_fallback': 'old', 'staging_fallback_sec': 360}; + final sleep = deriveDayBundle(input)['sleep'] as Map; + expect(sleep['staging_method'], 'hmm'); + expect(sleep['staging_effective_method'], 'hmm'); + expect(sleep.containsKey('staging_fallback'), isFalse); + expect(sleep.containsKey('staging_fallback_sec'), isFalse); + }); + + // why: old and malformed backups carry unknown metadata, never fake seconds. + // # catches: unchecked casts, inferred zero, or discarding future reasons. + test('typed parsing tolerates legacy, malformed and future provenance', () { + expect(stagingProvenanceOf(null), isEmpty); + expect(stagingProvenanceOf({}), isEmpty); + expect(stagingProvenanceOf({'staging_method': 1, + 'staging_effective_method': false, 'staging_fallback': [], + 'staging_fallback_sec': 10}), isEmpty); + for (final sec in [-1, double.nan, double.infinity, '10', null]) { + expect(stagingProvenanceOf({'staging_fallback': 'future', + 'staging_fallback_sec': sec}), {'staging_fallback': 'future'}); + } + expect(stagingProvenanceOf({'staging_method': 'hmm', + 'staging_effective_method': 'future', 'staging_fallback': 'future', + 'staging_fallback_sec': 30.4}), {'staging_method': 'hmm', + 'staging_effective_method': 'future', 'staging_fallback': 'future', + 'staging_fallback_sec': 30}); + }); +} diff --git a/test/sleep_hmm_aasm_wake_test.dart b/test/sleep_hmm_aasm_wake_test.dart new file mode 100644 index 000000000..02e762126 --- /dev/null +++ b/test/sleep_hmm_aasm_wake_test.dart @@ -0,0 +1,90 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/compute/sleep_pipeline_hmm.dart'; + +Map epoch(double hr, double rmssd, {double enmo = 0.0, double skinTemp = 0.0}) => + {'hr': hr, 'rmssd': rmssd, 'enmo': enmo, 'skin_temp': skinTemp}; + +SleepStageAnalyzer baselined() { + final a = SleepStageAnalyzer(); + a.computeBaseline(List.generate(7200, (_) => epoch(55, 45, enmo: 0.01, skinTemp: 3350))); + return a; +} + +/// Mirror of the AASM 15 s minimum-wake-run rule applied in _hmmRestage. +List applyAasmWakeRule(List labels, {int minWakeRunSec = 15}) { + final merged = [...labels]; + var i = 0; + while (i < merged.length) { + if (merged[i] != 'wake') { + i++; + continue; + } + var j = i; + while (j < merged.length && merged[j] == 'wake') { + j++; + } + if (j - i < minWakeRunSec) { + final surrounding = i > 0 ? merged[i - 1] : 'light'; + for (var k = i; k < j; k++) { + merged[k] = surrounding == 'unobserved' ? 'light' : surrounding; + } + } + i = j; + } + return merged; +} + +void main() { + test('AASM: short wake run (<15 s) is an arousal, reverts to sleep stage', () { + // 10 s of wake between deep phases must NOT stay wake (AASM: arousal + // >=3 s but < half a 30 s epoch is not stage W). + final a = baselined(); + final night = >[ + ...List.generate(30, (_) => epoch(53, 50, enmo: 0.005, skinTemp: 3350)), // deep + ...List.generate(10, (_) => epoch(70, 25, enmo: 0.3, skinTemp: 3450)), // brief arousal + ...List.generate(30, (_) => epoch(53, 50, enmo: 0.005, skinTemp: 3350)), // deep + ]; + final states = a.viterbiDecode(night); + final labels = states.map((s) => switch (s) { + SleepState.wake => 'wake', + SleepState.nremLight => 'light', + SleepState.nremDeep => 'deep', + SleepState.rem => 'rem', + }).toList(); + final cleaned = applyAasmWakeRule(labels); + expect(cleaned.sublist(30, 40).where((l) => l == 'wake').length, lessThan(10), + reason: 'a 10 s wake burst is an arousal, not wake'); + }); + + test('AASM: long wake run (>=15 s) stays wake', () { + final a = baselined(); + final night = >[ + ...List.generate(30, (_) => epoch(53, 50, enmo: 0.005, skinTemp: 3350)), + ...List.generate(60, (_) => epoch(85, 15, enmo: 0.4, skinTemp: 3500)), // real awakening + ...List.generate(30, (_) => epoch(53, 50, enmo: 0.005, skinTemp: 3350)), + ]; + final states = a.viterbiDecode(night); + final labels = states.map((s) => switch (s) { + SleepState.wake => 'wake', + SleepState.nremLight => 'light', + SleepState.nremDeep => 'deep', + SleepState.rem => 'rem', + }).toList(); + final cleaned = applyAasmWakeRule(labels); + expect(cleaned.sublist(30, 90).where((l) => l == 'wake').length, greaterThan(30), + reason: 'a sustained awakening must remain wake (counts to WASO)'); + }); + + test('AASM: staging continues after wake — no restart (deep resumes as deep)', () { + final a = baselined(); + final night = >[ + ...List.generate(30, (_) => epoch(53, 50, enmo: 0.005, skinTemp: 3350)), + ...List.generate(30, (_) => epoch(85, 15, enmo: 0.4, skinTemp: 3500)), + ...List.generate(30, (_) => epoch(53, 50, enmo: 0.005, skinTemp: 3350)), + ]; + final states = a.viterbiDecode(night); + // After the wake block, the deep-sleep signature must decode as deep again + // (Viterbi continuity), not be forced back to light/wake. + expect(states[75], SleepState.nremDeep); + }); +} diff --git a/test/sleep_hmm_hybrid_test.dart b/test/sleep_hmm_hybrid_test.dart new file mode 100644 index 000000000..99f7f9bbb --- /dev/null +++ b/test/sleep_hmm_hybrid_test.dart @@ -0,0 +1,117 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/compute/sleep_pipeline_hmm.dart'; + +Map epoch(double hr, double rmssd, {double enmo = 0.0, double skinTemp = 0.0}) => + {'hr': hr, 'rmssd': rmssd, 'enmo': enmo, 'skin_temp': skinTemp}; + +SleepStageAnalyzer baselined() { + final a = SleepStageAnalyzer(); + // 120 min opening window: calm sleep baseline, HR ~55, RMSSD ~45, + // low motion (enmo ~0.01), skin temp ~3350 (33.5 °C). + a.computeBaseline(List.generate(7200, (_) => epoch(55, 45, enmo: 0.01, skinTemp: 3350))); + return a; +} + +void main() { + test('hybrid: movement-only arousal (HR barely up) is detected as wake', () { + final a = baselined(); + final night = >[ + ...List.generate(30, (_) => epoch(54, 46, enmo: 0.01, skinTemp: 3350)), + // 60 s of tossing/turning: HR only +3, but ENMO clearly elevated. + ...List.generate(60, (_) => epoch(58, 40, enmo: 0.25, skinTemp: 3400)), + ...List.generate(30, (_) => epoch(54, 46, enmo: 0.01, skinTemp: 3350)), + ]; + final states = a.viterbiDecode(night); + final wakeDuringArousal = states.sublist(30, 90).where((s) => s == SleepState.wake).length; + expect(wakeDuringArousal, greaterThan(20), + reason: 'motion-only arousal should yield wake, not light'); + }); + + test('hybrid: quiet sleep stays sleep even with normal HR/RMSSD', () { + final a = baselined(); + final night = List.generate(120, (_) => epoch(56, 44, enmo: 0.008, skinTemp: 3345)); + final states = a.viterbiDecode(night); + expect(states.where((s) => s == SleepState.wake).length, lessThan(10)); + }); + + test('hybrid: high HR arousal is still wake', () { + final a = baselined(); + final night = >[ + ...List.generate(30, (_) => epoch(54, 46, enmo: 0.01, skinTemp: 3350)), + ...List.generate(60, (_) => epoch(85, 18, enmo: 0.02, skinTemp: 3350)), + ...List.generate(30, (_) => epoch(54, 46, enmo: 0.01, skinTemp: 3350)), + ]; + final states = a.viterbiDecode(night); + expect(states.sublist(30, 90).where((s) => s == SleepState.wake).length, greaterThan(20)); + }); + + test('hybrid: absent ENMO (0 sentinel) is neutral, never "still"', () { + final a = baselined(); + // enmo 0 everywhere: baseline has no motion median -> motion factor neutral. + final night = List.generate(60, (_) => epoch(54, 46, enmo: -1.0, skinTemp: 3350)); // absent sentinel + final states = a.viterbiDecode(night); + expect(states.where((s) => s == SleepState.wake).length, lessThan(10), + reason: 'absent motion must not be read as evidence either way'); + }); + + test('hybrid: unusable HR falls back to motion-only evidence', () { + final a = baselined(); + final night = >[ + ...List.generate(30, (_) => epoch(0, 0, enmo: -1.0)), + ...List.generate(60, (_) => epoch(0, 0, enmo: 0.3)), // moving, no HR + ...List.generate(30, (_) => epoch(0, 0, enmo: -1.0)), + ]; + final states = a.viterbiDecode(night); + expect(states.sublist(30, 90).where((s) => s == SleepState.wake).length, greaterThan(20)); + }); + + test('neutral: fully missing epoch (no HR/RR, absent ENMO) is uniform, not wake-biased', () { + final a = baselined(); + final night = >[ + ...List.generate(30, (_) => epoch(54, 46, enmo: 0.01, skinTemp: 3350)), + ...List.generate(60, (_) => epoch(0, 0, enmo: -1.0)), // gap: nothing measured + ...List.generate(30, (_) => epoch(54, 46, enmo: 0.01, skinTemp: 3350)), + ]; + final states = a.viterbiDecode(night); + // The gap must not drag the surrounding sleep into wake. + expect(states.where((s) => s == SleepState.wake).length, lessThan(10), + reason: 'a fully unmeasured stretch must stay neutral, never wake'); + }); + + test('neutral: absent temp/motion do not add wake evidence (noisy-OR excludes them)', () { + final a = baselined(); + // same HR/RMSSD, but optional sensors absent -> wake prob must not exceed + // the case where they are present-and-calm + final withSensors = a.viterbiDecode(List.generate(60, (_) => epoch(60, 50, enmo: 0.01, skinTemp: 3350))); + final withoutSensors = a.viterbiDecode(List.generate(60, (_) => epoch(60, 50, enmo: -1.0, skinTemp: 0))); + final wakeWith = withSensors.where((s) => s == SleepState.wake).length; + final wakeWithout = withoutSensors.where((s) => s == SleepState.wake).length; + expect(wakeWithout, lessThanOrEqualTo(wakeWith), + reason: 'absent optional sensors must be neutral, never extra wake evidence'); + }); + + test('hr-only bonus: quiet wakefulness (still, HR +14 bpm over baseline) -> wake', () { + final a = baselined(); // baseline HR 55 + final night = >[ + ...List.generate(30, (_) => epoch(54, 46, enmo: 0.005, skinTemp: 3350)), + // lying still and awake: HR 69 (~1.25x), RMSSD near normal, no motion + ...List.generate(60, (_) => epoch(69, 44, enmo: 0.002, skinTemp: 3350)), + ...List.generate(30, (_) => epoch(54, 46, enmo: 0.005, skinTemp: 3350)), + ]; + final states = a.viterbiDecode(night); + expect(states.sublist(30, 90).where((s) => s == SleepState.wake).length, greaterThan(20), + reason: 'still but clearly elevated HR must read as wake (quiet wakefulness)'); + }); + + test('hr-only bonus guarded: REM-like (HR 1.22x, still, RMSSD normal) stays non-wake', () { + final a = baselined(); + final night = >[ + ...List.generate(30, (_) => epoch(54, 46, enmo: 0.005, skinTemp: 3350)), + ...List.generate(60, (_) => epoch(67, 45, enmo: 0.004, skinTemp: 3350)), // REM-like HR spike, still + ...List.generate(30, (_) => epoch(54, 46, enmo: 0.005, skinTemp: 3350)), + ]; + final states = a.viterbiDecode(night); + expect(states.sublist(30, 90).where((s) => s == SleepState.wake).length, lessThan(30), + reason: 'REM-like HR with atonia must not flip to wake (REM guard)'); + }); +} diff --git a/test/sleep_hmm_motion_baseline_test.dart b/test/sleep_hmm_motion_baseline_test.dart new file mode 100644 index 000000000..e634b1ee7 --- /dev/null +++ b/test/sleep_hmm_motion_baseline_test.dart @@ -0,0 +1,30 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/compute/sleep_pipeline_hmm.dart'; + +Map epoch(double hr, double rmssd, double ts, {double enmo = 0.0, double skinTemp = 0.0}) => + {'hr': hr, 'rmssd': rmssd, 'ts': ts, 'enmo': enmo, 'skin_temp': skinTemp}; + +void main() { + test('motion baseline uses the stable core, not the restless opening', () { + final a = SleepStageAnalyzer(); + // 8 h night: first 2 h restless (enmo 0.2 — settling in), core calm (0.01). + final night = >[ + for (var t = 0; t < 7200; t++) epoch(55, 45, t.toDouble(), enmo: 0.2, skinTemp: 3350), + for (var t = 7200; t < 21600; t++) epoch(55, 45, t.toDouble(), enmo: 0.01, skinTemp: 3350), + for (var t = 21600; t < 28800; t++) epoch(55, 45, t.toDouble(), enmo: 0.2, skinTemp: 3350), + ]; + // opening-window baseline (old behavior) would see enmo 0.2: + a.computeBaseline(night.take(7200).toList()); + expect(a.enmoMedianNight, 0.2); + // core-based motion baseline sees the calm middle 50 %: + a.computeMotionBaseline(night); + expect(a.enmoMedianNight, 0.01); + }); + + test('motion baseline: absent ENMO (0 sentinel) never becomes the baseline', () { + final a = SleepStageAnalyzer(); + final night = [for (var t = 0; t < 1000; t++) epoch(55, 45, t.toDouble(), enmo: -1.0)]; + a.computeMotionBaseline(night); + expect(a.enmoMedianNight, isNull); + }); +} diff --git a/test/sleep_hmm_multinight_test.dart b/test/sleep_hmm_multinight_test.dart new file mode 100644 index 000000000..f490f93d6 --- /dev/null +++ b/test/sleep_hmm_multinight_test.dart @@ -0,0 +1,69 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/compute/sleep_pipeline_hmm.dart'; + +Map epoch(double hr, double rmssd, {double enmo = 0.0, double skinTemp = 0.0}) => + {'hr': hr, 'rmssd': rmssd, 'enmo': enmo, 'skin_temp': skinTemp}; + +SleepStageAnalyzer baselined() { + final a = SleepStageAnalyzer(); + a.computeBaseline(List.generate(7200, (_) => epoch(55, 45, enmo: 0.01, skinTemp: 3350))); + return a; +} + +void main() { + test('multi-night: 7d and long-term baselines are computed from histories', () { + final a = baselined(); + final rhr = List.generate(30, (_) => 55.0); + final rmssd = List.generate(30, (_) => 45.0); + a.computeMultiNightBaselines(rhrHistory: rhr, rmssdHistory: rmssd); + expect(a.hrMin7d, 55.0); + expect(a.hrMinLong, 55.0); + expect(a.rmssdMean7d, 45.0); + expect(a.atypicalNight, isFalse); + // effective baseline blends nightly + 7d + expect(a.effectiveHrMin, 55.0); + }); + + test('multi-night: a deviating night is flagged atypical (outlier detection)', () { + final a = SleepStageAnalyzer(); + // tonight's baseline is 12 bpm higher than the personal norm (illness/alcohol) + a.computeBaseline(List.generate(7200, (_) => epoch(67, 45, enmo: 0.01, skinTemp: 3350))); + final rhr = List.generate(30, (_) => 55.0); + a.computeMultiNightBaselines(rhrHistory: rhr, rmssdHistory: List.filled(30, 45.0)); + expect(a.atypicalNight, isTrue, + reason: 'HR baseline +12 bpm vs long-term with tiny MAD must be an outlier night'); + }); + + test('multi-night: no history -> no 7d/long-term, nightly baseline still works', () { + final a = baselined(); + a.computeMultiNightBaselines(); + expect(a.hrMin7d, isNull); + expect(a.hrMinLong, isNull); + expect(a.effectiveHrMin, 55.0); // falls back to nightly + expect(a.atypicalNight, isFalse); + }); + + test('multi-night: normal variation within long-term spread is NOT atypical', () { + final a = baselined(); + // long-term history with realistic spread (±3 bpm), tonight at +2 + final rhr = [52.0, 53.0, 54.0, 55.0, 56.0, 57.0, 58.0, ...List.filled(23, 55.0)]; + a.computeMultiNightBaselines(rhrHistory: rhr, rmssdHistory: List.filled(30, 45.0)); + expect(a.atypicalNight, isFalse); + }); + + test('multi-night: ENMO history feeds 7d/long-term motion calibration', () { + final a = baselined(); + final enmo = List.generate(30, (_) => 0.02); + a.computeMultiNightBaselines(enmoHistory: enmo); + expect(a.enmoMedian7d, 0.02); + expect(a.enmoMedianLong, 0.02); + // blended with the nightly core median (0.01 from baselined()) + expect(a.effectiveEnmoMedian, closeTo(0.015, 1e-9)); + }); + + test('multi-night: ENMO absent sentinel (-1) is excluded from the history median', () { + final a = baselined(); + a.computeMultiNightBaselines(enmoHistory: [-1.0, -1.0, 0.02, 0.02]); + expect(a.enmoMedianLong, 0.02); + }); +} diff --git a/test/support/hmm_night_fixture.dart b/test/support/hmm_night_fixture.dart new file mode 100644 index 000000000..4c27e2eab --- /dev/null +++ b/test/support/hmm_night_fixture.dart @@ -0,0 +1,23 @@ +import 'dart:math' as math; + +import 'package:openstrap_edge/compute/onehz_pipeline.dart'; + +// Six observed minutes, with alternating RR intervals giving nonzero RMSSD. +Map hmmNight({bool enabled = true, bool rr = true, + bool gap = false}) { + const t = 1577923200, n = 360; + return DayBundleInput(date: '2020-01-02', + dayTsSec: const [], dayHr: const [], dayRrTsMs: const [], dayRrMs: const [], + sleepTsSec: [for (var i = 0; i < n; i++) t + i], + sleepHr: [for (var i = 0; i < n; i++) gap && i >= 120 && i < 180 ? 0 : 55], + sleepRrTsMs: rr ? [for (var i = 0; i < n; i++) (t + i) * 1000.0] : const [], + sleepRrMs: rr ? [for (var i = 0; i < n; i++) 1000 + 30 * math.sin(i * 0.8)] : const [], + sleepSkinTemp: const [], sleepEnmo: List.filled(n, 0.01), + sleepJson: {'tst_sec': n, 'nrem_sec': n, 'light_sec': n, + 'deep_sec': 0, 'rem_sec': 0, 'wake_sec': 0, 'waso_sec': 0, + 'efficiency_pct': 100.0}, + hypnoStages: List.filled(n, 'light'), sleepOnsetSec: t, + sleepOffsetSec: t + n, profile: const {'age_years': 35, + 'weight_kg': 75, 'height_cm': 178, 'sex': 'male'}, + dayConfidence: 1, dayFlags: const [], useHmmStages: enabled).toJson(); +} diff --git a/test/whoop_freeze_golden_test.dart b/test/whoop_freeze_golden_test.dart index 3dc238007..d76b66a26 100644 --- a/test/whoop_freeze_golden_test.dart +++ b/test/whoop_freeze_golden_test.dart @@ -264,6 +264,21 @@ Object? _comparable(Object? v) => switch (v) { _ => v, }; +// why: v125 changes HMM provenance but must preserve every cardio payload byte. +// # catches: allowing a future algorithm version to reuse this projection, or +// changing any payload, metric, absence, confidence or stored payload hash. +Map _cardio124Projection(Map snapshot) { + expect(kAlgoVersion, 125); + final out = (jsonDecode(jsonEncode(snapshot)) as Map).cast(); + for (final table in ['day_result', 'metric_series_version', 'wake_day_features']) { + for (final row in out[table] as List) { + expect(row['algo_version'], 125); + row['algo_version'] = 124; + } + } + return out; +} + /// The first path where [a] and [b] differ, with both values, or null. A /// whole-snapshot string diff is megabytes, and CI truncates it. String? _firstDiff(Object? a, Object? b, String path) { @@ -370,7 +385,8 @@ void main() { expect(file.existsSync(), isTrue, reason: '${file.path} is missing; see the header.'); final golden = jsonDecode(file.readAsStringSync()); - expect(_firstDiff(_comparable(snap), _comparable(golden), r'$'), + expect(_firstDiff(_comparable(_cardio124Projection(snap)), + _comparable(golden), r'$'), isNull, reason: 'WHOOP output must stay identical. Do NOT ' 'regenerate this golden to make a change pass.');