Repository navigation
feat(sleep): hybrid HMM emission over HR + RMSSD + ENMO + skin temp + multi-night calibration #756
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
fd70934
435b2c9
a5ca173
204da9f
98c5829
065c203
90a879b
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -337,6 +337,13 @@ class DayBundleInput { | |
| final List<double> sleepRrTsMs; | ||
| final List<double> sleepRrMs; | ||
| final List<int> sleepSkinTemp; | ||
| /// Per-second ENMO (g) over the sleep window — HYBRID HMM input. -1 means | ||
| /// ABSENT (accel sentinel (0,0,0)), never a still wrist; the classifier | ||
| /// treats absent as "no motion evidence" rather than "perfectly still". | ||
| final List<double> sleepEnmo; | ||
| /// Trailing per-night ENMO medians (stable core), oldest first — the | ||
| /// persisted motion history for multi-night self-calibration. | ||
| final List<double> enmoHistory; | ||
|
|
||
| // ── the SINGLE-SOURCE sleep segmentation (JSON of SleepSegmentation) ────── | ||
| final Map<String, dynamic> sleepJson; // {window,tst_sec,…,confidence} | ||
|
|
@@ -436,6 +443,8 @@ class DayBundleInput { | |
| required this.sleepRrTsMs, | ||
| required this.sleepRrMs, | ||
| required this.sleepSkinTemp, | ||
| this.sleepEnmo = const [], | ||
| this.enmoHistory = const [], | ||
| required this.sleepJson, | ||
| required this.hypnoStages, | ||
| required this.sleepOnsetSec, | ||
|
|
@@ -471,6 +480,8 @@ class DayBundleInput { | |
| 'sleep_rr_ts_ms': sleepRrTsMs, | ||
| 'sleep_rr_ms': sleepRrMs, | ||
| 'sleep_skin_temp': sleepSkinTemp, | ||
| 'sleep_enmo': sleepEnmo, | ||
| 'enmo_history': enmoHistory, | ||
| 'sleep_json': sleepJson, | ||
| 'hypno_stages': hypnoStages, | ||
| 'sleep_onset_sec': sleepOnsetSec, | ||
|
|
@@ -523,6 +534,8 @@ class DayBundleInput { | |
| sleepRrTsMs: dbls('sleep_rr_ts_ms'), | ||
| sleepRrMs: dbls('sleep_rr_ms'), | ||
| sleepSkinTemp: ints('sleep_skin_temp'), | ||
| sleepEnmo: dbls('sleep_enmo'), | ||
| enmoHistory: dbls('enmo_history'), | ||
| sleepJson: ((m['sleep_json'] as Map?) ?? const {}) | ||
| .cast<String, dynamic>(), | ||
| hypnoStages: strs('hypno_stages'), | ||
|
|
@@ -561,7 +574,7 @@ class DayBundleInput { | |
| /// efficiency. Figures `segmentSleep` derives that edge does not recompute | ||
| /// here are dropped rather than left describing the old labels. Null when | ||
| /// the labels hold no sleep at all: the caller keeps the original night. | ||
| DayBundleInput? withHypnoStages(List<String> stages) { | ||
| DayBundleInput? withHypnoStages(List<String> stages, {bool atypical = false, double? enmoMedian}) { | ||
| var light = 0, deep = 0, rem = 0, wake = 0, unobserved = 0; | ||
| var firstSleep = -1, lastSleep = -1; | ||
| for (var i = 0; i < stages.length; i++) { | ||
|
|
@@ -606,7 +619,18 @@ class DayBundleInput { | |
| ..['rem_sec'] = rem | ||
| ..['wake_sec'] = wake | ||
| ..['efficiency_pct'] = observedSec > 0 ? 100.0 * tst / observedSec : null | ||
| ..['stager_source'] = 'hmm'; | ||
| ..['stager_source'] = 'hmm' | ||
| // Multi-night calibration flag: tonight's baseline deviated >2 robust z | ||
| // from the user's long-term baseline (illness, alcohol, travel...) — | ||
| // the staging is still published, but consumers can treat it as lower | ||
| // confidence. | ||
| ..['stager_atypical_night'] = atypical; | ||
|
coderabbitai[bot] marked this conversation as resolved.
Resolved
|
||
| if (enmoMedian != null && enmoMedian > 0) { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Include measured-zero nights in persisted motion history. If the stable-core ENMO median is zero, this guard drops a valid measurement. Repeated quiet nights therefore leave older positive medians in Accept nonnegative measured medians here and at the output and engine persistence gates. Keep null medians absent.
|
||
| // Persisted as the night's motion baseline -> metric_series | ||
| // 'enmo_median_night', feeding later nights' 7d/long-term motion | ||
| // calibration (enmo_history). | ||
| json['stager_enmo_median'] = enmoMedian; | ||
| } | ||
| return DayBundleInput( | ||
| date: date, | ||
| dayTsSec: dayTsSec, | ||
|
|
@@ -618,6 +642,8 @@ class DayBundleInput { | |
| sleepRrTsMs: sleepRrTsMs, | ||
| sleepRrMs: sleepRrMs, | ||
| sleepSkinTemp: sleepSkinTemp, | ||
| sleepEnmo: sleepEnmo, | ||
| enmoHistory: enmoHistory, | ||
| sleepJson: json, | ||
| hypnoStages: stages, | ||
| sleepOnsetSec: sleepOnsetSec, | ||
|
|
@@ -678,7 +704,10 @@ Map<String, dynamic> _deriveDayBundle(Map<String, dynamic> inputJson, | |
| // from the final labels and the night is tagged `stager_source: hmm`. | ||
| if (d.useHmmStages && d.hypnoStages.isNotEmpty) { | ||
| final restaged = _hmmRestage(d); | ||
| if (restaged != null) d = d.withHypnoStages(restaged) ?? d; | ||
| if (restaged != null) { | ||
| d = d.withHypnoStages(restaged.labels, | ||
| atypical: restaged.atypicalNight, enmoMedian: restaged.enmoMedian) ?? d; | ||
| } | ||
| } | ||
|
|
||
| // ── HR over the DAY (for the curve, strain zones, dip day-side) ─────────── | ||
|
|
@@ -1566,7 +1595,17 @@ Map<String, dynamic> _deriveDayBundle(Map<String, dynamic> inputJson, | |
| 'absence_reason': absenceReason, | ||
| // Present only when the experimental re-staging labelled this night (PR | ||
| // #628), so a night of the shipped stager is stored exactly as before. | ||
| if (d.sleepJson['stager_source'] == 'hmm') 'stager_source': 'hmm', | ||
| if (d.sleepJson['stager_source'] == 'hmm') ...{ | ||
| 'stager_source': 'hmm', | ||
| // Forward the atypical-night (outlier) flag alongside, so the persisted | ||
| // night carries the calibration confidence, not just the source tag. | ||
| if (d.sleepJson['stager_atypical_night'] == true) | ||
| 'stager_atypical_night': true, | ||
| // The night's stable-core ENMO median, persisted to metric_series as | ||
| // 'enmo_median_night' for multi-night motion calibration. | ||
| if ((d.sleepJson['stager_enmo_median'] as num?)?.toDouble() case final v?) | ||
| if (v > 0) 'stager_enmo_median': v, | ||
| }, | ||
| // Seconds the band's own last SLEEP cut off the end of the auto window; null = rule did not apply. Provenance, never a metric. | ||
| 'band_offset_trim_sec': bandOffsetTrimSec, | ||
| 'stager': _envelope( | ||
|
|
@@ -2591,13 +2630,30 @@ List<double> _rmssdPerSecond( | |
| /// later label. Within the night the decoder abstains per 30 s epoch: an | ||
| /// epoch with no plausible heart rate or no RMSSD keeps its shipped label, | ||
| /// and an unobserved second stays unobserved. | ||
| List<String>? _hmmRestage(DayBundleInput d) { | ||
| /// Result of a restage: the merged labels plus whether the night's baseline | ||
| /// deviated strongly from the user's long-term baseline (atypical night). | ||
| class HmmRestageResult { | ||
| final List<String> labels; | ||
| final bool atypicalNight; | ||
| final double? enmoMedian; | ||
| const HmmRestageResult(this.labels, this.atypicalNight, [this.enmoMedian]); | ||
| } | ||
|
|
||
| HmmRestageResult? _hmmRestage(DayBundleInput d) { | ||
| final n = d.hypnoStages.length; | ||
| final t0 = d.sleepOnsetSec; | ||
| var atypical = false; | ||
| double? enmoMedian; | ||
| final hrAt = List<double>.filled(n, 0.0); | ||
| final enmoAt = List<double>.filled(n, 0.0); | ||
| final tempAt = List<double>.filled(n, 0.0); | ||
| for (var i = 0; i < d.sleepTsSec.length && i < d.sleepHr.length; i++) { | ||
| final k = d.sleepTsSec[i] - t0; | ||
| if (k >= 0 && k < n) hrAt[k] = d.sleepHr[i].toDouble(); | ||
| if (k >= 0 && k < n) { | ||
| hrAt[k] = d.sleepHr[i].toDouble(); | ||
| if (i < d.sleepEnmo.length) enmoAt[k] = d.sleepEnmo[i]; | ||
| if (i < d.sleepSkinTemp.length) tempAt[k] = d.sleepSkinTemp[i].toDouble(); | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🟡 Medium · Temperature units skew staging When HMM staging uses raw-ADC skin-temperature data or history on a different scale from nightly samples, Normalize nightly and historical skin-temperature values to a consistent, device-appropriate scale before calibration and emission scoring. Also at Prompt for AI agentsIn `lib/compute/onehz_pipeline.dart` at line 2629:
**Temperature units skew staging**
When HMM staging uses raw-ADC skin-temperature data or history on a different scale from nightly samples, `_hmmRestage` passes raw temperature values through, while the emission divides temperature differences by 100 as if they were centi-degrees; mismatched history and epoch scales also skew the calibration, producing incorrect thermal wake evidence.
Normalize nightly and historical skin-temperature values to a consistent, device-appropriate scale before calibration and emission scoring.
Also at `lib/compute/sleep_pipeline_hmm.dart:112`, `lib/compute/sleep_pipeline_hmm.dart:254`. |
||
| } | ||
| } | ||
| final corrected = correctRr(d.sleepRrMs, rrTsMs: d.sleepRrTsMs); | ||
| // nnTimesMs is relative to the start of the first input beat; put it back | ||
|
|
@@ -2612,14 +2668,30 @@ List<String>? _hmmRestage(DayBundleInput d) { | |
| [for (var i = 0; i < n; i++) t0 + i], | ||
| windowSec: 60, | ||
| ); | ||
| final labels = _classifySleepStagesHMM([ | ||
| for (var i = 0; i < n; i++) | ||
| { | ||
| 'ts': t0 + i, | ||
| 'hr': d.hypnoStages[i] == 'unobserved' ? 0.0 : hrAt[i], | ||
| 'rmssd': d.hypnoStages[i] == 'unobserved' ? 0.0 : rmssdAt[i], | ||
| }, | ||
| ]); | ||
| final labels = _classifySleepStagesHMM( | ||
| [ | ||
| for (var i = 0; i < n; i++) | ||
| { | ||
| 'ts': t0 + i, | ||
| 'hr': d.hypnoStages[i] == 'unobserved' ? 0.0 : hrAt[i], | ||
| 'rmssd': d.hypnoStages[i] == 'unobserved' ? 0.0 : rmssdAt[i], | ||
| // HYBRID: motion + skin temp per second (0 = absent sentinel). | ||
| 'enmo': enmoAt[i], | ||
| 'skin_temp': tempAt[i], | ||
| }, | ||
| ], | ||
| // Multi-night self-calibration: 7d rolling + long-term baselines from the | ||
| // persisted nightly histories (HR, RMSSD, skin temp) feed the blended | ||
| // emission baseline and the atypical-night (outlier) flag. | ||
| rhrHistory: d.rhrHistory, | ||
| rmssdHistory: d.rmssdHistory, | ||
| skinTempHistory: d.skinTempAdcHistory, | ||
| enmoHistory: d.enmoHistory, | ||
| onAnalyzer: (a) { | ||
| atypical = a.atypicalNight; | ||
| enmoMedian = a.enmoMedianNight; | ||
| }, | ||
| ); | ||
| if (labels == null) return null; | ||
| final merged = <String>[ | ||
| for (var i = 0; i < n; i++) | ||
|
|
@@ -2638,14 +2710,48 @@ List<String>? _hmmRestage(DayBundleInput d) { | |
| } | ||
| prev = merged[i]; | ||
| } | ||
| return merged; | ||
| // AASM-aligned wake post-processing on the merged per-second labels: | ||
| // - a wake run shorter than 15 s is an AROUSAL, not wake (AASM: arousal | ||
| // >= 3 s, wake epoch needs > 50% of the 30 s epoch awake) -> the seconds | ||
| // revert to the surrounding sleep stage; | ||
| // - after a REAL (>= 15 s) wake phase the following sleep does NOT restart | ||
| // from scratch — staging continues where it left off (AASM scores | ||
| // sequentially, no reset), which the Viterbi path already respects; long | ||
| // wake runs simply stay wake and count towards WASO. | ||
| const minWakeRunSec = 15; | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🟡 Medium · Brief awakenings remain wake When a brief awakening occurs within a decoded 30-second epoch, Apply the short-wake rule using within-epoch wake evidence rather than only the expanded epoch labels. Also at Prompt for AI agentsIn `lib/compute/onehz_pipeline.dart` at line 2691:
**Brief awakenings remain wake**
When a brief awakening occurs within a decoded 30-second epoch, `_classifySleepStagesHMM` expands one epoch label across its 30 seconds, so `_hmmRestage` sees wake runs of at least 30 seconds; its under-15-second cleanup never replaces the brief awakening with the surrounding sleep stage.
Apply the short-wake rule using within-epoch wake evidence rather than only the expanded epoch labels.
Also at `lib/compute/onehz_pipeline.dart:2702`. |
||
| var i = 0; | ||
| while (i < merged.length) { | ||
| if (merged[i] != 'wake' || merged[i] == 'unobserved') { | ||
| i++; | ||
| continue; | ||
| } | ||
| var j = i; | ||
| while (j < merged.length && merged[j] == 'wake') { | ||
| j++; | ||
| } | ||
| if (j - i < minWakeRunSec) { | ||
| final surrounding = i > 0 ? merged[i - 1] : 'light'; | ||
| for (var k = i; k < j; k++) { | ||
| merged[k] = surrounding == 'unobserved' ? 'light' : surrounding; | ||
| } | ||
| } | ||
| i = j; | ||
| } | ||
| return HmmRestageResult(merged, atypical, enmoMedian); | ||
| } | ||
|
|
||
| /// The Viterbi decode over 30 s epochs of [epochData] (per-second records | ||
| /// with `ts`, `hr`, `rmssd`). Null when the whole night abstains (see | ||
| /// [_hmmRestage]); otherwise one label per record, null where its epoch had | ||
| /// no usable heart rate or RMSSD. | ||
| List<String?>? _classifySleepStagesHMM(List<Map<String, dynamic>> epochData) { | ||
| List<String?>? _classifySleepStagesHMM( | ||
| List<Map<String, dynamic>> epochData, { | ||
| List<double> rhrHistory = const [], | ||
| List<double> rmssdHistory = const [], | ||
| List<double> skinTempHistory = const [], | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🟡 Medium · Motion-only staging is discarded When an epoch has usable motion but lacks valid HR or RMSSD, Allow epochs scored by the motion-only emission path to return and publish their decoded labels. Also at Prompt for AI agentsIn `lib/compute/onehz_pipeline.dart` at line 2721:
**Motion-only staging is discarded**
When an epoch has usable motion but lacks valid HR or RMSSD, `_classifySleepStagesHMM` returns a label only when both HR and RMSSD are valid, so `_hmmRestage` falls back to the shipped stage and discards the motion-only emission result; movement cannot affect the published staging.
Allow epochs scored by the motion-only emission path to return and publish their decoded labels.
Also at `lib/compute/sleep_pipeline_hmm.dart:228-230`. |
||
| List<double> enmoHistory = const [], | ||
| void Function(SleepStageAnalyzer)? onAnalyzer, | ||
| }) { | ||
| if (epochData.isEmpty) return null; | ||
| final analyzer = SleepStageAnalyzer(); | ||
| // Baseline from the first 120 minutes BY ELAPSED TIME, not the first 240 | ||
|
|
@@ -2655,6 +2761,19 @@ List<String?>? _classifySleepStagesHMM(List<Map<String, dynamic>> epochData) { | |
| .where((e) => ((e['ts'] as num?)?.toDouble() ?? 0.0) - t0 <= 120 * 60) | ||
| .toList(); | ||
| analyzer.computeBaseline(first120m); | ||
| // Motion baseline from the stable core of the night (middle 50 %), not the | ||
| // opening window — settling-in movement would inflate the person's normal | ||
| // sleep-motion level and push the wake threshold too high. | ||
| analyzer.computeMotionBaseline(epochData); | ||
| // THREE-LEVEL calibration: blend in the 7-day rolling and long-term | ||
| // baselines from the persisted nightly histories. | ||
| analyzer.computeMultiNightBaselines( | ||
| rhrHistory: rhrHistory, | ||
| rmssdHistory: rmssdHistory, | ||
| skinTempHistory: skinTempHistory, | ||
| enmoHistory: enmoHistory, | ||
| ); | ||
|
sourcery-ai[bot] marked this conversation as resolved.
Resolved
coderabbitai[bot] marked this conversation as resolved.
Resolved
|
||
| onAnalyzer?.call(analyzer); | ||
| // No baseline (RR that only starts after the first 120 minutes, say) makes | ||
| // every emission neutral, and wake's self-transition then labels a measured | ||
| // night as wake. Keep the shipped labels instead. | ||
|
|
@@ -2680,11 +2799,15 @@ List<String?>? _classifySleepStagesHMM(List<Map<String, dynamic>> epochData) { | |
| final idx = <int, int>{}; | ||
| for (var i = 0; i < epochData.length; i++) { | ||
| final k = idx.putIfAbsent(epochOf[i], () { | ||
| epochs.add({'hr': 0.0, 'rmssd': 0.0, 'nh': 0, 'nr': 0}); | ||
| // HYBRID: epochs also average ENMO and skin temp (0 sentinel = absent, | ||
| // excluded from the mean, never counted as "still"/"cold"). | ||
| epochs.add({'hr': 0.0, 'rmssd': 0.0, 'enmo': 0.0, 'skin_temp': 0.0, 'nh': 0, 'nr': 0, 'ne': 0, 'nt': 0}); | ||
| return epochs.length - 1; | ||
| }); | ||
| final hr = (epochData[i]['hr'] as num?)?.toDouble() ?? 0.0; | ||
| final rmssd = (epochData[i]['rmssd'] as num?)?.toDouble() ?? 0.0; | ||
| final enmo = (epochData[i]['enmo'] as num?)?.toDouble() ?? 0.0; | ||
| final temp = (epochData[i]['skin_temp'] as num?)?.toDouble() ?? 0.0; | ||
| final ep = epochs[k]; | ||
| if (hr > 30.0 && hr < 200.0) { | ||
| ep['hr'] = (ep['hr'] as double) + hr; | ||
|
|
@@ -2694,11 +2817,22 @@ List<String?>? _classifySleepStagesHMM(List<Map<String, dynamic>> epochData) { | |
| ep['rmssd'] = (ep['rmssd'] as double) + rmssd; | ||
| ep['nr'] = (ep['nr'] as int) + 1; | ||
| } | ||
| if (enmo >= 0.0) { // measured (incl. real zero motion); -1 = absent | ||
| ep['enmo'] = (ep['enmo'] as double) + enmo; | ||
| ep['ne'] = (ep['ne'] as int) + 1; | ||
| } | ||
| if (temp > 0.0) { | ||
| ep['skin_temp'] = (ep['skin_temp'] as double) + temp; | ||
| ep['nt'] = (ep['nt'] as int) + 1; | ||
| } | ||
| } | ||
| for (final ep in epochs) { | ||
| final nh = ep['nh'] as int, nr = ep['nr'] as int; | ||
| final ne = ep['ne'] as int, nt = ep['nt'] as int; | ||
| ep['hr'] = nh == 0 ? 0.0 : (ep['hr'] as double) / nh; | ||
| ep['rmssd'] = nr == 0 ? 0.0 : (ep['rmssd'] as double) / nr; | ||
| ep['enmo'] = ne == 0 ? 0.0 : (ep['enmo'] as double) / ne; | ||
| ep['skin_temp'] = nt == 0 ? 0.0 : (ep['skin_temp'] as double) / nt; | ||
| } | ||
| final states = analyzer.viterbiDecode(epochs); | ||
| if (states.length != epochs.length) return null; | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.