From dd8b69d78aa283889f850b2b5ac4900578d5ecb0 Mon Sep 17 00:00:00 2001 From: MikenRafter Date: Tue, 17 Nov 2026 10:35:16 -0700 Subject: [PATCH 01/10] test: specify exact imported flags and legacy mirror repair Exercise eleven independent JSON shapes through raw insert and payload update, the fallback after a refused index build, schema-62 upgrade, repeated same-version repair, and covering-index lookup. Use raw SQL for spaced fixtures because putDayResult canonicalizes JSON. Replace the LIKE oracle with top-level JSON boolean semantics. The new cases fail on cd64eaa89. --- test/finalized_day_ids_index_test.dart | 129 ++++++++++++++++++++++--- 1 file changed, 117 insertions(+), 12 deletions(-) diff --git a/test/finalized_day_ids_index_test.dart b/test/finalized_day_ids_index_test.dart index 230a96645..958e6bac4 100644 --- a/test/finalized_day_ids_index_test.dart +++ b/test/finalized_day_ids_index_test.dart @@ -1,8 +1,10 @@ // `finalizedDayIds` reads an indexed `day_result.imported` flag instead of -// LIKE-scanning every finalized bundle. The answer must be the same set the -// scan gave, for rows written before the column existed (backfill) and after +// scanning every finalized bundle. Only a top-level JSON boolean true is +// imported, for rows written before the column existed (backfill) and after // (triggers), on every write path. +import 'dart:convert'; + import 'package:flutter_test/flutter_test.dart'; import 'package:path/path.dart' as p; import 'package:sqflite_common_ffi/sqflite_ffi.dart'; @@ -17,13 +19,15 @@ Future _useFreshDb(String name) async { const _algo = 7; -Future> _likeScan(Database db) async => { +Future> _exactScan(Database db) async => { for (final r in await db.query( 'day_result', columns: ['day_id'], where: 'algo_version = ? AND finalized = 1 ' - "AND payload_json NOT LIKE '%\"imported\":true%'", + "AND CASE WHEN json_valid(payload_json) THEN " + "COALESCE(json_type(payload_json, '\$.imported') = 'true', 0) " + "ELSE 0 END = 0", whereArgs: [_algo], )) r['day_id'] as String, @@ -31,7 +35,7 @@ Future> _likeScan(Database db) async => { Future _indexed(Database db) async => (await db.rawQuery( "SELECT 1 FROM sqlite_master WHERE type = 'index' " - "AND name = 'idx_day_result_finalized'", + "AND name = 'idx_day_result_finalized_v2'", )).isNotEmpty; Future _put(String day, String payload, {bool finalized = true}) => @@ -43,6 +47,13 @@ Future _put(String day, String payload, {bool finalized = true}) => finalized: finalized, ); +Future _rawPut(Database db, String day, String payload) async { + await db.insert('day_result', { + 'day_id': day, 'algo_version': _algo, 'payload_json': payload, + 'window_json': '{}', 'computed_at': 0, 'finalized': 1, + }); +} + void main() { setUpAll(() { sqfliteFfiInit(); @@ -53,7 +64,101 @@ void main() { await LocalDb.close(); }); - test('same set as the LIKE scan, through the triggers', () async { + + // WHY: substring matching confuses nested keys and misses spaced booleans. + // CLAIM(imported-exact); catches: LIKE, json_extract without a type check. + test('only a top-level boolean true marks an imported day on every write', () async { + await _useFreshDb('finalized_shapes.db'); + final db = await LocalDb.instance; + final shapes = { + '{}': false, + '{"imported":true}': true, + '{"imported":false}': false, + '{"nested":{"imported":true}}': false, + jsonEncode({'note': '"imported":true'}): false, + '{"not_imported":true}': false, + '{"imported":1}': false, + '{"imported":"true"}': false, + '{"imported":null}': false, + '{ "imported" : true }': true, + 'invalid JSON': false, + }; + final expected = {}; + var n = 0; + for (final entry in shapes.entries) { + final day = '2026-01-${(++n).toString().padLeft(2, '0')}'; + await _rawPut(db, day, entry.key); + if (!entry.value) expected.add(day); + expect((await db.query('day_result', columns: ['imported'], + where: 'day_id = ?', whereArgs: [day])).single['imported'], + entry.value ? 1 : 0, reason: entry.key); + } + expect(await LocalDb.finalizedDayIds(_algo), expected); + // Raw SQL payload rewrites also go through the mirror's update trigger. + await db.update('day_result', {'payload_json': '{ "imported" : true }'}, + where: 'day_id = ?', whereArgs: ['2026-01-01']); + expected.remove('2026-01-01'); + expect(await LocalDb.finalizedDayIds(_algo), expected); + await db.update('day_result', {'payload_json': '{"nested":{"imported":true}}'}, + where: 'day_id = ?', whereArgs: ['2026-01-01']); + expected.add('2026-01-01'); + expect(await LocalDb.finalizedDayIds(_algo), expected); + }); + + // WHY: schema-62 already has a completed LIKE backfill and triggers. Both + // upgrade and same-version repair must invalidate that completion marker. + // CLAIM(imported-exact); catches: replacing only the trigger definitions. + for (final version in [62, 63]) { + test('repairs a stale legacy mirror opened at schema $version twice', () async { + await _useFreshDb('finalized_legacy_$version.db'); + var db = await LocalDb.instance; + await db.execute('DROP TRIGGER IF EXISTS trg_day_result_imported_v2_ins'); + await db.execute('DROP TRIGGER IF EXISTS trg_day_result_imported_v2_upd'); + await db.execute('DROP TRIGGER IF EXISTS trg_day_result_imported_ins'); + await db.execute('DROP TRIGGER IF EXISTS trg_day_result_imported_upd'); + await db.execute('DROP INDEX IF EXISTS idx_day_result_finalized_v2'); + await db.execute('CREATE INDEX IF NOT EXISTS idx_day_result_finalized ' + 'ON day_result(algo_version, finalized, imported, day_id)'); + for (final suffix in ['ins', 'upd']) { + await db.execute('CREATE TRIGGER trg_day_result_imported_$suffix ' + 'AFTER ${suffix == 'ins' ? 'INSERT' : 'UPDATE OF payload_json'} ' + 'ON day_result BEGIN UPDATE day_result ' + "SET imported = (NEW.payload_json LIKE '%\"imported\":true%') " + 'WHERE rowid = NEW.rowid; END'); + } + await _rawPut(db, '2026-01-01', '{"nested":{"imported":true}}'); + await _rawPut(db, '2026-01-02', '{ "imported" : true }'); + expect((await db.query('day_result', columns: ['imported'], + orderBy: 'day_id')).map((r) => r['imported']).toList(), [1, 0]); + await db.setVersion(version); + for (var open = 0; open < 2; open++) { + await LocalDb.close(); + expect(await LocalDb.finalizedDayIds(_algo), {'2026-01-01'}); + db = await LocalDb.instance; + expect((await db.query('day_result', columns: ['imported'], + orderBy: 'day_id')).map((r) => r['imported']).toList(), [0, 1]); + expect(await _indexed(db), isTrue); + } + }); + } + + // WHY: a refused index build must keep the fallback's classification exact. + // CLAIM(imported-exact); catches: retaining LIKE only in the fallback path. + test('failed index creation falls back to the exact top-level key', () async { + await _useFreshDb('finalized_fallback.db'); + final db = await LocalDb.instance; + await db.execute('CREATE TABLE idx_day_result_finalized_v2 (id INTEGER)'); + await _rawPut(db, '2026-01-01', '{"nested":{"imported":true}}'); + await _rawPut(db, '2026-01-02', '{ "imported" : true }'); + await _rawPut(db, '2026-01-03', 'invalid JSON'); + expect(await LocalDb.finalizedDayIds(_algo), {'2026-01-01', '2026-01-03'}); + expect(await _indexed(db), isFalse); + await db.execute('DROP TABLE idx_day_result_finalized_v2'); + expect(await LocalDb.finalizedDayIds(_algo), {'2026-01-01', '2026-01-03'}); + expect(await _indexed(db), isTrue); + }); + + test('same set as the exact scan, through the triggers', () async { await _useFreshDb('finalized_ids_triggers.db'); await _put('2026-01-01', '{"scalars":{}}'); await _put('2026-01-02', '{"imported":true,"source":"whoop_export"}'); @@ -74,16 +179,16 @@ void main() { ); final ids = await LocalDb.finalizedDayIds(_algo); expect(ids, {'2026-01-02'}); - expect(ids, await _likeScan(db)); + expect(ids, await _exactScan(db)); }); test('backfills rows written before the column existed', () async { await _useFreshDb('finalized_ids_backfill.db'); final db = await LocalDb.instance; // Emulate a pre-column database: no triggers, no index, stale flags. - await db.execute('DROP TRIGGER IF EXISTS trg_day_result_imported_ins'); - await db.execute('DROP TRIGGER IF EXISTS trg_day_result_imported_upd'); - await db.execute('DROP INDEX IF EXISTS idx_day_result_finalized'); + await db.execute('DROP TRIGGER IF EXISTS trg_day_result_imported_v2_ins'); + await db.execute('DROP TRIGGER IF EXISTS trg_day_result_imported_v2_upd'); + await db.execute('DROP INDEX IF EXISTS idx_day_result_finalized_v2'); for (var i = 0; i < 450; i++) { final day = '2025-${(1 + i ~/ 28).toString().padLeft(2, '0')}-' @@ -99,7 +204,7 @@ void main() { 'imported': i % 3 == 0 ? 0 : 1, }); } - final expected = await _likeScan(db); + final expected = await _exactScan(db); expect(expected, isNotEmpty); // Reopen: the repair pass restores the triggers, the first read backfills. await LocalDb.close(); @@ -113,7 +218,7 @@ void main() { ); expect( plan.map((r) => '${r['detail']}').join(' '), - contains('COVERING INDEX idx_day_result_finalized'), + contains('COVERING INDEX idx_day_result_finalized_v2'), ); }); } From b27315aac822d1090d12f1c235b8827896c70a4a Mon Sep 17 00:00:00 2001 From: MikenRafter Date: Tue, 17 Nov 2026 10:37:13 -0700 Subject: [PATCH 02/10] data: mirror only the exact top-level imported boolean Use one guarded json_type expression for insert/update triggers, paged backfill and finalized-day fallback. Nested keys, numeric one and string true remain measured; whitespace around a top-level boolean is accepted. Add schema rung 63 and idempotent open repair. Replace legacy triggers and invalidate their backfill with a versioned covering-index marker; existing payloads are repaired in 200-row pages on first read, outside database open. --- lib/data/db.dart | 41 +++++++++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/lib/data/db.dart b/lib/data/db.dart index dc799032e..9e50f97f3 100644 --- a/lib/data/db.dart +++ b/lib/data/db.dart @@ -688,7 +688,7 @@ class LocalDb { /// pass it: sqflite throws `ArgumentError('onCreate must be null if no /// version is specified')` BEFORE opening anything when `onCreate` is given /// without `version` (sqflite_common database_mixin.dart). - static const int schemaVersion = 62; + static const int schemaVersion = 63; /// In-memory cache identity, advanced after open, wipe or committed import. /// It is local to this isolate; no cache survives a process restart. @@ -1544,6 +1544,11 @@ class LocalDb { await _createStoreIdentity(db); await _createDayCheckpoint(db); } + if (oldV < 63) { + // Replace the LIKE mirror's triggers and completion marker only. + // Paged payload repair stays off the launch watchdog, on first read. + await _ensureDayResultImportedColumn(db); + } }, onOpen: (db) async { await _repairOpenSchema(db); @@ -10920,24 +10925,27 @@ class LocalDb { where: indexed ? 'algo_version = ? AND finalized = 1 AND imported = 0' : 'algo_version = ? AND finalized = 1 ' - 'AND payload_json NOT LIKE $_importedPayloadLike', + 'AND (${_importedPayloadSql('payload_json')}) = 0', whereArgs: [algoVersion], ); return {for (final r in rows) r['day_id'] as String}; } - /// The one test for "this bundle is an imported snapshot". The `imported` - /// column is computed from it by trigger, so the column and the old scan - /// cannot disagree about a row. - static const String _importedPayloadLike = "'%\"imported\":true%'"; + /// CLAIM(imported-exact): only a top-level JSON boolean true marks an + /// imported snapshot [finalized_day_ids_index_test]. Guard malformed JSON + /// so a damaged bundle cannot abort a write or the fallback scan. + static String _importedPayloadSql(String column) => + "CASE WHEN json_valid($column) THEN " + "COALESCE(json_type($column, '\$.imported') = 'true', 0) ELSE 0 END"; /// Covering index for [finalizedDayIds]. Created only once every existing /// row's `imported` is filled, so its presence is also the "backfill done" /// marker: a database restored from a file without it backfills again. - static const String _dayResultFinalizedIndex = 'idx_day_result_finalized'; + // Version the completion marker: an old LIKE backfill is not trustworthy. + static const String _dayResultFinalizedIndex = 'idx_day_result_finalized_v2'; /// `day_result.imported` and the triggers that keep it equal to - /// [_importedPayloadLike] on every insert and payload update — whichever + /// [_importedPayloadSql] on every insert and payload update — whichever /// path writes the row (derive, import, restore merge, raw SQL). Cheap and /// idempotent: ADD COLUMN with a constant default rewrites nothing, so this /// is safe on the every-open repair path. The backfill of existing rows is @@ -10947,18 +10955,23 @@ class LocalDb { await _addColumnIfMissing( db, 'day_result', 'imported', 'INTEGER NOT NULL DEFAULT 0', ); + // Cheap every-open repair, including same-version restored databases. + // The v2 index is built only after the paged exact-key backfill completes. + await db.execute('DROP TRIGGER IF EXISTS trg_day_result_imported_ins'); + await db.execute('DROP TRIGGER IF EXISTS trg_day_result_imported_upd'); + await db.execute('DROP INDEX IF EXISTS idx_day_result_finalized'); await db.execute( - 'CREATE TRIGGER IF NOT EXISTS trg_day_result_imported_ins ' + 'CREATE TRIGGER IF NOT EXISTS trg_day_result_imported_v2_ins ' 'AFTER INSERT ON day_result BEGIN ' 'UPDATE day_result ' - 'SET imported = (NEW.payload_json LIKE $_importedPayloadLike) ' + 'SET imported = (${_importedPayloadSql('NEW.payload_json')}) ' 'WHERE rowid = NEW.rowid; END', ); await db.execute( - 'CREATE TRIGGER IF NOT EXISTS trg_day_result_imported_upd ' + 'CREATE TRIGGER IF NOT EXISTS trg_day_result_imported_v2_upd ' 'AFTER UPDATE OF payload_json ON day_result BEGIN ' 'UPDATE day_result ' - 'SET imported = (NEW.payload_json LIKE $_importedPayloadLike) ' + 'SET imported = (${_importedPayloadSql('NEW.payload_json')}) ' 'WHERE rowid = NEW.rowid; END', ); } @@ -10970,7 +10983,7 @@ class LocalDb { /// never runs it under the launch watchdog. Rows written meanwhile are /// already right (the triggers), and re-filling one recomputes the same /// value, so a concurrent derive cannot race it. Any failure leaves the - /// index absent and the caller on the old scan; the next call resumes. + /// index absent and the caller on the exact-key scan; the next call resumes. static Future _ensureDayResultImportedFlag(Database db) async { Future ready() async => (await db.rawQuery( "SELECT 1 FROM sqlite_master WHERE type = 'index' AND name = ?", @@ -10993,7 +11006,7 @@ class LocalDb { final hi = (page.last['r'] as num).toInt(); await db.rawUpdate( 'UPDATE day_result ' - 'SET imported = (payload_json LIKE $_importedPayloadLike) ' + 'SET imported = (${_importedPayloadSql('payload_json')}) ' 'WHERE rowid > ? AND rowid <= ?', [after, hi], ); From b0bc55be0dcfa0246158ff2ad7306fec7aa0cf29 Mon Sep 17 00:00:00 2001 From: MikenRafter Date: Tue, 17 Nov 2026 10:39:00 -0700 Subject: [PATCH 03/10] test: allow later schema rungs in checkpoint storage contracts Rung 63 follows the checkpoint rung at 62. Keep the revision and checkpoint behavioral assertions, require at least their introducing rung, and verify the opened database matches the current schema instead of pinning an obsolete literal. --- test/ed2a_input_revision_test.dart | 2 +- test/ed2b_checkpoints_test.dart | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/ed2a_input_revision_test.dart b/test/ed2a_input_revision_test.dart index 00fc1db97..b053afc3c 100644 --- a/test/ed2a_input_revision_test.dart +++ b/test/ed2a_input_revision_test.dart @@ -156,7 +156,7 @@ void main() { }); test('rung 61 input revisions remain present with rung 62 checkpoints', () async { - expect(LocalDb.schemaVersion, 62, reason: 'checkpoint storage follows revisions at 61'); + expect(LocalDb.schemaVersion, greaterThanOrEqualTo(62), reason: 'checkpoint storage follows revisions at 61'); final db = await open(); expect(await db.getVersion(), LocalDb.schemaVersion); expect(await _hasRevisions(db), isTrue); diff --git a/test/ed2b_checkpoints_test.dart b/test/ed2b_checkpoints_test.dart index 41bc68409..afd8d4263 100644 --- a/test/ed2b_checkpoints_test.dart +++ b/test/ed2b_checkpoints_test.dart @@ -112,10 +112,10 @@ void main() { }); test('E16 fresh storage uses rung 62 and a versioned composite key', () async { - expect(LocalDb.schemaVersion, 62, + expect(LocalDb.schemaVersion, greaterThanOrEqualTo(62), reason: '61 belongs to ED-2a revisions; checkpoints use the next rung'); final db = await _storage(); - expect(await db.getVersion(), 62); + expect(await db.getVersion(), LocalDb.schemaVersion); expect(await db.query('store_identity'), hasLength(1)); expect(await LocalDb.storeIdentity(), matches(RegExp(r'^[0-9a-f]{32}$'))); final columns = {for (final row in await db.rawQuery( @@ -152,7 +152,7 @@ void main() { expect(await db.query('input_rev', orderBy: 'bucket'), revs); expect(await db.rawQuery('PRAGMA table_info(decoded_onehz)'), rawShape); await _storage(); - expect(await db.getVersion(), 62); + expect(await db.getVersion(), LocalDb.schemaVersion); expect(await db.query('day_checkpoint'), isEmpty, reason: 'migration creates empty scratch storage without deriving old days'); }); From 7be2954f0b7065e9dbbc73e7d439e03ec78223ac Mon Sep 17 00:00:00 2001 From: MikenRafter Date: Tue, 17 Nov 2026 15:39:23 -0700 Subject: [PATCH 04/10] test: compute the imported-flag oracle in Dart The expected set repeated the production CASE/json_type expression, so the trigger and backfill tests compared the expression with itself. Build it from SeriesCodec.decodePayloadJson(p)['imported'] == true, which is what isMeasuredDayRow reads, and check the eleven-shape table against it. --- test/finalized_day_ids_index_test.dart | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/test/finalized_day_ids_index_test.dart b/test/finalized_day_ids_index_test.dart index 958e6bac4..4dcd57010 100644 --- a/test/finalized_day_ids_index_test.dart +++ b/test/finalized_day_ids_index_test.dart @@ -9,6 +9,7 @@ import 'package:flutter_test/flutter_test.dart'; import 'package:path/path.dart' as p; import 'package:sqflite_common_ffi/sqflite_ffi.dart'; import 'package:openstrap_edge/data/db.dart'; +import 'package:openstrap_edge/data/series_codec.dart'; Future _useFreshDb(String name) async { await LocalDb.close(); @@ -19,18 +20,19 @@ Future _useFreshDb(String name) async { const _algo = 7; -Future> _exactScan(Database db) async => { +/// The expected set, computed in Dart exactly as `LocalDb.isMeasuredDayRow` +/// reads a payload (`p['imported'] == true`). It deliberately shares no code +/// with the SQL expression under test, so the test is a SQL/Dart parity check +/// and not the expression compared with itself. +Future> _dartOracle(Database db) async => { for (final r in await db.query( 'day_result', - columns: ['day_id'], - where: - 'algo_version = ? AND finalized = 1 ' - "AND CASE WHEN json_valid(payload_json) THEN " - "COALESCE(json_type(payload_json, '\$.imported') = 'true', 0) " - "ELSE 0 END = 0", + columns: ['day_id', 'payload_json'], + where: 'algo_version = ? AND finalized = 1', whereArgs: [_algo], )) - r['day_id'] as String, + if (SeriesCodec.decodePayloadJson(r['payload_json'])?['imported'] != true) + r['day_id'] as String, }; Future _indexed(Database db) async => (await db.rawQuery( @@ -87,6 +89,9 @@ void main() { var n = 0; for (final entry in shapes.entries) { final day = '2026-01-${(++n).toString().padLeft(2, '0')}'; + // The table agrees with the Dart reader, so SQL and Dart cannot drift. + expect(SeriesCodec.decodePayloadJson(entry.key)?['imported'] == true, + entry.value, reason: 'oracle ${entry.key}'); await _rawPut(db, day, entry.key); if (!entry.value) expected.add(day); expect((await db.query('day_result', columns: ['imported'], @@ -179,7 +184,7 @@ void main() { ); final ids = await LocalDb.finalizedDayIds(_algo); expect(ids, {'2026-01-02'}); - expect(ids, await _exactScan(db)); + expect(ids, await _dartOracle(db)); }); test('backfills rows written before the column existed', () async { @@ -204,7 +209,7 @@ void main() { 'imported': i % 3 == 0 ? 0 : 1, }); } - final expected = await _exactScan(db); + final expected = await _dartOracle(db); expect(expected, isNotEmpty); // Reopen: the repair pass restores the triggers, the first read backfills. await LocalDb.close(); From ced7d3ccc3c6abb2808ce0b824285aeb146d083c Mon Sep 17 00:00:00 2001 From: MikenRafter Date: Tue, 17 Nov 2026 15:39:35 -0700 Subject: [PATCH 05/10] test: keep the LIKE mirror when SQLite has no JSON1 The exact-key triggers call json_valid and json_type on every day_result write. Pin that a database without JSON1 keeps the legacy LIKE triggers, fallback and index name, and that a restored JSON mirror is swapped out there. The probe seam is inert until the next commit, so both tests fail on cd64eaa89-era behaviour. --- lib/data/db.dart | 5 ++ test/finalized_day_ids_index_test.dart | 63 ++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/lib/data/db.dart b/lib/data/db.dart index 9e50f97f3..5a11f7861 100644 --- a/lib/data/db.dart +++ b/lib/data/db.dart @@ -10931,6 +10931,11 @@ class LocalDb { return {for (final r in rows) r['day_id'] as String}; } + /// Test seam: makes the JSON1 probe report "unavailable", as on an Android + /// framework SQLite built without it. + @visibleForTesting + static bool debugJsonProbeFails = false; + /// CLAIM(imported-exact): only a top-level JSON boolean true marks an /// imported snapshot [finalized_day_ids_index_test]. Guard malformed JSON /// so a damaged bundle cannot abort a write or the fallback scan. diff --git a/test/finalized_day_ids_index_test.dart b/test/finalized_day_ids_index_test.dart index 4dcd57010..9ee49a461 100644 --- a/test/finalized_day_ids_index_test.dart +++ b/test/finalized_day_ids_index_test.dart @@ -226,4 +226,67 @@ void main() { contains('COVERING INDEX idx_day_result_finalized_v2'), ); }); + + Future> triggers(Database db) async => [ + for (final r in await db.rawQuery( + "SELECT name FROM sqlite_master WHERE type = 'trigger' " + "AND tbl_name = 'day_result' ORDER BY name", + )) + r['name'] as String, + ]; + + Future hasIndex(Database db, String name) async => (await db.rawQuery( + "SELECT 1 FROM sqlite_master WHERE type = 'index' AND name = ?", + [name], + )).isNotEmpty; + + // WHY: day_result is written by every derive. A trigger body that calls + // json_valid/json_type cannot compile on a SQLite without JSON1 (the Android + // framework builds some devices ship), and then EVERY insert fails, so no day + // could be saved. Without JSON1 the legacy LIKE mirror must stay in place. + // CLAIM(imported-exact); catches: a JSON1 requirement on the write path. + test('without JSON1 a fresh database keeps the legacy LIKE mirror', () async { + LocalDb.debugJsonProbeFails = true; + addTearDown(() => LocalDb.debugJsonProbeFails = false); + await _useFreshDb('finalized_no_json1.db'); + final db = await LocalDb.instance; + expect(await triggers(db), + ['trg_day_result_imported_ins', 'trg_day_result_imported_upd']); + for (final r in await db.rawQuery( + "SELECT sql FROM sqlite_master WHERE type = 'trigger' " + "AND tbl_name = 'day_result'", + )) { + expect(r['sql'], isNot(contains('json_')), reason: 'no JSON1 in a trigger'); + } + await _rawPut(db, '2026-01-01', '{"nested":{"imported":true}}'); + await _rawPut(db, '2026-01-02', '{ "imported" : true }'); + // The legacy substring test, kept as it was: nested matches, spaced does not. + expect((await db.query('day_result', columns: ['imported'], + orderBy: 'day_id')).map((r) => r['imported']).toList(), [1, 0]); + expect(await LocalDb.finalizedDayIds(_algo), {'2026-01-02'}); + expect(await hasIndex(db, 'idx_day_result_finalized'), isTrue); + expect(await hasIndex(db, 'idx_day_result_finalized_v2'), isFalse); + }); + + // WHY: a database file restored from a JSON1 device onto one without JSON1 + // arrives carrying the JSON triggers; they would fail every write there. + // CLAIM(imported-exact); catches: leaving JSON triggers when JSON1 is absent. + test('reopening without JSON1 swaps a restored JSON mirror for the LIKE one', + () async { + addTearDown(() => LocalDb.debugJsonProbeFails = false); + await _useFreshDb('finalized_restored_json.db'); + var db = await LocalDb.instance; + expect(await triggers(db), [ + 'trg_day_result_imported_v2_ins', + 'trg_day_result_imported_v2_upd', + ]); + await LocalDb.finalizedDayIds(_algo); + expect(await hasIndex(db, 'idx_day_result_finalized_v2'), isTrue); + await LocalDb.close(); + LocalDb.debugJsonProbeFails = true; + db = await LocalDb.instance; + expect(await triggers(db), + ['trg_day_result_imported_ins', 'trg_day_result_imported_upd']); + expect(await hasIndex(db, 'idx_day_result_finalized_v2'), isFalse); + }); } From 629c7a4393df845bcd434ada2ecb31265598c365 Mon Sep 17 00:00:00 2001 From: MikenRafter Date: Tue, 17 Nov 2026 15:55:48 -0700 Subject: [PATCH 06/10] data: probe JSON1 before installing the exact imported-flag triggers The exact-key triggers call json_valid and json_type on every day_result write. A SQLite without JSON1 cannot compile them, and then no day could be saved. Probe once in _ensureDayResultImportedColumn. Without JSON1 keep the legacy LIKE triggers, fallback and index name, and drop a JSON mirror a restored file carries. Put a substring prefilter in front of json_valid so the common write (a derived bundle without the key) skips both JSON parses. The paged repair's comment now says it restarts rather than resumes. --- lib/data/db.dart | 115 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 83 insertions(+), 32 deletions(-) diff --git a/lib/data/db.dart b/lib/data/db.dart index 5a11f7861..d94e4d544 100644 --- a/lib/data/db.dart +++ b/lib/data/db.dart @@ -10911,21 +10911,22 @@ class LocalDb { /// and replace the snapshot ([isMeasuredDayRow]). Every caller intersects /// this with days that have raw, so a raw-less import is still left alone. /// - /// Read off [_dayResultFinalizedIndex] once [_ensureDayResultImportedFlag] - /// has filled `day_result.imported`. The LIKE it replaces read every - /// finalized day's whole bundle — and `finalized` itself sits after - /// `payload_json` in the row, so even the plain column test walked each - /// bundle's overflow pages — several times per derive pass. + /// Read off the completion index ([_finalizedIndexName]) once + /// [_ensureDayResultImportedFlag] has filled `day_result.imported`. The scan + /// it replaces read every finalized day's whole bundle — and `finalized` + /// itself sits after `payload_json` in the row, so even the plain column + /// test walked each bundle's overflow pages — several times per derive pass. static Future> finalizedDayIds(int algoVersion) async { final db = await instance; final indexed = await _ensureDayResultImportedFlag(db); + final exact = _importedExactMode ?? await _probeJson1(db); final rows = await db.query( 'day_result', columns: ['day_id'], where: indexed ? 'algo_version = ? AND finalized = 1 AND imported = 0' : 'algo_version = ? AND finalized = 1 ' - 'AND (${_importedPayloadSql('payload_json')}) = 0', + 'AND (${_importedPayloadSql('payload_json', exact: exact)}) = 0', whereArgs: [algoVersion], ); return {for (final r in rows) r['day_id'] as String}; @@ -10936,18 +10937,49 @@ class LocalDb { @visibleForTesting static bool debugJsonProbeFails = false; - /// CLAIM(imported-exact): only a top-level JSON boolean true marks an - /// imported snapshot [finalized_day_ids_index_test]. Guard malformed JSON - /// so a damaged bundle cannot abort a write or the fallback scan. - static String _importedPayloadSql(String column) => - "CASE WHEN json_valid($column) THEN " - "COALESCE(json_type($column, '\$.imported') = 'true', 0) ELSE 0 END"; + /// Whether this process's SQLite has JSON1, as last probed by + /// [_ensureDayResultImportedColumn] on open. Null until the first open. + static bool? _importedExactMode; + + /// One constant query. True when `json_type` exists and reads a top-level + /// boolean. The mirror's triggers fire on EVERY `day_result` write, and a + /// trigger body that names a missing function cannot compile, so every + /// insert would fail: the Android framework SQLite (3.18/3.22 on Android + /// 8-10) is not guaranteed to carry JSON1. Without it the legacy LIKE mirror + /// stays. + static Future _probeJson1(DatabaseExecutor db) async { + if (debugJsonProbeFails) return _importedExactMode = false; + try { + final r = await db.rawQuery( + "SELECT json_type('{\"imported\":true}', '\$.imported') AS t", + ); + return _importedExactMode = r.single['t'] == 'true'; + } catch (_) { + return _importedExactMode = false; + } + } + + /// CLAIM(imported-exact): with JSON1, only a top-level JSON boolean true + /// marks an imported snapshot [finalized_day_ids_index_test]. Guard malformed + /// JSON so a damaged bundle cannot abort a write or the fallback scan. The + /// substring prefilter skips both JSON parses for the common write (a derived + /// bundle without the key); a spaced key still contains it. Without JSON1 + /// ([exact] false) this is the old substring test, which also counts a + /// nested key. + static String _importedPayloadSql(String column, {required bool exact}) => + exact + ? "CASE WHEN $column LIKE '%\"imported\"%' AND json_valid($column) " + "THEN COALESCE(json_type($column, '\$.imported') = 'true', 0) " + "ELSE 0 END" + : "($column LIKE '%\"imported\":true%')"; /// Covering index for [finalizedDayIds]. Created only once every existing /// row's `imported` is filled, so its presence is also the "backfill done" - /// marker: a database restored from a file without it backfills again. - // Version the completion marker: an old LIKE backfill is not trustworthy. - static const String _dayResultFinalizedIndex = 'idx_day_result_finalized_v2'; + /// marker: a database restored from a file without it backfills again. The + /// exact-key backfill has its own name (an old LIKE backfill is not + /// trustworthy for it); without JSON1 the legacy name and LIKE backfill stay. + static String _finalizedIndexName(bool exact) => + exact ? 'idx_day_result_finalized_v2' : 'idx_day_result_finalized'; /// `day_result.imported` and the triggers that keep it equal to /// [_importedPayloadSql] on every insert and payload update — whichever @@ -10960,39 +10992,58 @@ class LocalDb { await _addColumnIfMissing( db, 'day_result', 'imported', 'INTEGER NOT NULL DEFAULT 0', ); - // Cheap every-open repair, including same-version restored databases. - // The v2 index is built only after the paged exact-key backfill completes. - await db.execute('DROP TRIGGER IF EXISTS trg_day_result_imported_ins'); - await db.execute('DROP TRIGGER IF EXISTS trg_day_result_imported_upd'); - await db.execute('DROP INDEX IF EXISTS idx_day_result_finalized'); + // Cheap every-open repair, including same-version restored databases. Only + // DERIVED objects are dropped and recreated (two triggers and an index over + // a column this method owns), so it stays additive and idempotent: a second + // open finds the right objects and changes nothing. The completion index is + // built only after the paged backfill, see [_ensureDayResultImportedFlag]. + final exact = await _probeJson1(db); + // The other mode's objects go first: a restored file can carry JSON + // triggers onto a SQLite that cannot compile them. + for (final stale in exact + ? const ['trg_day_result_imported_ins', 'trg_day_result_imported_upd'] + : const [ + 'trg_day_result_imported_v2_ins', + 'trg_day_result_imported_v2_upd', + ]) { + await db.execute('DROP TRIGGER IF EXISTS $stale'); + } + await db.execute( + 'DROP INDEX IF EXISTS ${_finalizedIndexName(!exact)}', + ); + final tag = exact ? '_v2' : ''; await db.execute( - 'CREATE TRIGGER IF NOT EXISTS trg_day_result_imported_v2_ins ' + 'CREATE TRIGGER IF NOT EXISTS trg_day_result_imported${tag}_ins ' 'AFTER INSERT ON day_result BEGIN ' 'UPDATE day_result ' - 'SET imported = (${_importedPayloadSql('NEW.payload_json')}) ' + 'SET imported = (${_importedPayloadSql('NEW.payload_json', exact: exact)}) ' 'WHERE rowid = NEW.rowid; END', ); await db.execute( - 'CREATE TRIGGER IF NOT EXISTS trg_day_result_imported_v2_upd ' + 'CREATE TRIGGER IF NOT EXISTS trg_day_result_imported${tag}_upd ' 'AFTER UPDATE OF payload_json ON day_result BEGIN ' 'UPDATE day_result ' - 'SET imported = (${_importedPayloadSql('NEW.payload_json')}) ' + 'SET imported = (${_importedPayloadSql('NEW.payload_json', exact: exact)}) ' 'WHERE rowid = NEW.rowid; END', ); } /// True once `day_result.imported` is trustworthy for every row and - /// [_dayResultFinalizedIndex] exists. The first call on a database that - /// predates the column fills it in pages of 200 rows, each its own short - /// statement, then builds the index — off `openDatabase`, so a long history - /// never runs it under the launch watchdog. Rows written meanwhile are + /// the completion index ([_finalizedIndexName]) exists. The first call on a + /// database that predates the column fills it in pages of 200 rows, each its + /// own short statement, then builds the index — off `openDatabase`, so a long + /// history never runs it under the launch watchdog. Rows written meanwhile are /// already right (the triggers), and re-filling one recomputes the same /// value, so a concurrent derive cannot race it. Any failure leaves the - /// index absent and the caller on the exact-key scan; the next call resumes. + /// index absent and the caller on the scan; the next call RESTARTS the + /// backfill from the first row (rows already right recompute the same + /// value), it does not resume. static Future _ensureDayResultImportedFlag(Database db) async { + final exact = _importedExactMode ?? await _probeJson1(db); + final index = _finalizedIndexName(exact); Future ready() async => (await db.rawQuery( "SELECT 1 FROM sqlite_master WHERE type = 'index' AND name = ?", - [_dayResultFinalizedIndex], + [index], )) .isNotEmpty; if (await ready()) return true; @@ -11011,14 +11062,14 @@ class LocalDb { final hi = (page.last['r'] as num).toInt(); await db.rawUpdate( 'UPDATE day_result ' - 'SET imported = (${_importedPayloadSql('payload_json')}) ' + 'SET imported = (${_importedPayloadSql('payload_json', exact: exact)}) ' 'WHERE rowid > ? AND rowid <= ?', [after, hi], ); after = hi; } await db.execute( - 'CREATE INDEX IF NOT EXISTS $_dayResultFinalizedIndex ' + 'CREATE INDEX IF NOT EXISTS $index ' 'ON day_result(algo_version, finalized, imported, day_id)', ); return true; From 036b8a610c3693329b488da5d274fbec54f4a1a8 Mon Sep 17 00:00:00 2001 From: MikenRafter Date: Tue, 17 Nov 2026 15:55:52 -0700 Subject: [PATCH 07/10] test: say which schema rungs the checkpoint storage test accepts --- test/ed2b_checkpoints_test.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/ed2b_checkpoints_test.dart b/test/ed2b_checkpoints_test.dart index afd8d4263..e3872c813 100644 --- a/test/ed2b_checkpoints_test.dart +++ b/test/ed2b_checkpoints_test.dart @@ -111,7 +111,7 @@ void main() { edSetZone(originalZone); }); - test('E16 fresh storage uses rung 62 and a versioned composite key', () async { + test('E16 fresh storage has the checkpoint tables (rung 62 or later) and a versioned composite key', () async { expect(LocalDb.schemaVersion, greaterThanOrEqualTo(62), reason: '61 belongs to ED-2a revisions; checkpoints use the next rung'); final db = await _storage(); From 079ce67a8f74fa345bc5855b96dff2bdd1768212 Mon Sep 17 00:00:00 2001 From: MikenRafter Date: Tue, 17 Nov 2026 18:30:32 -0700 Subject: [PATCH 08/10] data: repair the imported mirror without a schema bump --- lib/data/db.dart | 7 +------ test/ed2a_input_revision_test.dart | 2 +- test/ed2b_checkpoints_test.dart | 8 ++++---- test/finalized_day_ids_index_test.dart | 4 ++-- 4 files changed, 8 insertions(+), 13 deletions(-) diff --git a/lib/data/db.dart b/lib/data/db.dart index d94e4d544..fbafb1169 100644 --- a/lib/data/db.dart +++ b/lib/data/db.dart @@ -688,7 +688,7 @@ class LocalDb { /// pass it: sqflite throws `ArgumentError('onCreate must be null if no /// version is specified')` BEFORE opening anything when `onCreate` is given /// without `version` (sqflite_common database_mixin.dart). - static const int schemaVersion = 63; + static const int schemaVersion = 62; /// In-memory cache identity, advanced after open, wipe or committed import. /// It is local to this isolate; no cache survives a process restart. @@ -1544,11 +1544,6 @@ class LocalDb { await _createStoreIdentity(db); await _createDayCheckpoint(db); } - if (oldV < 63) { - // Replace the LIKE mirror's triggers and completion marker only. - // Paged payload repair stays off the launch watchdog, on first read. - await _ensureDayResultImportedColumn(db); - } }, onOpen: (db) async { await _repairOpenSchema(db); diff --git a/test/ed2a_input_revision_test.dart b/test/ed2a_input_revision_test.dart index b053afc3c..00fc1db97 100644 --- a/test/ed2a_input_revision_test.dart +++ b/test/ed2a_input_revision_test.dart @@ -156,7 +156,7 @@ void main() { }); test('rung 61 input revisions remain present with rung 62 checkpoints', () async { - expect(LocalDb.schemaVersion, greaterThanOrEqualTo(62), reason: 'checkpoint storage follows revisions at 61'); + expect(LocalDb.schemaVersion, 62, reason: 'checkpoint storage follows revisions at 61'); final db = await open(); expect(await db.getVersion(), LocalDb.schemaVersion); expect(await _hasRevisions(db), isTrue); diff --git a/test/ed2b_checkpoints_test.dart b/test/ed2b_checkpoints_test.dart index e3872c813..41bc68409 100644 --- a/test/ed2b_checkpoints_test.dart +++ b/test/ed2b_checkpoints_test.dart @@ -111,11 +111,11 @@ void main() { edSetZone(originalZone); }); - test('E16 fresh storage has the checkpoint tables (rung 62 or later) and a versioned composite key', () async { - expect(LocalDb.schemaVersion, greaterThanOrEqualTo(62), + test('E16 fresh storage uses rung 62 and a versioned composite key', () async { + expect(LocalDb.schemaVersion, 62, reason: '61 belongs to ED-2a revisions; checkpoints use the next rung'); final db = await _storage(); - expect(await db.getVersion(), LocalDb.schemaVersion); + expect(await db.getVersion(), 62); expect(await db.query('store_identity'), hasLength(1)); expect(await LocalDb.storeIdentity(), matches(RegExp(r'^[0-9a-f]{32}$'))); final columns = {for (final row in await db.rawQuery( @@ -152,7 +152,7 @@ void main() { expect(await db.query('input_rev', orderBy: 'bucket'), revs); expect(await db.rawQuery('PRAGMA table_info(decoded_onehz)'), rawShape); await _storage(); - expect(await db.getVersion(), LocalDb.schemaVersion); + expect(await db.getVersion(), 62); expect(await db.query('day_checkpoint'), isEmpty, reason: 'migration creates empty scratch storage without deriving old days'); }); diff --git a/test/finalized_day_ids_index_test.dart b/test/finalized_day_ids_index_test.dart index 9ee49a461..e951a7f6b 100644 --- a/test/finalized_day_ids_index_test.dart +++ b/test/finalized_day_ids_index_test.dart @@ -110,10 +110,10 @@ void main() { expect(await LocalDb.finalizedDayIds(_algo), expected); }); - // WHY: schema-62 already has a completed LIKE backfill and triggers. Both + // WHY: existing databases have a completed LIKE backfill and triggers. Both // upgrade and same-version repair must invalidate that completion marker. // CLAIM(imported-exact); catches: replacing only the trigger definitions. - for (final version in [62, 63]) { + for (final version in [61, 62]) { test('repairs a stale legacy mirror opened at schema $version twice', () async { await _useFreshDb('finalized_legacy_$version.db'); var db = await LocalDb.instance; From 0c4d07ae133a8d64b18682179059c768e1dcb4a8 Mon Sep 17 00:00:00 2001 From: MikenRafter Date: Tue, 17 Nov 2026 19:31:55 -0700 Subject: [PATCH 09/10] data: invalidate the exact imported mirror after downgrade --- lib/data/db.dart | 20 ++++++++++--- test/finalized_day_ids_index_test.dart | 40 ++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/lib/data/db.dart b/lib/data/db.dart index fbafb1169..dff27d110 100644 --- a/lib/data/db.dart +++ b/lib/data/db.dart @@ -10988,11 +10988,23 @@ class LocalDb { db, 'day_result', 'imported', 'INTEGER NOT NULL DEFAULT 0', ); // Cheap every-open repair, including same-version restored databases. Only - // DERIVED objects are dropped and recreated (two triggers and an index over - // a column this method owns), so it stays additive and idempotent: a second - // open finds the right objects and changes nothing. The completion index is - // built only after the paged backfill, see [_ensureDayResultImportedFlag]. + // DERIVED objects are dropped and recreated (mirror triggers and completion + // indexes over a column this method owns), so it stays additive and + // idempotent: a second open finds the right objects and changes nothing. + // The completion index is built only after the paged backfill, see + // [_ensureDayResultImportedFlag]. final exact = await _probeJson1(db); + // CLAIM(imported-exact): an older build re-fills all rows with LIKE while + // v2 can survive. Its legacy objects invalidate that completion marker + // [finalized_day_ids_index_test: re-upgrade repairs every row]. + if (exact && (await db.rawQuery( + "SELECT 1 FROM sqlite_master WHERE " + "(type = 'index' AND name = 'idx_day_result_finalized') OR " + "(type = 'trigger' AND name IN " + "('trg_day_result_imported_ins', 'trg_day_result_imported_upd')) LIMIT 1", + )).isNotEmpty) { + await db.execute('DROP INDEX IF EXISTS idx_day_result_finalized_v2'); + } // The other mode's objects go first: a restored file can carry JSON // triggers onto a SQLite that cannot compile them. for (final stale in exact diff --git a/test/finalized_day_ids_index_test.dart b/test/finalized_day_ids_index_test.dart index e951a7f6b..49c476182 100644 --- a/test/finalized_day_ids_index_test.dart +++ b/test/finalized_day_ids_index_test.dart @@ -147,6 +147,46 @@ void main() { }); } + // WHY: an older build can backfill every row with LIKE while the v2 index + // survives. Any legacy marker must invalidate that stale exact completion. + // CLAIM(imported-exact); catches: trusting v2 after a downgrade/re-upgrade. + for (final marker in ['index', 'ins', 'upd']) { + test('re-upgrade repairs every row when legacy $marker survives', () async { + await _useFreshDb('finalized_downgrade_$marker.db'); + var db = await LocalDb.instance; + await _rawPut(db, '2026-01-01', '{"nested":{"imported":true}}'); + await _rawPut(db, '2026-01-02', '{ "imported" : true }'); + expect(await LocalDb.finalizedDayIds(_algo), {'2026-01-01'}); + expect(await _indexed(db), isTrue); + // The older build's first read re-fills ALL existing rows. Updating only + // imported does not fire either payload_json trigger, so v2 stays stale. + await db.rawUpdate( + "UPDATE day_result SET imported = (payload_json LIKE '%\"imported\":true%')", + ); + if (marker == 'index') { + await db.execute('CREATE INDEX idx_day_result_finalized ' + 'ON day_result(algo_version, finalized, imported, day_id)'); + } else { + await db.execute('CREATE TRIGGER trg_day_result_imported_$marker ' + 'AFTER ${marker == 'ins' ? 'INSERT' : 'UPDATE OF payload_json'} ' + 'ON day_result BEGIN UPDATE day_result ' + "SET imported = (NEW.payload_json LIKE '%\"imported\":true%') " + 'WHERE rowid = NEW.rowid; END'); + } + expect((await db.query('day_result', columns: ['imported'], + orderBy: 'day_id')).map((r) => r['imported']).toList(), [1, 0]); + await LocalDb.close(); + db = await LocalDb.instance; + expect(await _indexed(db), isFalse, reason: 'legacy $marker invalidates v2'); + expect(await LocalDb.finalizedDayIds(_algo), {'2026-01-01'}); + expect((await db.query('day_result', columns: ['imported'], + orderBy: 'day_id')).map((r) => r['imported']).toList(), [0, 1]); + expect(await _indexed(db), isTrue); + await LocalDb.close(); + expect(await LocalDb.finalizedDayIds(_algo), {'2026-01-01'}); + }); + } + // WHY: a refused index build must keep the fallback's classification exact. // CLAIM(imported-exact); catches: retaining LIKE only in the fallback path. test('failed index creation falls back to the exact top-level key', () async { From db5a902f75aa1263e7a6e305914ad37a484b0e11 Mon Sep 17 00:00:00 2001 From: MikenRafter Date: Tue, 17 Nov 2026 22:14:54 -0700 Subject: [PATCH 10/10] data: match decoded imported members and last duplicate values --- lib/data/db.dart | 52 +++++----- test/finalized_day_ids_index_test.dart | 126 ++++++++++++++++++++----- 2 files changed, 135 insertions(+), 43 deletions(-) diff --git a/lib/data/db.dart b/lib/data/db.dart index dff27d110..b8729ef4b 100644 --- a/lib/data/db.dart +++ b/lib/data/db.dart @@ -10936,7 +10936,7 @@ class LocalDb { /// [_ensureDayResultImportedColumn] on open. Null until the first open. static bool? _importedExactMode; - /// One constant query. True when `json_type` exists and reads a top-level + /// One constant query. True when `json_each` can read the last top-level /// boolean. The mirror's triggers fire on EVERY `day_result` write, and a /// trigger body that names a missing function cannot compile, so every /// insert would fail: the Android framework SQLite (3.18/3.22 on Android @@ -10946,7 +10946,8 @@ class LocalDb { if (debugJsonProbeFails) return _importedExactMode = false; try { final r = await db.rawQuery( - "SELECT json_type('{\"imported\":true}', '\$.imported') AS t", + "SELECT type AS t FROM json_each('{\"imported\":false,\"imported\":true}') " + "WHERE key = 'imported' ORDER BY id DESC LIMIT 1", ); return _importedExactMode = r.single['t'] == 'true'; } catch (_) { @@ -10954,27 +10955,29 @@ class LocalDb { } } - /// CLAIM(imported-exact): with JSON1, only a top-level JSON boolean true - /// marks an imported snapshot [finalized_day_ids_index_test]. Guard malformed - /// JSON so a damaged bundle cannot abort a write or the fallback scan. The - /// substring prefilter skips both JSON parses for the common write (a derived - /// bundle without the key); a spaced key still contains it. Without JSON1 + /// CLAIM(imported-exact): with JSON1, the last decoded top-level `imported` + /// member must be boolean true, matching Dart [finalized_day_ids_index_test]. + /// Guard malformed JSON so a damaged bundle cannot abort a write or scan. + /// The prefilter skips JSON parsing for ordinary derived bundles without the + /// key, but admits Unicode escapes that can spell it differently. Without JSON1 /// ([exact] false) this is the old substring test, which also counts a /// nested key. static String _importedPayloadSql(String column, {required bool exact}) => exact - ? "CASE WHEN $column LIKE '%\"imported\"%' AND json_valid($column) " - "THEN COALESCE(json_type($column, '\$.imported') = 'true', 0) " + ? "CASE WHEN ($column LIKE '%\"imported\"%' " + "OR instr($column, '\\u') > 0) AND json_valid($column) " + "THEN COALESCE((SELECT type = 'true' FROM json_each($column) " + "WHERE key = 'imported' ORDER BY id DESC LIMIT 1), 0) " "ELSE 0 END" : "($column LIKE '%\"imported\":true%')"; /// Covering index for [finalizedDayIds]. Created only once every existing /// row's `imported` is filled, so its presence is also the "backfill done" /// marker: a database restored from a file without it backfills again. The - /// exact-key backfill has its own name (an old LIKE backfill is not - /// trustworthy for it); without JSON1 the legacy name and LIKE backfill stay. + /// decoded-key backfill has its own name (neither LIKE nor v2's first-key + /// lookup is trustworthy); without JSON1 the legacy name and backfill stay. static String _finalizedIndexName(bool exact) => - exact ? 'idx_day_result_finalized_v2' : 'idx_day_result_finalized'; + exact ? 'idx_day_result_finalized_v3' : 'idx_day_result_finalized'; /// `day_result.imported` and the triggers that keep it equal to /// [_importedPayloadSql] on every insert and payload update — whichever @@ -10995,30 +10998,37 @@ class LocalDb { // [_ensureDayResultImportedFlag]. final exact = await _probeJson1(db); // CLAIM(imported-exact): an older build re-fills all rows with LIKE while - // v2 can survive. Its legacy objects invalidate that completion marker + // v3 can survive. Its legacy objects invalidate that completion marker // [finalized_day_ids_index_test: re-upgrade repairs every row]. if (exact && (await db.rawQuery( "SELECT 1 FROM sqlite_master WHERE " - "(type = 'index' AND name = 'idx_day_result_finalized') OR " + "(type = 'index' AND name IN " + "('idx_day_result_finalized', 'idx_day_result_finalized_v2')) OR " "(type = 'trigger' AND name IN " - "('trg_day_result_imported_ins', 'trg_day_result_imported_upd')) LIMIT 1", + "('trg_day_result_imported_ins', 'trg_day_result_imported_upd', " + "'trg_day_result_imported_v2_ins', 'trg_day_result_imported_v2_upd')) LIMIT 1", )).isNotEmpty) { - await db.execute('DROP INDEX IF EXISTS idx_day_result_finalized_v2'); + await db.execute('DROP INDEX IF EXISTS idx_day_result_finalized_v3'); } // The other mode's objects go first: a restored file can carry JSON // triggers onto a SQLite that cannot compile them. - for (final stale in exact + for (final stale in [ + 'trg_day_result_imported_v2_ins', + 'trg_day_result_imported_v2_upd', + ...exact ? const ['trg_day_result_imported_ins', 'trg_day_result_imported_upd'] : const [ - 'trg_day_result_imported_v2_ins', - 'trg_day_result_imported_v2_upd', - ]) { + 'trg_day_result_imported_v3_ins', + 'trg_day_result_imported_v3_upd', + ], + ]) { await db.execute('DROP TRIGGER IF EXISTS $stale'); } await db.execute( 'DROP INDEX IF EXISTS ${_finalizedIndexName(!exact)}', ); - final tag = exact ? '_v2' : ''; + await db.execute('DROP INDEX IF EXISTS idx_day_result_finalized_v2'); + final tag = exact ? '_v3' : ''; await db.execute( 'CREATE TRIGGER IF NOT EXISTS trg_day_result_imported${tag}_ins ' 'AFTER INSERT ON day_result BEGIN ' diff --git a/test/finalized_day_ids_index_test.dart b/test/finalized_day_ids_index_test.dart index 49c476182..175923430 100644 --- a/test/finalized_day_ids_index_test.dart +++ b/test/finalized_day_ids_index_test.dart @@ -37,7 +37,7 @@ Future> _dartOracle(Database db) async => { Future _indexed(Database db) async => (await db.rawQuery( "SELECT 1 FROM sqlite_master WHERE type = 'index' " - "AND name = 'idx_day_result_finalized_v2'", + "AND name = 'idx_day_result_finalized_v3'", )).isNotEmpty; Future _put(String day, String payload, {bool finalized = true}) => @@ -83,6 +83,14 @@ void main() { '{"imported":"true"}': false, '{"imported":null}': false, '{ "imported" : true }': true, + r'{"\u0069mported":true}': true, + r'{"nested":{"\u0069mported":true}}': false, + '{"imported":false,"imported":true}': true, + '{"imported":true,"imported":false}': false, + r'{"imported":false,"\u0069mported":true}': true, + r'{"\u0069mported":true,"imported":null}': false, + '{"imported":true,"imported":1}': false, + '{"imported":true,"nested":{"imported":false}}': true, 'invalid JSON': false, }; final expected = {}; @@ -99,6 +107,7 @@ void main() { entry.value ? 1 : 0, reason: entry.key); } expect(await LocalDb.finalizedDayIds(_algo), expected); + expect(expected, await _dartOracle(db)); // Raw SQL payload rewrites also go through the mirror's update trigger. await db.update('day_result', {'payload_json': '{ "imported" : true }'}, where: 'day_id = ?', whereArgs: ['2026-01-01']); @@ -110,6 +119,34 @@ void main() { expect(await LocalDb.finalizedDayIds(_algo), expected); }); + // WHY: Dart decodes escaped names and keeps the last duplicate member; + // SQLite path lookup and a literal-name prefilter disagree with that reader. + // CLAIM(imported-exact); catches: literal-only prefilter, first-member lookup. + for (final entry in { + 'escaped': r'{"\u0069mported":true}', + 'duplicate': '{"imported":false,"imported":true}', + }.entries) { + test('${entry.key} imported member matches Dart on insert update and backfill', + () async { + await _useFreshDb('finalized_${entry.key}.db'); + var db = await LocalDb.instance; + await _rawPut(db, '2026-01-01', entry.value); + expect(await _dartOracle(db), isEmpty); + expect(await LocalDb.finalizedDayIds(_algo), await _dartOracle(db)); + await db.update('day_result', {'payload_json': '{}'}); + expect(await LocalDb.finalizedDayIds(_algo), {'2026-01-01'}); + await db.update('day_result', {'payload_json': entry.value}); + expect(await LocalDb.finalizedDayIds(_algo), await _dartOracle(db)); + // Force an existing-row backfill, independently of either write trigger. + await db.execute('DROP INDEX idx_day_result_finalized_v3'); + await db.update('day_result', {'imported': 0}); + await LocalDb.close(); + db = await LocalDb.instance; + expect(await LocalDb.finalizedDayIds(_algo), await _dartOracle(db)); + expect((await db.query('day_result')).single['imported'], 1); + }); + } + // WHY: existing databases have a completed LIKE backfill and triggers. Both // upgrade and same-version repair must invalidate that completion marker. // CLAIM(imported-exact); catches: replacing only the trigger definitions. @@ -117,11 +154,11 @@ void main() { test('repairs a stale legacy mirror opened at schema $version twice', () async { await _useFreshDb('finalized_legacy_$version.db'); var db = await LocalDb.instance; - await db.execute('DROP TRIGGER IF EXISTS trg_day_result_imported_v2_ins'); - await db.execute('DROP TRIGGER IF EXISTS trg_day_result_imported_v2_upd'); + await db.execute('DROP TRIGGER IF EXISTS trg_day_result_imported_v3_ins'); + await db.execute('DROP TRIGGER IF EXISTS trg_day_result_imported_v3_upd'); await db.execute('DROP TRIGGER IF EXISTS trg_day_result_imported_ins'); await db.execute('DROP TRIGGER IF EXISTS trg_day_result_imported_upd'); - await db.execute('DROP INDEX IF EXISTS idx_day_result_finalized_v2'); + await db.execute('DROP INDEX IF EXISTS idx_day_result_finalized_v3'); await db.execute('CREATE INDEX IF NOT EXISTS idx_day_result_finalized ' 'ON day_result(algo_version, finalized, imported, day_id)'); for (final suffix in ['ins', 'upd']) { @@ -147,9 +184,51 @@ void main() { }); } - // WHY: an older build can backfill every row with LIKE while the v2 index + // WHY: a restored draft-v2 database carries both the old classifier's + // triggers and its completion marker. Reopening must replace and rebackfill. + // CLAIM(imported-exact); catches: CREATE IF NOT EXISTS retaining old SQL. + test('reopening replaces the old exact mirror and repairs its completed rows', + () async { + await _useFreshDb('finalized_previous_exact.db'); + var db = await LocalDb.instance; + // Remove the current mirror, then recreate the previous draft's objects. + for (final r in await db.rawQuery( + "SELECT name FROM sqlite_master WHERE type = 'trigger' " + "AND tbl_name = 'day_result'", + )) { + await db.execute('DROP TRIGGER ${r['name']}'); + } + await db.execute('DROP INDEX IF EXISTS idx_day_result_finalized_v3'); + await db.execute('DROP INDEX IF EXISTS idx_day_result_finalized_v2'); + for (final suffix in ['ins', 'upd']) { + await db.execute('CREATE TRIGGER trg_day_result_imported_v2_$suffix ' + 'AFTER ${suffix == 'ins' ? 'INSERT' : 'UPDATE OF payload_json'} ' + 'ON day_result BEGIN UPDATE day_result ' + "SET imported = CASE WHEN NEW.payload_json LIKE '%\"imported\"%' " + 'AND json_valid(NEW.payload_json) THEN ' + "COALESCE(json_type(NEW.payload_json, '\$.imported') = 'true', 0) " + 'ELSE 0 END WHERE rowid = NEW.rowid; END'); + } + await _rawPut(db, '2026-01-01', r'{"\u0069mported":true}'); + await _rawPut(db, '2026-01-02', '{"imported":false,"imported":true}'); + await db.execute('CREATE INDEX idx_day_result_finalized_v2 ' + 'ON day_result(algo_version, finalized, imported, day_id)'); + expect((await db.query('day_result')).map((r) => r['imported']), [0, 0]); + await LocalDb.close(); + db = await LocalDb.instance; + expect(await LocalDb.finalizedDayIds(_algo), await _dartOracle(db)); + await _rawPut(db, '2026-01-03', r'{"\u0069mported":true}'); + await db.update('day_result', { + 'payload_json': '{"imported":true,"imported":false}', + }, where: 'day_id = ?', whereArgs: ['2026-01-02']); + expect(await LocalDb.finalizedDayIds(_algo), {'2026-01-02'}); + await LocalDb.close(); + expect(await LocalDb.finalizedDayIds(_algo), {'2026-01-02'}); + }); + + // WHY: an older build can backfill every row with LIKE while the v3 index // survives. Any legacy marker must invalidate that stale exact completion. - // CLAIM(imported-exact); catches: trusting v2 after a downgrade/re-upgrade. + // CLAIM(imported-exact); catches: trusting v3 after a downgrade/re-upgrade. for (final marker in ['index', 'ins', 'upd']) { test('re-upgrade repairs every row when legacy $marker survives', () async { await _useFreshDb('finalized_downgrade_$marker.db'); @@ -159,7 +238,7 @@ void main() { expect(await LocalDb.finalizedDayIds(_algo), {'2026-01-01'}); expect(await _indexed(db), isTrue); // The older build's first read re-fills ALL existing rows. Updating only - // imported does not fire either payload_json trigger, so v2 stays stale. + // imported does not fire either payload_json trigger, so v3 stays stale. await db.rawUpdate( "UPDATE day_result SET imported = (payload_json LIKE '%\"imported\":true%')", ); @@ -177,7 +256,7 @@ void main() { orderBy: 'day_id')).map((r) => r['imported']).toList(), [1, 0]); await LocalDb.close(); db = await LocalDb.instance; - expect(await _indexed(db), isFalse, reason: 'legacy $marker invalidates v2'); + expect(await _indexed(db), isFalse, reason: 'legacy $marker invalidates v3'); expect(await LocalDb.finalizedDayIds(_algo), {'2026-01-01'}); expect((await db.query('day_result', columns: ['imported'], orderBy: 'day_id')).map((r) => r['imported']).toList(), [0, 1]); @@ -192,14 +271,17 @@ void main() { test('failed index creation falls back to the exact top-level key', () async { await _useFreshDb('finalized_fallback.db'); final db = await LocalDb.instance; - await db.execute('CREATE TABLE idx_day_result_finalized_v2 (id INTEGER)'); + await db.execute('CREATE TABLE idx_day_result_finalized_v3 (id INTEGER)'); await _rawPut(db, '2026-01-01', '{"nested":{"imported":true}}'); await _rawPut(db, '2026-01-02', '{ "imported" : true }'); await _rawPut(db, '2026-01-03', 'invalid JSON'); - expect(await LocalDb.finalizedDayIds(_algo), {'2026-01-01', '2026-01-03'}); + await _rawPut(db, '2026-01-04', r'{"\u0069mported":true}'); + await _rawPut(db, '2026-01-05', '{"imported":false,"imported":true}'); + await _rawPut(db, '2026-01-06', '{"imported":true,"imported":false}'); + expect(await LocalDb.finalizedDayIds(_algo), await _dartOracle(db)); expect(await _indexed(db), isFalse); - await db.execute('DROP TABLE idx_day_result_finalized_v2'); - expect(await LocalDb.finalizedDayIds(_algo), {'2026-01-01', '2026-01-03'}); + await db.execute('DROP TABLE idx_day_result_finalized_v3'); + expect(await LocalDb.finalizedDayIds(_algo), await _dartOracle(db)); expect(await _indexed(db), isTrue); }); @@ -231,9 +313,9 @@ void main() { await _useFreshDb('finalized_ids_backfill.db'); final db = await LocalDb.instance; // Emulate a pre-column database: no triggers, no index, stale flags. - await db.execute('DROP TRIGGER IF EXISTS trg_day_result_imported_v2_ins'); - await db.execute('DROP TRIGGER IF EXISTS trg_day_result_imported_v2_upd'); - await db.execute('DROP INDEX IF EXISTS idx_day_result_finalized_v2'); + await db.execute('DROP TRIGGER IF EXISTS trg_day_result_imported_v3_ins'); + await db.execute('DROP TRIGGER IF EXISTS trg_day_result_imported_v3_upd'); + await db.execute('DROP INDEX IF EXISTS idx_day_result_finalized_v3'); for (var i = 0; i < 450; i++) { final day = '2025-${(1 + i ~/ 28).toString().padLeft(2, '0')}-' @@ -263,7 +345,7 @@ void main() { ); expect( plan.map((r) => '${r['detail']}').join(' '), - contains('COVERING INDEX idx_day_result_finalized_v2'), + contains('COVERING INDEX idx_day_result_finalized_v3'), ); }); @@ -281,7 +363,7 @@ void main() { )).isNotEmpty; // WHY: day_result is written by every derive. A trigger body that calls - // json_valid/json_type cannot compile on a SQLite without JSON1 (the Android + // json_valid/json_each cannot compile on a SQLite without JSON1 (the Android // framework builds some devices ship), and then EVERY insert fails, so no day // could be saved. Without JSON1 the legacy LIKE mirror must stay in place. // CLAIM(imported-exact); catches: a JSON1 requirement on the write path. @@ -305,7 +387,7 @@ void main() { orderBy: 'day_id')).map((r) => r['imported']).toList(), [1, 0]); expect(await LocalDb.finalizedDayIds(_algo), {'2026-01-02'}); expect(await hasIndex(db, 'idx_day_result_finalized'), isTrue); - expect(await hasIndex(db, 'idx_day_result_finalized_v2'), isFalse); + expect(await hasIndex(db, 'idx_day_result_finalized_v3'), isFalse); }); // WHY: a database file restored from a JSON1 device onto one without JSON1 @@ -317,16 +399,16 @@ void main() { await _useFreshDb('finalized_restored_json.db'); var db = await LocalDb.instance; expect(await triggers(db), [ - 'trg_day_result_imported_v2_ins', - 'trg_day_result_imported_v2_upd', + 'trg_day_result_imported_v3_ins', + 'trg_day_result_imported_v3_upd', ]); await LocalDb.finalizedDayIds(_algo); - expect(await hasIndex(db, 'idx_day_result_finalized_v2'), isTrue); + expect(await hasIndex(db, 'idx_day_result_finalized_v3'), isTrue); await LocalDb.close(); LocalDb.debugJsonProbeFails = true; db = await LocalDb.instance; expect(await triggers(db), ['trg_day_result_imported_ins', 'trg_day_result_imported_upd']); - expect(await hasIndex(db, 'idx_day_result_finalized_v2'), isFalse); + expect(await hasIndex(db, 'idx_day_result_finalized_v3'), isFalse); }); }