feat: sp hardcore scanning changes

lib/src/provider/electrum_methods/methods.dart

@@ -27,4 +27,5 @@ export 'methods/relay_fee.dart';
 export 'methods/scripthash_unsubscribe.dart';
 export 'methods/server_peer_subscribe.dart';
 export 'methods/status.dart';
+export 'methods/tweaks_get.dart';
 export 'methods/tweaks_subscribe.dart';

lib/src/provider/electrum_methods/methods/tweaks_get.dart

@@ -0,0 +1,62 @@
+import 'package:bitcoin_base/src/provider/service/electrum/electrum.dart';
+
+class ElectrumTweaksGetResponse {
+  final int amount;
+  final bool spent;
+
+  ElectrumTweaksGetResponse({required this.amount, required this.spent});
+
+  static ElectrumTweaksGetResponse fromJson(Map<String, dynamic> json) {
+    return ElectrumTweaksGetResponse(
+      amount: json['amount'] as int,
+      spent: json['spent'] as bool,
+    );
+  }
+}
+
+/// Single-shot lookup of one Silent Payments output's resolved amount and
+/// spent status — the two-pass amount fetch's second pass (ADR-0015),
+/// issued once a `blockchain.tweaks.subscribe` match is found (v1's
+/// `blockTweaks` and v2's decoded records both carry the output pubkey and
+/// vout, but never an amount).
+///
+/// blockchain.tweaks.get
+/// https://github.com/Blockstream/electrs (electrs-tweaks fork,
+/// `src/electrum/server.rs`'s `blockchain_tweaks_get`)
+class ElectrumTweaksGet extends ElectrumRequest<ElectrumTweaksGetResponse, Map<String, dynamic>> {
+  ElectrumTweaksGet({required this.txid, required this.vout, required this.height});
+
+  /// The funding transaction's txid, as conventional display (big-endian)
+  /// hex — same convention as every other JSON-carried txid in this
+  /// codebase (unlike the v2 *binary* wire protocol, which uses
+  /// internal/reversed order; this is a plain JSON-RPC call, not that
+  /// format).
+  final String txid;
+  final int vout;
+
+  /// The funding transaction's block height (the height the match was
+  /// found at, `tweakHeight` in `_handleScanSilentPayments`) — NOT the
+  /// spending transaction's height if this output was later spent. The
+  /// server keys its lookup by `(height, txid)` exactly, so passing the
+  /// wrong height here fails with a "no tweak row" error even when the
+  /// txid/vout are correct.
+  final int height;
+
+  @override
+  String get method => ElectrumRequestMethods.tweaksGet.method;
+
+  @override
+  List toParams() => [txid, vout, height];
+
+  /// Throws `RPCError` (from `package:blockchain_utils`) on a server-side
+  /// JSON-RPC error — e.g. no tweak row at this (txid, height), or an
+  /// ineligible vout. That is a structural, permanent failure for this
+  /// specific match (a malformed/incorrect call), not a transient
+  /// connection problem — callers must not fold it into ADR-0015's
+  /// indefinite connection-retry loop, or a single unresolvable match
+  /// wedges the worker's range advancement forever.
+  @override
+  ElectrumTweaksGetResponse onResponse(Map<String, dynamic> result) {
+    return ElectrumTweaksGetResponse.fromJson(result);
+  }
+}

lib/src/provider/electrum_methods/methods/tweaks_subscribe.dart

@@ -1,3 +1,5 @@
+import 'dart:convert';
+
 import 'package:bitcoin_base/src/provider/service/electrum/electrum.dart';
 
 class TweakOutputData {
@@ -24,17 +26,47 @@ class ElectrumTweaksSubscribeResponse {
   final int block;
   final Map<String, TweakData> blockTweaks;
 
+  /// Raw v2 block-record bytes (already base64-decoded), present only when
+  /// this response used the compact binary protocol (`protocolVersion: 2`
+  /// on the request) — see electrs-tweaks's `doc/tweaks_v2_protocol.md`.
+  /// Hand these straight to `sp_scanner`'s `ScanSession.scanBlock`; do not
+  /// attempt to parse them here — the decoder lives in `sp_scanner`, not
+  /// this package (sp-scan-bench/docs/adr/0002-binary-protocol-decode-in-sp-scanner.md).
+  /// When present, `blockTweaks` is empty and `block` is not derivable from
+  /// this response alone (the height is inside the blob itself).
+  final List<int>? tweaksV2Bytes;
+
   ElectrumTweaksSubscribeResponse({
     required this.block,
     required this.blockTweaks,
     this.message,
+    this.tweaksV2Bytes,
   });
 
   static ElectrumTweaksSubscribeResponse? fromJson(Map<String, dynamic> json) {
     if (json.isEmpty) {
       return null;
     }
 
+    // Per-block (and per-message) notifications arrive already unwrapped to
+    // `params[0]` by the RPC layer (`_findResult` in electrum_tcp_service.dart
+    // returns `data["params"]?[0]` for method-matched, id-less messages) — so
+    // a v2 block notification is `json` itself being `{"tweaks_v2": "..."}`,
+    // the same way a v1 block notification is `json` itself being
+    // `{"<height>": {...}}`. This must be checked before the `containsKey
+    // ('params')` branch below, which instead detects the *terminal*
+    // id-matched RPC result (the one message on this stream that still has
+    // its outer `{jsonrpc, method, params}` envelope intact — see
+    // electrs-tweaks's doc/tweaks_v2_protocol.md §4).
+    final tweaksV2 = json['tweaks_v2'];
+    if (tweaksV2 is String) {
+      return ElectrumTweaksSubscribeResponse(
+        block: 0,
+        blockTweaks: const {},
+        tweaksV2Bytes: base64Decode(tweaksV2),
+      );
+    }
+
     if (json.containsKey('params')) {
       final params = json['params'] as List<dynamic>;
       final message = params.first["message"];
@@ -94,22 +126,37 @@ class ElectrumTweaksSubscribeResponse {
 class ElectrumTweaksSubscribe
     extends ElectrumRequest<ElectrumTweaksSubscribeResponse?, Map<String, dynamic>> {
   /// blockchain.tweaks.subscribe
+  ///
+  /// [protocolVersion] is optional and omitted from the wire request by
+  /// default, matching every caller before this field existed. Only pass it
+  /// after negotiating (never blindly): `2` requests the compact binary
+  /// protocol (see electrs-tweaks's `doc/tweaks_v2_protocol.md`) and the
+  /// server hard-errors on a version it doesn't support rather than
+  /// silently downgrading, so a caller must first confirm the server
+  /// advertises `>= 2` (via `max_protocol_version` on a prior result) and
+  /// that this build's `sp_scanner` decoder also supports it
+  /// (`sp_scanner.maxWireVersion()`) before setting this.
   ElectrumTweaksSubscribe({
     required this.height,
     required this.count,
     required this.historicalMode,
+    this.protocolVersion,
   });
 
   final int height;
   final int count;
   final bool historicalMode;
+  final int? protocolVersion;
 
   @override
   String get method => ElectrumRequestMethods.tweaksSubscribe.method;
 
   @override
   List toParams() {
-    return [height, count, historicalMode];
+    if (protocolVersion == null) {
+      return [height, count, historicalMode];
+    }
+    return [height, count, historicalMode, protocolVersion];
   }
 
   /// The header of the current block chain tip.

lib/src/provider/service/electrum/electrum_tcp_service.dart

@@ -254,6 +254,23 @@ class ElectrumTCPService implements BitcoinBaseElectrumRPCService {
               }
             }
           }
+        } else if (id != null) {
+          // Any other server-side RPC error (rejected subscribe/request
+          // outside the two shapes special-cased above) used to fall
+          // through to the plain `return data["result"] ?? ...` below,
+          // which is null for an error response — silently delivered to
+          // the caller as if it were a normal (empty) result, with no way
+          // for a subscriber's error handler or a one-shot caller's catch
+          // block to ever see it. Forward it to this id's own task the
+          // same way the batch-limit case already does for its tasks.
+          final task = _tasks[id];
+          if (task != null) {
+            if (task.isSubscription) {
+              task.subject?.addError(_errors[id]!);
+            } else {
+              task.completer?.completeError(_errors[id]!);
+            }
+          }
         }
       }
     }

lib/src/provider/service/electrum/methods.dart

@@ -94,6 +94,13 @@ class ElectrumRequestMethods {
   static const ElectrumRequestMethods tweaksSubscribe =
       ElectrumRequestMethods._(tweaksSubscribeMethod);
 
+  /// Single-shot lookup of one Silent Payments output's amount and spent
+  /// status, by (txid, vout, funding height) — the two-pass amount fetch's
+  /// second pass (ADR-0015), used once a `blockchain.tweaks.subscribe`
+  /// match is found.
+  static const String tweaksGetMethod = "blockchain.tweaks.get";
+  static const ElectrumRequestMethods tweaksGet = ElectrumRequestMethods._(tweaksGetMethod);
+
   /// Return the minimum fee a low-priority transaction must pay in order to be accepted to the daemon’s memory pool.
   static const ElectrumRequestMethods relayFee = ElectrumRequestMethods._("blockchain.relayfee");
 

test/tweaks_get_test.dart

@@ -0,0 +1,42 @@
+// Tests for `blockchain.tweaks.get` (ElectrumTweaksGet), the two-pass
+// amount fetch's second pass (ADR-0015) added for the sp-scan-bench Silent
+// Payments scan-speed effort. Request/response shape verified against the
+// real server implementation (electrs-tweaks's `src/electrum/server.rs`,
+// `blockchain_tweaks_get`), not guessed: params `[txid, vout, height]`,
+// response `{"amount": <u64>, "spent": <bool>}`.
+import 'package:bitcoin_base/bitcoin_base.dart';
+import 'package:test/test.dart';
+
+void main() {
+  group('ElectrumTweaksGet request params', () {
+    test('sends txid, vout, height in that order', () {
+      final req = ElectrumTweaksGet(
+        txid: '38088f720c1f30e5c54a56e385984ebd11d6855b14d6158f8833771501709e68',
+        vout: 0,
+        height: 112,
+      );
+
+      expect(req.toParams(),
+          ['38088f720c1f30e5c54a56e385984ebd11d6855b14d6158f8833771501709e68', 0, 112]);
+    });
+
+    test('method is blockchain.tweaks.get', () {
+      final req = ElectrumTweaksGet(txid: 'ab' * 32, vout: 1, height: 500);
+      expect(req.method, 'blockchain.tweaks.get');
+    });
+  });
+
+  group('ElectrumTweaksGetResponse.fromJson', () {
+    test('parses a real unspent response shape', () {
+      final response = ElectrumTweaksGetResponse.fromJson({'amount': 99990000, 'spent': false});
+      expect(response.amount, 99990000);
+      expect(response.spent, false);
+    });
+
+    test('parses a spent response shape', () {
+      final response = ElectrumTweaksGetResponse.fromJson({'amount': 1234, 'spent': true});
+      expect(response.amount, 1234);
+      expect(response.spent, true);
+    });
+  });
+}

test/tweaks_subscribe_test.dart

@@ -0,0 +1,128 @@
+// Tests for the blockchain.tweaks.subscribe v1/v2 request+response plumbing
+// added for the sp-scan-bench Silent Payments scan-speed effort. The v2
+// fixture bytes below are the real capture from electrs-tweaks's regtest
+// build (electrs-tweaks/doc/tweaks_v2_fixture.md — a genuine BIP-352
+// payment, not synthesized), byte-layout spec in
+// electrs-tweaks/doc/tweaks_v2_protocol.md. This package only needs to
+// recognize and pass through the `tweaks_v2` blob unparsed — the decoder
+// itself lives in `sp_scanner`
+// (sp-scan-bench/docs/adr/0002-binary-protocol-decode-in-sp-scanner.md).
+//
+// IMPORTANT on JSON shapes used here: `fromJson` is called with whatever
+// `ElectrumTcpService._findResult` (electrum_tcp_service.dart) hands the
+// subscription's stream. That layer already unwraps every per-block/
+// per-message *notification* to `params[0]` before it reaches here (so a
+// real v1 block notification arrives as `{"<height>": {...}}` directly, not
+// wrapped in `{jsonrpc, method, params}`) — EXCEPT the one terminal
+// id-matched RPC result for the original subscribe call, which keeps its
+// full envelope intact and lands on the same stream. Both shapes are
+// exercised below; getting this wrong silently breaks the corresponding
+// `fromJson` branch without any test noticing.
+import 'package:bitcoin_base/bitcoin_base.dart';
+import 'package:test/test.dart';
+
+void main() {
+  group('ElectrumTweaksSubscribe request params', () {
+    test('omits protocolVersion from the wire request when not set', () {
+      final req = ElectrumTweaksSubscribe(height: 1, count: 1000, historicalMode: true);
+      expect(req.toParams(), [1, 1000, true]);
+    });
+
+    test('includes protocolVersion when explicitly negotiated', () {
+      final req = ElectrumTweaksSubscribe(
+          height: 1, count: 1000, historicalMode: true, protocolVersion: 2);
+      expect(req.toParams(), [1, 1000, true, 2]);
+    });
+  });
+
+  group('ElectrumTweaksSubscribeResponse.fromJson — v1 unchanged', () {
+    test('parses a real v1 block notification (electrs-tweaks fixture, height 112), unwrapped', () {
+      // Unwrapped: this is what actually arrives on the subscription stream
+      // for a per-block push — see the file-level note above.
+      final json = {
+        '112': {
+          '38088f720c1f30e5c54a56e385984ebd11d6855b14d6158f8833771501709e68': {
+            'output_pubkeys': {
+              '0': ['46db9bd8d491531b2e783d32e07acb0624093fcdad75573e7e6da39ac21d0c13', 99990000]
+            },
+            'tweak': '0302cc2a75db0e06919f9d3312c17c831b68ff1ead95498a529fd366d64921e3c0',
+          }
+        }
+      };
+
+      final response = ElectrumTweaksSubscribeResponse.fromJson(json);
+      expect(response, isNotNull);
+      expect(response!.block, 112);
+      expect(response.message, isNull);
+      expect(response.tweaksV2Bytes, isNull);
+      final tx = response
+          .blockTweaks['38088f720c1f30e5c54a56e385984ebd11d6855b14d6158f8833771501709e68']!;
+      expect(tx.tweak, '0302cc2a75db0e06919f9d3312c17c831b68ff1ead95498a529fd366d64921e3c0');
+      expect(
+        tx.outputPubkeys['46db9bd8d491531b2e783d32e07acb0624093fcdad75573e7e6da39ac21d0c13']!
+            .amount,
+        99990000,
+      );
+    });
+
+    test('the mid-stream "done" notification (unwrapped) yields a message-bearing, non-null response', () {
+      // This is the per-response-cycle "no more blocks in this batch" signal
+      // — the caller checks `response.message != null`, not `response ==
+      // null`, to detect it (electrum_wallet.dart's listenFn: `noData =
+      // response.message != null`).
+      final response = ElectrumTweaksSubscribeResponse.fromJson({'message': 'done'});
+      expect(response, isNotNull);
+      expect(response!.message, 'done');
+    });
+
+    test('the terminal id-matched RPC result (still envelope-wrapped) yields null', () {
+      // Unlike every notification, the one response that completes the
+      // original subscribe call's own id keeps its outer envelope intact
+      // when it lands on the same subscription stream (see the file-level
+      // note). This is the shape electrs-tweaks's doc/tweaks_v2_protocol.md
+      // §4 describes for `max_protocol_version`'s home — fromJson discards
+      // it as a harmless echo of the same "done" the mid-stream notification
+      // already delivered.
+      final json = {
+        'jsonrpc': '2.0',
+        'method': 'blockchain.tweaks.subscribe',
+        'params': [
+          {'max_protocol_version': 2, 'message': 'done'}
+        ],
+      };
+      expect(ElectrumTweaksSubscribeResponse.fromJson(json), isNull);
+    });
+  });
+
+  group('ElectrumTweaksSubscribeResponse.fromJson — v2 pass-through', () {
+    test('decodes the tweaks_v2 base64 field to raw bytes, unparsed', () {
+      // electrs-tweaks/doc/tweaks_v2_fixture.md blob 2 (height 112, the
+      // genuine BIP-352 payment) — raw hex, reproduced here as the base64
+      // that was actually captured on the wire. Unwrapped, matching how a
+      // real v2 block notification arrives (see file-level note).
+      const v2Base64 =
+          'cAAAAAFonnABFXcziI8V1hRbhdYRvU6YheNWSsXlMB8Mco8IOAMCzCp12w4GkZ+dMxLBfIMbaP8erZVJilKf02bWSSHjwAEARtub2NSRUxsueD0y4HrLBiQJP82tdVc+fm2jmsIdDBM=';
+      const expectedHex =
+          '7000000001689e7001157733888f15d6145b85d611bd4e9885e3564ac5e5301f0c728f0838'
+          '0302cc2a75db0e06919f9d3312c17c831b68ff1ead95498a529fd366d64921e3c0'
+          '010046db9bd8d491531b2e783d32e07acb0624093fcdad75573e7e6da39ac21d0c13';
+
+      final response = ElectrumTweaksSubscribeResponse.fromJson({'tweaks_v2': v2Base64});
+
+      expect(response, isNotNull);
+      expect(response!.tweaksV2Bytes, isNotNull);
+      expect(response.blockTweaks, isEmpty);
+
+      final actualHex =
+          response.tweaksV2Bytes!.map((b) => b.toRadixString(16).padLeft(2, '0')).join();
+      expect(actualHex, expectedHex);
+    });
+
+    test('decodes the zero-tx empty-tail bookmark blob (height 1), unwrapped', () {
+      const v2Base64 = 'AQAAAAA='; // height=1 (u32 LE), tx_count=0
+      final response = ElectrumTweaksSubscribeResponse.fromJson({'tweaks_v2': v2Base64});
+      expect(response, isNotNull);
+      expect(response!.tweaksV2Bytes, [0x01, 0x00, 0x00, 0x00, 0x00]);
+    });
+  });
+}