From 0659f496a8e16bd2924256e5fcc13e5f36c50f30 Mon Sep 17 00:00:00 2001 From: ZhuchkaTriplesix Date: Thu, 15 Oct 2026 07:37:21 +0300 Subject: [PATCH] perf(security): scan PII column names and values without regex detectPiiColumn ran two regexes, replaceAllMapped / replaceAll / split / where / toList / join and a third regex for every column of every query; maskPiiValue compiled RegExp(r'\D') and built a digits string for every phone and card cell. Column names are tokenised in one pass over code units, the api_key rule is a neighbour check on the tokens, and the answer per name is cached (bounded, per isolate). Phone and card masking count digits and read the last ones from the end without a cleaned string. Output is unchanged; tests compare with the previous implementation on listed and random inputs. Closes #1540 Closes #1516 --- lib/core/security/pii_masker.dart | 118 ++++++++++---- .../security/pii_masker_scanner_test.dart | 147 ++++++++++++++++++ 2 files changed, 236 insertions(+), 29 deletions(-) create mode 100644 test/core/security/pii_masker_scanner_test.dart diff --git a/lib/core/security/pii_masker.dart b/lib/core/security/pii_masker.dart index 6daa724e..e74718ae 100644 --- a/lib/core/security/pii_masker.dart +++ b/lib/core/security/pii_masker.dart @@ -15,39 +15,65 @@ const _phoneTokens = { }; const _cardTokens = {'card', 'cc', 'pan', 'iban'}; +bool _isLower(int c) => c >= 0x61 && c <= 0x7A; +bool _isUpper(int c) => c >= 0x41 && c <= 0x5A; +bool _isDigit(int c) => c >= 0x30 && c <= 0x39; + /// Splits a column name into lower-case words: on non-alphanumerics and on /// camelCase boundaries (`userEmail` -> `user`, `email`). +/// +/// One pass over the code units (no regex, no split / join). A boundary is a +/// lower-case letter or digit followed by an upper-case letter. List _tokens(String name) { - final spaced = name - .replaceAllMapped( - RegExp(r'([a-z0-9])([A-Z])'), - (m) => '${m[1]} ${m[2]}', - ) - .replaceAll(RegExp(r'[^A-Za-z0-9]+'), ' '); - return spaced - .toLowerCase() - .split(' ') - .where((t) => t.isNotEmpty) - .toList(); + final tokens = []; + var start = -1; + for (var i = 0; i < name.length; i++) { + final c = name.codeUnitAt(i); + final alnum = _isLower(c) || _isUpper(c) || _isDigit(c); + if (!alnum) { + if (start >= 0) { + tokens.add(name.substring(start, i).toLowerCase()); + start = -1; + } + continue; + } + if (start >= 0 && _isUpper(c)) { + final prev = name.codeUnitAt(i - 1); + if (_isLower(prev) || _isDigit(prev)) { + tokens.add(name.substring(start, i).toLowerCase()); + start = i; + continue; + } + } + if (start < 0) start = i; + } + if (start >= 0) tokens.add(name.substring(start).toLowerCase()); + return tokens; } -/// What [columnName] probably holds, or null when it does not look sensitive. -/// -/// Matching is per word, so `user_email` and `userEmail` match but `hotel` and -/// `email_verified_at` do not trip the phone / email rules by accident. -PiiKind? detectPiiColumn(String columnName) { +const _keyPrefixTokens = {'api', 'private', 'access', 'secret', 'auth'}; + +/// Timestamps / flags *about* a sensitive field are not the value itself. +const _metaTokens = {'at', 'verified', 'confirmed', 'sent', 'count', 'id'}; + +/// Most distinct column names remembered by [detectPiiColumn]. +const int _detectCacheLimit = 1024; + +/// Column names repeat (`id`, `name`, `created_at`...) from query to query, so +/// the answer per name is kept. Per isolate; cleared when it gets large. +final Map _detectCache = {}; + +PiiKind? _detectUncached(String columnName) { final tokens = _tokens(columnName); if (tokens.isEmpty) return null; // `api_key`, `private_key`, `access_key` are split into two words. - final joined = tokens.join('_'); - if (RegExp(r'(^|_)(api|private|access|secret|auth)_key($|_)') - .hasMatch(joined)) { - return PiiKind.secret; + for (var i = 0; i + 1 < tokens.length; i++) { + if (tokens[i + 1] == 'key' && _keyPrefixTokens.contains(tokens[i])) { + return PiiKind.secret; + } } - // Timestamps / flags *about* a sensitive field are not the value itself. - const metaTokens = {'at', 'verified', 'confirmed', 'sent', 'count', 'id'}; final lastToken = tokens.last; - if (metaTokens.contains(lastToken) && tokens.length > 1) return null; + if (_metaTokens.contains(lastToken) && tokens.length > 1) return null; for (final t in tokens) { if (_secretTokens.contains(t)) return PiiKind.secret; if (_cardTokens.contains(t)) return PiiKind.card; @@ -57,6 +83,18 @@ PiiKind? detectPiiColumn(String columnName) { return null; } +/// What [columnName] probably holds, or null when it does not look sensitive. +/// +/// Matching is per word, so `user_email` and `userEmail` match but `hotel` and +/// `email_verified_at` do not trip the phone / email rules by accident. +PiiKind? detectPiiColumn(String columnName) { + if (_detectCache.containsKey(columnName)) return _detectCache[columnName]; + final kind = _detectUncached(columnName); + if (_detectCache.length >= _detectCacheLimit) _detectCache.clear(); + _detectCache[columnName] = kind; + return kind; +} + /// [detectPiiColumn] for every column. List detectPiiColumns(List columns) => [for (final c in columns) detectPiiColumn(c)]; @@ -76,12 +114,34 @@ String maskPiiValue(PiiKind kind, String value) { final visible = local.length <= 2 ? 1 : 2; return '${local.substring(0, visible)}***@$domain'; case PiiKind.phone: - final digits = value.replaceAll(RegExp(r'\D'), ''); - if (digits.length < 8) return kPiiMaskDots; - return '•••••••${digits.substring(digits.length - 2)}'; + if (_countDigits(value, 8) < 8) return kPiiMaskDots; + return '•••••••${_lastDigits(value, 2)}'; case PiiKind.card: - final digits = value.replaceAll(RegExp(r'\D'), ''); - if (digits.length < 12) return kPiiMaskDots; - return '•••• •••• •••• ${digits.substring(digits.length - 4)}'; + if (_countDigits(value, 12) < 12) return kPiiMaskDots; + return '•••• •••• •••• ${_lastDigits(value, 4)}'; + } +} + +/// Number of ASCII digits in [s], counting no further than [limit]. +int _countDigits(String s, int limit) { + var n = 0; + for (var i = 0; i < s.length; i++) { + if (_isDigit(s.codeUnitAt(i))) { + n++; + if (n >= limit) return n; + } + } + return n; +} + +/// The last [take] ASCII digits of [s], in order. [s] must hold at least +/// [take] digits. +String _lastDigits(String s, int take) { + final out = List.filled(take, 0x30); + var k = take; + for (var i = s.length - 1; i >= 0 && k > 0; i--) { + final c = s.codeUnitAt(i); + if (_isDigit(c)) out[--k] = c; } + return String.fromCharCodes(out); } diff --git a/test/core/security/pii_masker_scanner_test.dart b/test/core/security/pii_masker_scanner_test.dart new file mode 100644 index 00000000..d1fa4c69 --- /dev/null +++ b/test/core/security/pii_masker_scanner_test.dart @@ -0,0 +1,147 @@ +import 'dart:math'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:querya_desktop/core/security/pii_masker.dart'; + +/// `detectPiiColumn` and `maskPiiValue` as they were before #1540 / #1516 +/// (regex, replaceAll, split / join): the reference the scanners must match. +List _refTokens(String name) { + final spaced = name + .replaceAllMapped( + RegExp(r'([a-z0-9])([A-Z])'), + (m) => '${m[1]} ${m[2]}', + ) + .replaceAll(RegExp(r'[^A-Za-z0-9]+'), ' '); + return spaced.toLowerCase().split(' ').where((t) => t.isNotEmpty).toList(); +} + +PiiKind? _refDetect(String columnName) { + const secret = { + 'password', 'passwd', 'pwd', 'pass', 'passcode', 'passphrase', 'token', + 'secret', 'apikey', 'hash', 'salt', + }; + const email = {'email', 'mail'}; + const phone = {'phone', 'mobile', 'cell', 'tel', 'telephone', 'msisdn', 'fax'}; + const card = {'card', 'cc', 'pan', 'iban'}; + final tokens = _refTokens(columnName); + if (tokens.isEmpty) return null; + final joined = tokens.join('_'); + if (RegExp(r'(^|_)(api|private|access|secret|auth)_key($|_)') + .hasMatch(joined)) { + return PiiKind.secret; + } + const meta = {'at', 'verified', 'confirmed', 'sent', 'count', 'id'}; + if (meta.contains(tokens.last) && tokens.length > 1) return null; + for (final t in tokens) { + if (secret.contains(t)) return PiiKind.secret; + if (card.contains(t)) return PiiKind.card; + if (email.contains(t)) return PiiKind.email; + if (phone.contains(t)) return PiiKind.phone; + } + return null; +} + +String _refMask(PiiKind kind, String value) { + if (value.isEmpty || value == 'NULL') return value; + switch (kind) { + case PiiKind.secret: + return kPiiMaskDots; + case PiiKind.email: + final at = value.indexOf('@'); + if (at <= 0 || at == value.length - 1) return kPiiMaskDots; + final local = value.substring(0, at); + final domain = value.substring(at + 1); + final visible = local.length <= 2 ? 1 : 2; + return '${local.substring(0, visible)}***@$domain'; + case PiiKind.phone: + final digits = value.replaceAll(RegExp(r'\D'), ''); + if (digits.length < 8) return kPiiMaskDots; + return '•••••••${digits.substring(digits.length - 2)}'; + case PiiKind.card: + final digits = value.replaceAll(RegExp(r'\D'), ''); + if (digits.length < 12) return kPiiMaskDots; + return '•••• •••• •••• ${digits.substring(digits.length - 4)}'; + } +} + +void main() { + group('PII column detection scanner (#1540)', () { + test('agrees with the previous implementation on listed names', () { + const names = [ + '', ' ', '_', 'id', 'name', 'password', 'PASSWORD_HASH', 'auth_token', + 'api_key', 'apiKey', 'API_KEY', 'APIKey', 'api2Key', 'private_key', + 'accessKey', 'secret_key_id', 'my_api_key_2', 'monkey', 'key', + 'clientSecret', 'email', 'userEmail', 'contact_email', 'EMAIL', + 'email_verified_at', 'phone', 'mobile_number', 'phoneNumber2', + 'card_number', 'credit_card', 'cardId', 'card_id', 'CreditCardNumber', + 'hotel', 'compass', 'cardinality', 'telegram_handle', 'created_at', + 'user.email', 'users.email', '"email"', 'e-mail', 'Email Address', + 'ÉMAIL', 'телефон', 'tel', 'msisdn_2', 'iban', 'IBAN_code', 'pan', + ]; + for (final n in names) { + expect(detectPiiColumn(n), _refDetect(n), reason: '"$n"'); + // The second call is served from the cache. + expect(detectPiiColumn(n), _refDetect(n), reason: 'cached "$n"'); + } + }); + + test('agrees with the previous implementation on random names', () { + const parts = [ + 'api', 'key', 'Key', 'KEY', 'user', 'User', 'email', 'Email', 'phone', + 'card', 'Card', 'id', 'at', 'count', 'secret', 'token', 'x', 'A', '1', + '_', '-', '.', ' ', 'é', + ]; + final random = Random(1540); + for (var n = 0; n < 4000; n++) { + final name = [ + for (var k = 1 + random.nextInt(5); k > 0; k--) + parts[random.nextInt(parts.length)], + ].join(); + expect(detectPiiColumn(name), _refDetect(name), reason: '"$name"'); + } + }); + + test('more distinct names than the cache holds still answer correctly', () { + for (var i = 0; i < 2500; i++) { + expect(detectPiiColumn('col_$i'), isNull); + } + expect(detectPiiColumn('user_email'), PiiKind.email); + expect(detectPiiColumn('user_email'), PiiKind.email); + }); + }); + + group('PII value masking scanner (#1516)', () { + test('agrees with the previous implementation', () { + const values = [ + '', 'NULL', 'null', 'abc', '1234567', '12345678', '+1 (555) 123-4567', + '555.123.4567 ext 89', '4111 1111 1111 1111', '4111-1111-1111-111', + '41111111111', '411111111111', 'DE89 3704 0044 0532 0130 00', + '٠١٢٣٤٥٦٧٨٩٠١٢', '１２３４５６７８９０１２', 'a@b.c', 'ab@x.org', + 'alice@example.com', '@x', 'x@', + ]; + for (final kind in PiiKind.values) { + for (final v in values) { + expect(maskPiiValue(kind, v), _refMask(kind, v), + reason: '$kind "$v"'); + } + } + }); + + test('agrees with the previous implementation on random digit soup', () { + const pieces = [ + '0', '1', '7', '9', ' ', '-', '(', ')', '+', '.', 'x', 'é', '٣', + ]; + final random = Random(1516); + for (var n = 0; n < 3000; n++) { + final v = [ + for (var k = random.nextInt(24); k > 0; k--) + pieces[random.nextInt(pieces.length)], + ].join(); + for (final kind in [PiiKind.phone, PiiKind.card]) { + expect(maskPiiValue(kind, v), _refMask(kind, v), + reason: '$kind "$v"'); + } + } + }); + }); +}