Skip to content

perf(security): regex-free PII column detection and value masking (#1540, #1516) #1608

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ZhuchkaTriplesix merged 1 commit into from
Oct 15, 2026
Merged

perf(security): regex-free PII column detection and value masking (#1540, #1516) #1608

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
118 changes: 89 additions & 29 deletions lib/core/security/pii_masker.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,39 +15,65 @@ const _phoneTokens = {
};
const _cardTokens = {'card', 'cc', 'pan', 'iban'};

bool _isLower(int c) => c >= 0x61 && c <= 0x7A;
bool _isUpper(int c) => c >= 0x41 && c <= 0x5A;
bool _isDigit(int c) => c >= 0x30 && c <= 0x39;

/// Splits a column name into lower-case words: on non-alphanumerics and on
/// camelCase boundaries (`userEmail` -> `user`, `email`).
///
/// One pass over the code units (no regex, no split / join). A boundary is a
/// lower-case letter or digit followed by an upper-case letter.
List<String> _tokens(String name) {
final spaced = name
.replaceAllMapped(
RegExp(r'([a-z0-9])([A-Z])'),
(m) => '${m[1]} ${m[2]}',
)
.replaceAll(RegExp(r'[^A-Za-z0-9]+'), ' ');
return spaced
.toLowerCase()
.split(' ')
.where((t) => t.isNotEmpty)
.toList();
final tokens = <String>[];
var start = -1;
for (var i = 0; i < name.length; i++) {
final c = name.codeUnitAt(i);
final alnum = _isLower(c) || _isUpper(c) || _isDigit(c);
if (!alnum) {
if (start >= 0) {
tokens.add(name.substring(start, i).toLowerCase());
start = -1;
}
continue;
}
if (start >= 0 && _isUpper(c)) {
final prev = name.codeUnitAt(i - 1);
if (_isLower(prev) || _isDigit(prev)) {
tokens.add(name.substring(start, i).toLowerCase());
start = i;
continue;
}
}
if (start < 0) start = i;
}
if (start >= 0) tokens.add(name.substring(start).toLowerCase());
return tokens;
}

/// What [columnName] probably holds, or null when it does not look sensitive.
///
/// Matching is per word, so `user_email` and `userEmail` match but `hotel` and
/// `email_verified_at` do not trip the phone / email rules by accident.
PiiKind? detectPiiColumn(String columnName) {
const _keyPrefixTokens = {'api', 'private', 'access', 'secret', 'auth'};

/// Timestamps / flags *about* a sensitive field are not the value itself.
const _metaTokens = {'at', 'verified', 'confirmed', 'sent', 'count', 'id'};

/// Most distinct column names remembered by [detectPiiColumn].
const int _detectCacheLimit = 1024;

/// Column names repeat (`id`, `name`, `created_at`...) from query to query, so
/// the answer per name is kept. Per isolate; cleared when it gets large.
final Map<String, PiiKind?> _detectCache = <String, PiiKind?>{};

PiiKind? _detectUncached(String columnName) {
final tokens = _tokens(columnName);
if (tokens.isEmpty) return null;
// `api_key`, `private_key`, `access_key` are split into two words.
final joined = tokens.join('_');
if (RegExp(r'(^|_)(api|private|access|secret|auth)_key($|_)')
.hasMatch(joined)) {
return PiiKind.secret;
for (var i = 0; i + 1 < tokens.length; i++) {
if (tokens[i + 1] == 'key' && _keyPrefixTokens.contains(tokens[i])) {
return PiiKind.secret;
}
}
// Timestamps / flags *about* a sensitive field are not the value itself.
const metaTokens = {'at', 'verified', 'confirmed', 'sent', 'count', 'id'};
final lastToken = tokens.last;
if (metaTokens.contains(lastToken) && tokens.length > 1) return null;
if (_metaTokens.contains(lastToken) && tokens.length > 1) return null;
for (final t in tokens) {
if (_secretTokens.contains(t)) return PiiKind.secret;
if (_cardTokens.contains(t)) return PiiKind.card;
Expand All @@ -57,6 +83,18 @@ PiiKind? detectPiiColumn(String columnName) {
return null;
}

/// What [columnName] probably holds, or null when it does not look sensitive.
///
/// Matching is per word, so `user_email` and `userEmail` match but `hotel` and
/// `email_verified_at` do not trip the phone / email rules by accident.
PiiKind? detectPiiColumn(String columnName) {
if (_detectCache.containsKey(columnName)) return _detectCache[columnName];
final kind = _detectUncached(columnName);
if (_detectCache.length >= _detectCacheLimit) _detectCache.clear();
_detectCache[columnName] = kind;
return kind;
}

/// [detectPiiColumn] for every column.
List<PiiKind?> detectPiiColumns(List<String> columns) =>
[for (final c in columns) detectPiiColumn(c)];
Expand All @@ -76,12 +114,34 @@ String maskPiiValue(PiiKind kind, String value) {
final visible = local.length <= 2 ? 1 : 2;
return '${local.substring(0, visible)}***@$domain';
case PiiKind.phone:
final digits = value.replaceAll(RegExp(r'\D'), '');
if (digits.length < 8) return kPiiMaskDots;
return '•••••••${digits.substring(digits.length - 2)}';
if (_countDigits(value, 8) < 8) return kPiiMaskDots;
return '•••••••${_lastDigits(value, 2)}';
case PiiKind.card:
final digits = value.replaceAll(RegExp(r'\D'), '');
if (digits.length < 12) return kPiiMaskDots;
return '•••• •••• •••• ${digits.substring(digits.length - 4)}';
if (_countDigits(value, 12) < 12) return kPiiMaskDots;
return '•••• •••• •••• ${_lastDigits(value, 4)}';
}
}

/// Number of ASCII digits in [s], counting no further than [limit].
int _countDigits(String s, int limit) {
var n = 0;
for (var i = 0; i < s.length; i++) {
if (_isDigit(s.codeUnitAt(i))) {
n++;
if (n >= limit) return n;
}
}
return n;
}

/// The last [take] ASCII digits of [s], in order. [s] must hold at least
/// [take] digits.
String _lastDigits(String s, int take) {
final out = List<int>.filled(take, 0x30);
var k = take;
for (var i = s.length - 1; i >= 0 && k > 0; i--) {
final c = s.codeUnitAt(i);
if (_isDigit(c)) out[--k] = c;
}
return String.fromCharCodes(out);
}
147 changes: 147 additions & 0 deletions test/core/security/pii_masker_scanner_test.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,147 @@
import 'dart:math';

import 'package:flutter_test/flutter_test.dart';
import 'package:querya_desktop/core/security/pii_masker.dart';

/// `detectPiiColumn` and `maskPiiValue` as they were before #1540 / #1516
/// (regex, replaceAll, split / join): the reference the scanners must match.
List<String> _refTokens(String name) {
final spaced = name
.replaceAllMapped(
RegExp(r'([a-z0-9])([A-Z])'),
(m) => '${m[1]} ${m[2]}',
)
.replaceAll(RegExp(r'[^A-Za-z0-9]+'), ' ');
return spaced.toLowerCase().split(' ').where((t) => t.isNotEmpty).toList();
}

PiiKind? _refDetect(String columnName) {
const secret = {
'password', 'passwd', 'pwd', 'pass', 'passcode', 'passphrase', 'token',
'secret', 'apikey', 'hash', 'salt',
};
const email = {'email', 'mail'};
const phone = {'phone', 'mobile', 'cell', 'tel', 'telephone', 'msisdn', 'fax'};
const card = {'card', 'cc', 'pan', 'iban'};
final tokens = _refTokens(columnName);
if (tokens.isEmpty) return null;
final joined = tokens.join('_');
if (RegExp(r'(^|_)(api|private|access|secret|auth)_key($|_)')
.hasMatch(joined)) {
return PiiKind.secret;
}
const meta = {'at', 'verified', 'confirmed', 'sent', 'count', 'id'};
if (meta.contains(tokens.last) && tokens.length > 1) return null;
for (final t in tokens) {
if (secret.contains(t)) return PiiKind.secret;
if (card.contains(t)) return PiiKind.card;
if (email.contains(t)) return PiiKind.email;
if (phone.contains(t)) return PiiKind.phone;
}
return null;
}

String _refMask(PiiKind kind, String value) {
if (value.isEmpty || value == 'NULL') return value;
switch (kind) {
case PiiKind.secret:
return kPiiMaskDots;
case PiiKind.email:
final at = value.indexOf('@');
if (at <= 0 || at == value.length - 1) return kPiiMaskDots;
final local = value.substring(0, at);
final domain = value.substring(at + 1);
final visible = local.length <= 2 ? 1 : 2;
return '${local.substring(0, visible)}***@$domain';
case PiiKind.phone:
final digits = value.replaceAll(RegExp(r'\D'), '');
if (digits.length < 8) return kPiiMaskDots;
return '•••••••${digits.substring(digits.length - 2)}';
case PiiKind.card:
final digits = value.replaceAll(RegExp(r'\D'), '');
if (digits.length < 12) return kPiiMaskDots;
return '•••• •••• •••• ${digits.substring(digits.length - 4)}';
}
}

void main() {
group('PII column detection scanner (#1540)', () {
test('agrees with the previous implementation on listed names', () {
const names = <String>[
'', ' ', '_', 'id', 'name', 'password', 'PASSWORD_HASH', 'auth_token',
'api_key', 'apiKey', 'API_KEY', 'APIKey', 'api2Key', 'private_key',
'accessKey', 'secret_key_id', 'my_api_key_2', 'monkey', 'key',
'clientSecret', 'email', 'userEmail', 'contact_email', 'EMAIL',
'email_verified_at', 'phone', 'mobile_number', 'phoneNumber2',
'card_number', 'credit_card', 'cardId', 'card_id', 'CreditCardNumber',
'hotel', 'compass', 'cardinality', 'telegram_handle', 'created_at',
'user.email', 'users.email', '"email"', 'e-mail', 'Email Address',
'ÉMAIL', 'телефон', 'tel', 'msisdn_2', 'iban', 'IBAN_code', 'pan',
];
for (final n in names) {
expect(detectPiiColumn(n), _refDetect(n), reason: '"$n"');
// The second call is served from the cache.
expect(detectPiiColumn(n), _refDetect(n), reason: 'cached "$n"');
}
});

test('agrees with the previous implementation on random names', () {
const parts = <String>[
'api', 'key', 'Key', 'KEY', 'user', 'User', 'email', 'Email', 'phone',
'card', 'Card', 'id', 'at', 'count', 'secret', 'token', 'x', 'A', '1',
'_', '-', '.', ' ', 'é',
];
final random = Random(1540);
for (var n = 0; n < 4000; n++) {
final name = [
for (var k = 1 + random.nextInt(5); k > 0; k--)
parts[random.nextInt(parts.length)],
].join();
expect(detectPiiColumn(name), _refDetect(name), reason: '"$name"');
}
});

test('more distinct names than the cache holds still answer correctly', () {
for (var i = 0; i < 2500; i++) {
expect(detectPiiColumn('col_$i'), isNull);
}
expect(detectPiiColumn('user_email'), PiiKind.email);
expect(detectPiiColumn('user_email'), PiiKind.email);
});
});

group('PII value masking scanner (#1516)', () {
test('agrees with the previous implementation', () {
const values = <String>[
'', 'NULL', 'null', 'abc', '1234567', '12345678', '+1 (555) 123-4567',
'555.123.4567 ext 89', '4111 1111 1111 1111', '4111-1111-1111-111',
'41111111111', '411111111111', 'DE89 3704 0044 0532 0130 00',
'٠١٢٣٤٥٦٧٨٩٠١٢', '１２３４５６７８９０１２', 'a@b.c', 'ab@x.org',
'alice@example.com', '@x', 'x@',
];
for (final kind in PiiKind.values) {
for (final v in values) {
expect(maskPiiValue(kind, v), _refMask(kind, v),
reason: '$kind "$v"');
}
}
});

test('agrees with the previous implementation on random digit soup', () {
const pieces = <String>[
'0', '1', '7', '9', ' ', '-', '(', ')', '+', '.', 'x', 'é', '٣',
];
final random = Random(1516);
for (var n = 0; n < 3000; n++) {
final v = [
for (var k = random.nextInt(24); k > 0; k--)
pieces[random.nextInt(pieces.length)],
].join();
for (final kind in [PiiKind.phone, PiiKind.card]) {
expect(maskPiiValue(kind, v), _refMask(kind, v),
reason: '$kind "$v"');
}
}
});
});
}
Loading