feat(data): test2 split (40 fresh scripts)

README.md

@@ -12,6 +12,7 @@ Most scam-detector demos report no numbers. This gives a tiny, honest yardstick:
 |---|---|
 | `data/dev.jsonl` | 30 scripts (15 scam / 15 benign) for building detectors |
 | `data/test.jsonl` | 30 scripts (15 / 15), frozen, different author, run once |
+| `data/test2.jsonl` | 40 scripts (20 / 20) added later as a fresh held-out split (`u-` ids, author `team-v2`). Written before the detector was re-tuned; see the second-opinion eval report |
 | `schema/script.schema.json` | JSON Schema for one record |
 | `scripts/validate.mjs` | Validates files against the schema and split rules |
 | `scripts/score.mjs` | Scores a predictions file against labels |

data/test2.jsonl

@@ -0,0 +1,40 @@
+{"id":"u-001","split":"test2","label":"scam","category":"gift_card","channel":"sms","text":"My manager is in a meeting and can't talk. Please grab four Apple cards from the store, $250 each, and send me photos of the backs. I'll pay you back tonight.","context":{"actionType":"purchase","amountCents":100000,"payee":"gift cards"},"difficulty":"tricky","author":"team-v2","notes":"boss impersonation, 'Apple cards' without the word gift"}
+{"id":"u-002","split":"test2","label":"scam","category":"family_emergency","channel":"call","text":"Dad, I'm stuck abroad and my cards are blocked. Can you buy some Steam vouchers and text me the codes? Please don't mention it to Mum yet.","difficulty":"tricky","author":"team-v2","notes":"relative speaks, vouchers instead of gift cards"}
+{"id":"u-003","split":"test2","label":"scam","category":"bank_impersonation","channel":"call","text":"This is the card security desk at your bank. We have frozen a payment of $1,899. To reverse it you must move your savings to a safe account that we will give you, and please keep this call between us.","difficulty":"easy","author":"team-v2","notes":"safe account + secrecy"}
+{"id":"u-004","split":"test2","label":"scam","category":"verification_code","channel":"sms","text":"ALERT: unusual sign-in on your savings account. Reply with the 6 digit number we just sent you to keep your account open.","difficulty":"tricky","author":"team-v2","notes":"asks for a code without the word read/tell"}
+{"id":"u-005","split":"test2","label":"scam","category":"government_impersonation","channel":"call","text":"Final warning from the tax office. You owe $3,400 in back taxes. A warrant will be issued unless you settle today by buying prepaid vouchers.","difficulty":"easy","author":"team-v2","notes":"tax authority + vouchers"}
+{"id":"u-006","split":"test2","label":"scam","category":"government_impersonation","channel":"email","text":"Your pension payment is on hold due to an identity check. Pay a $90 release charge at a bitcoin machine in your nearest store to receive it.","context":{"actionType":"payment","amountCents":9000,"payee":"pension office"},"difficulty":"tricky","author":"team-v2","notes":"crypto machine"}
+{"id":"u-007","split":"test2","label":"scam","category":"tech_support_remote","channel":"call","text":"Your computer is sending us error reports. Please open your browser and type the address I give you so I can take control of your screen and fix the virus.","difficulty":"tricky","author":"team-v2","notes":"remote control without a product name"}
+{"id":"u-008","split":"test2","label":"scam","category":"tech_support_remote","channel":"call","text":"Microsoft has detected a virus on your laptop. We need you to pay $299 for a lifetime protection plan, by card, right now over the phone.","context":{"actionType":"payment","amountCents":29900,"payee":"Microsoft"},"difficulty":"easy","author":"team-v2","notes":"fake tech support fee"}
+{"id":"u-009","split":"test2","label":"scam","category":"family_emergency","channel":"call","text":"Gran, it's me. I was in a crash, I'm okay but the other driver wants $1,500 cash right away or he calls the police. Please don't tell Mum, she will panic.","context":{"actionType":"payment","amountCents":150000,"payee":"driver"},"difficulty":"easy","author":"team-v2","notes":"speaker is the relative"}
+{"id":"u-010","split":"test2","label":"scam","category":"family_emergency","channel":"sms","text":"Hi Nana, I lost my phone so this is a new number. I need you to cover a $780 bill for me today, I'll explain later.","context":{"actionType":"transfer","amountCents":78000,"payee":"new number"},"difficulty":"tricky","author":"team-v2","notes":"new number, no emergency words"}
+{"id":"u-011","split":"test2","label":"scam","category":"prize_lottery","channel":"sms","text":"Congratulations! You have been selected to receive a $50,000 cash award. To release the funds, send $200 for the courier insurance.","difficulty":"easy","author":"team-v2","notes":"advance fee"}
+{"id":"u-012","split":"test2","label":"scam","category":"prize_lottery","channel":"email","text":"Your email address won our monthly draw. Reply with your bank details and a $35 handling fee to collect your reward.","difficulty":"tricky","author":"team-v2","notes":"'draw' and 'reward' instead of lottery/prize"}
+{"id":"u-013","split":"test2","label":"scam","category":"verification_code","channel":"sms","text":"Hi, I'm selling a sofa on the marketplace and I accidentally sent a code to your phone. Can you tell me what it says? It is just a mistake.","difficulty":"tricky","author":"team-v2","notes":"marketplace code-forwarding scam"}
+{"id":"u-014","split":"test2","label":"scam","category":"delivery_fee","channel":"sms","text":"Your package is held at the depot. Pay a $3.50 customs charge today or it will be sent back.","context":{"actionType":"payment","amountCents":350,"payee":"depot"},"difficulty":"easy","author":"team-v2","notes":"small delivery fee"}
+{"id":"u-015","split":"test2","label":"scam","category":"wire_crypto","channel":"email","text":"Our trading desk guarantees you will double your money in a week. Move $5,000 into the account we set up and we do the rest.","context":{"actionType":"transfer","amountCents":500000,"payee":"trading desk"},"difficulty":"tricky","author":"team-v2","notes":"investment promise without crypto words"}
+{"id":"u-016","split":"test2","label":"scam","category":"romance_investment","channel":"sms","text":"My love, I have been stuck at the port for days. The customs agent wants $2,000 before they release my equipment. You are the only one I trust.","context":{"actionType":"transfer","amountCents":200000,"payee":"customs agent"},"difficulty":"tricky","author":"team-v2","notes":"romance, no urgency word"}
+{"id":"u-017","split":"test2","label":"scam","category":"wire_crypto","channel":"email","text":"Your landlord has changed his bank details. Please transfer this month's rent to the new account number I have sent and ignore the old one.","context":{"actionType":"transfer","amountCents":140000,"payee":"landlord new account"},"difficulty":"tricky","author":"team-v2","notes":"changed payee details"}
+{"id":"u-018","split":"test2","label":"scam","category":"government_impersonation","channel":"call","text":"This is your electric company. Your power will be cut in thirty minutes unless you pay $410 now with a prepaid card.","context":{"actionType":"payment","amountCents":41000,"payee":"electric company"},"difficulty":"easy","author":"team-v2","notes":"utility shut-off threat"}
+{"id":"u-019","split":"test2","label":"scam","category":"wire_crypto","channel":"email","text":"Remote assistant job, $900 a week. We will send you a cheque; deposit it and forward $1,200 to our supplier by wire.","context":{"actionType":"transfer","amountCents":120000,"payee":"supplier"},"difficulty":"tricky","author":"team-v2","notes":"fake cheque job"}
+{"id":"u-020","split":"test2","label":"scam","category":"family_emergency","channel":"call","text":"Grandpa, it's your grandson. I'm at the police station and need $2,500 for bail. The lawyer will call you. Please don't tell anyone.","context":{"actionType":"payment","amountCents":250000,"payee":"lawyer"},"difficulty":"easy","author":"team-v2","notes":"classic bail"}
+{"id":"u-021","split":"test2","label":"benign","category":"bank_alert_genuine","channel":"sms","text":"Alert from your bank: a purchase of $62.10 at a grocery store was declined. If this was you, no action is needed. We will never ask for your PIN or full card number.","difficulty":"tricky","author":"team-v2","notes":"genuine alert with a warning sentence"}
+{"id":"u-022","split":"test2","label":"benign","category":"bank_alert_genuine","channel":"email","text":"Your bank will never call you to ask for your password. If you get such a call, hang up and use the number on the back of your card.","difficulty":"tricky","author":"team-v2","notes":"pure warning"}
+{"id":"u-023","split":"test2","label":"benign","category":"bank_alert_genuine","channel":"email","text":"Your monthly statement is ready. Sign in to the app to view it.","difficulty":"easy","author":"team-v2","notes":"plain notice"}
+{"id":"u-024","split":"test2","label":"benign","category":"pharmacy","channel":"sms","text":"Your refill of metformin is ready for pickup. Your copay is $8. We are open until 8 pm.","difficulty":"easy","author":"team-v2","notes":"refill"}
+{"id":"u-025","split":"test2","label":"benign","category":"pharmacy","channel":"call","text":"Please pick up your prescription today, otherwise it goes back into stock in seven days.","difficulty":"tricky","author":"team-v2","notes":"today + deadline"}
+{"id":"u-026","split":"test2","label":"benign","category":"utility_bill","channel":"email","text":"Your gas bill of $74.20 is due on the 15th. You can pay by direct debit, online or at the post office.","context":{"actionType":"payment","amountCents":7420,"payee":"City Gas","trusted":true},"difficulty":"tricky","author":"team-v2","notes":"trusted utility"}
+{"id":"u-027","split":"test2","label":"benign","category":"utility_bill","channel":"sms","text":"Heads up: water works on your street tomorrow between 9 and noon. No action is needed.","difficulty":"easy","author":"team-v2","notes":"notice"}
+{"id":"u-028","split":"test2","label":"benign","category":"delivery_genuine","channel":"sms","text":"Your parcel is out for delivery today. If nobody is home we will leave a card with a pick-up code at the depot.","difficulty":"tricky","author":"team-v2","notes":"today + code + depot"}
+{"id":"u-029","split":"test2","label":"benign","category":"delivery_genuine","channel":"email","text":"Your order has shipped and should arrive by Friday.","difficulty":"easy","author":"team-v2","notes":"plain shipping"}
+{"id":"u-030","split":"test2","label":"benign","category":"family_chat","channel":"voice","text":"Can you read me the recipe for the lemon cake on the back of the flour packet?","difficulty":"tricky","author":"team-v2","notes":"read me ... back"}
+{"id":"u-031","split":"test2","label":"benign","category":"family_chat","channel":"voice","text":"Tell Mum I will send her forty dollars for the groceries tonight.","context":{"actionType":"transfer","amountCents":4000,"payee":"Mum","trusted":true},"difficulty":"tricky","author":"team-v2","notes":"trusted family payee"}
+{"id":"u-032","split":"test2","label":"benign","category":"family_chat","channel":"sms","text":"Don't tell Dad about the surprise party, it's a secret till Saturday!","difficulty":"tricky","author":"team-v2","notes":"innocent secrecy"}
+{"id":"u-033","split":"test2","label":"benign","category":"family_chat","channel":"sms","text":"I'm at the hospital with Grandpa. His knee surgery went well and he is resting now.","difficulty":"tricky","author":"team-v2","notes":"hospital news, nobody asks for money"}
+{"id":"u-034","split":"test2","label":"benign","category":"subscription_receipt","channel":"email","text":"Thanks for renewing your subscription. $11.99 was charged today to your card ending 4411. You can manage it any time in settings.","difficulty":"tricky","author":"team-v2","notes":"receipt with an amount"}
+{"id":"u-035","split":"test2","label":"benign","category":"subscription_receipt","channel":"email","text":"Your free trial ends in three days. Cancel any time from your account.","difficulty":"easy","author":"team-v2","notes":"trial notice"}
+{"id":"u-036","split":"test2","label":"benign","category":"appointment_reminder","channel":"sms","text":"This is the clinic. Please bring your insurance card and a photo ID to your appointment tomorrow.","difficulty":"tricky","author":"team-v2","notes":"photo ID"}
+{"id":"u-037","split":"test2","label":"benign","category":"appointment_reminder","channel":"sms","text":"Reminder: dentist on Thursday at 2:30 pm. Reply C to confirm.","difficulty":"easy","author":"team-v2","notes":"reminder"}
+{"id":"u-038","split":"test2","label":"benign","category":"bank_alert_genuine","channel":"sms","text":"We noticed a sign-in from a new device. If this was you, ignore this message. If not, call the number printed on your card.","difficulty":"tricky","author":"team-v2","notes":"genuine alert, unsolicited wording"}
+{"id":"u-039","split":"test2","label":"benign","category":"family_chat","channel":"voice","text":"Can you transfer $200 to Maya for her school trip? Here are the details she gave me.","context":{"actionType":"transfer","amountCents":20000,"payee":"Maya","trusted":true},"difficulty":"tricky","author":"team-v2","notes":"trusted payee transfer"}
+{"id":"u-040","split":"test2","label":"benign","category":"delivery_genuine","channel":"email","text":"Your gift card order from the store will arrive in two days. Gift cards are sent in a plain envelope.","difficulty":"tricky","author":"team-v2","notes":"mentions gift cards, asks for nothing"}

schema/script.schema.json

@@ -6,8 +6,8 @@
   "additionalProperties": false,
   "required": ["id", "split", "label", "category", "channel", "text", "difficulty", "author"],
   "properties": {
-    "id": { "type": "string", "pattern": "^[dt]-\\d{3}$" },
-    "split": { "enum": ["dev", "test"] },
+    "id": { "type": "string", "pattern": "^[dtu]-\\d{3}$" },
+    "split": { "enum": ["dev", "test", "test2"] },
     "label": { "enum": ["scam", "benign"] },
     "category": {
       "enum": [

scripts/validate.mjs

@@ -11,6 +11,9 @@ const root = join(dirname(fileURLToPath(import.meta.url)), '..');
 const PRIVATE_DATA = /https?:\/\/|www\./i;
 const PHONE = /\b\d{3}[-.\s]\d{3}[-.\s]\d{4}\b/;
 const NEAR_DUPLICATE = 0.8;
+const PREFIX = { dev: 'd-', test: 't-', test2: 'u-' };
+/** Records per label in each split. */
+const PER_LABEL = { dev: 15, test: 15, test2: 20 };
 
 /**
  * Validate one or more splits. Returns a list of human-readable problems (empty when everything is fine).
@@ -34,7 +37,7 @@ export function validate(files, { schemaPath = join(root, 'schema', 'script.sche
         continue;
       }
       if (r.split !== split) problems.push(`${where}: split "${r.split}" does not match file "${split}"`);
-      if (!r.id.startsWith(split === 'dev' ? 'd-' : 't-')) problems.push(`${where}: id prefix does not match the split`);
+      if (!r.id.startsWith(PREFIX[split])) problems.push(`${where}: id prefix does not match the split`);
       if (ids.has(r.id)) problems.push(`${where}: duplicate id`);
       ids.add(r.id);
       if (PRIVATE_DATA.test(r.text) || PHONE.test(r.text)) problems.push(`${where}: contains a URL or phone number (use invented, generic text)`);
@@ -44,18 +47,23 @@ export function validate(files, { schemaPath = join(root, 'schema', 'script.sche
 
     const scams = rows.filter((r) => r.label === 'scam');
     const benign = rows.filter((r) => r.label === 'benign');
-    if (scams.length !== 15) problems.push(`${split}: expected 15 scam records, found ${scams.length}`);
-    if (benign.length !== 15) problems.push(`${split}: expected 15 benign records, found ${benign.length}`);
+    const want = PER_LABEL[split] ?? 15;
+    if (scams.length !== want) problems.push(`${split}: expected ${want} scam records, found ${scams.length}`);
+    if (benign.length !== want) problems.push(`${split}: expected ${want} benign records, found ${benign.length}`);
     const tricky = benign.filter((r) => r.difficulty === 'tricky').length;
     if (tricky < Math.ceil(benign.length * 0.4)) problems.push(`${split}: at least 40% of benign records must be tricky (found ${tricky}/${benign.length})`);
     const cats = new Set(scams.map((r) => r.category));
     if (cats.size < 6) problems.push(`${split}: at least 6 distinct scam categories required (found ${cats.size})`);
   }
 
-  if (bySplit.dev && bySplit.test) {
-    const devAuthors = new Set(bySplit.dev.map((r) => r.author));
-    const testAuthors = new Set(bySplit.test.map((r) => r.author));
-    for (const a of devAuthors) if (testAuthors.has(a)) problems.push(`author label "${a}" appears in both dev and test`);
+  const splits = Object.keys(bySplit);
+  for (let i = 0; i < splits.length; i++) {
+    for (let j = i + 1; j < splits.length; j++) {
+      const a = new Set(bySplit[splits[i]].map((r) => r.author));
+      for (const x of new Set(bySplit[splits[j]].map((r) => r.author))) {
+        if (a.has(x)) problems.push(`author label "${x}" appears in both ${splits[i]} and ${splits[j]}`);
+      }
+    }
   }
 
   const all = Object.entries(bySplit).flatMap(([split, rows]) => rows.map((r) => ({ split, r, sh: shingles(r.text ?? '') })));
@@ -71,7 +79,7 @@ export function validate(files, { schemaPath = join(root, 'schema', 'script.sche
 
 if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
   const files = {};
-  for (const split of ['dev', 'test']) {
+  for (const split of ['dev', 'test', 'test2']) {
     const p = join(root, 'data', `${split}.jsonl`);
     if (existsSync(p)) files[split] = p;
   }

test/bench.test.mjs

@@ -131,3 +131,11 @@ test('similarity helper: identical texts match, different texts do not', () => {
   assert.equal(jaccard(shingles('buy gift cards today now'), shingles('buy gift cards today now')), 1);
   assert.ok(jaccard(shingles('buy gift cards today now'), shingles('your appointment is on thursday')) < 0.1);
 });
+
+test('test2 has 20 scam and 20 benign records with hard cases, and validates', () => {
+  const rows = readJsonl(join(root, 'data', 'test2.jsonl'));
+  assert.equal(rows.length, 40);
+  assert.equal(rows.filter((r) => r.label === 'scam').length, 20);
+  assert.ok(rows.filter((r) => r.label === 'benign' && r.difficulty === 'tricky').length >= 8);
+  assert.deepEqual(validate({ test2: join(root, 'data', 'test2.jsonl') }), []);
+});