diff --git a/README.md b/README.md index 314e07c..2575c66 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ Most scam-detector demos report no numbers. This gives a tiny, honest yardstick: |---|---| | `data/dev.jsonl` | 30 scripts (15 scam / 15 benign) for building detectors | | `data/test.jsonl` | 30 scripts (15 / 15), frozen, different author, run once | +| `data/test2.jsonl` | 40 scripts (20 / 20) added later as a fresh held-out split (`u-` ids, author `team-v2`). Written before the detector was re-tuned; see the second-opinion eval report | | `schema/script.schema.json` | JSON Schema for one record | | `scripts/validate.mjs` | Validates files against the schema and split rules | | `scripts/score.mjs` | Scores a predictions file against labels | diff --git a/data/test2.jsonl b/data/test2.jsonl new file mode 100644 index 0000000..5303d19 --- /dev/null +++ b/data/test2.jsonl @@ -0,0 +1,40 @@ +{"id":"u-001","split":"test2","label":"scam","category":"gift_card","channel":"sms","text":"My manager is in a meeting and can't talk. Please grab four Apple cards from the store, $250 each, and send me photos of the backs. I'll pay you back tonight.","context":{"actionType":"purchase","amountCents":100000,"payee":"gift cards"},"difficulty":"tricky","author":"team-v2","notes":"boss impersonation, 'Apple cards' without the word gift"} +{"id":"u-002","split":"test2","label":"scam","category":"family_emergency","channel":"call","text":"Dad, I'm stuck abroad and my cards are blocked. Can you buy some Steam vouchers and text me the codes? Please don't mention it to Mum yet.","difficulty":"tricky","author":"team-v2","notes":"relative speaks, vouchers instead of gift cards"} +{"id":"u-003","split":"test2","label":"scam","category":"bank_impersonation","channel":"call","text":"This is the card security desk at your bank. We have frozen a payment of $1,899. To reverse it you must move your savings to a safe account that we will give you, and please keep this call between us.","difficulty":"easy","author":"team-v2","notes":"safe account + secrecy"} +{"id":"u-004","split":"test2","label":"scam","category":"verification_code","channel":"sms","text":"ALERT: unusual sign-in on your savings account. Reply with the 6 digit number we just sent you to keep your account open.","difficulty":"tricky","author":"team-v2","notes":"asks for a code without the word read/tell"} +{"id":"u-005","split":"test2","label":"scam","category":"government_impersonation","channel":"call","text":"Final warning from the tax office. You owe $3,400 in back taxes. A warrant will be issued unless you settle today by buying prepaid vouchers.","difficulty":"easy","author":"team-v2","notes":"tax authority + vouchers"} +{"id":"u-006","split":"test2","label":"scam","category":"government_impersonation","channel":"email","text":"Your pension payment is on hold due to an identity check. Pay a $90 release charge at a bitcoin machine in your nearest store to receive it.","context":{"actionType":"payment","amountCents":9000,"payee":"pension office"},"difficulty":"tricky","author":"team-v2","notes":"crypto machine"} +{"id":"u-007","split":"test2","label":"scam","category":"tech_support_remote","channel":"call","text":"Your computer is sending us error reports. Please open your browser and type the address I give you so I can take control of your screen and fix the virus.","difficulty":"tricky","author":"team-v2","notes":"remote control without a product name"} +{"id":"u-008","split":"test2","label":"scam","category":"tech_support_remote","channel":"call","text":"Microsoft has detected a virus on your laptop. We need you to pay $299 for a lifetime protection plan, by card, right now over the phone.","context":{"actionType":"payment","amountCents":29900,"payee":"Microsoft"},"difficulty":"easy","author":"team-v2","notes":"fake tech support fee"} +{"id":"u-009","split":"test2","label":"scam","category":"family_emergency","channel":"call","text":"Gran, it's me. I was in a crash, I'm okay but the other driver wants $1,500 cash right away or he calls the police. Please don't tell Mum, she will panic.","context":{"actionType":"payment","amountCents":150000,"payee":"driver"},"difficulty":"easy","author":"team-v2","notes":"speaker is the relative"} +{"id":"u-010","split":"test2","label":"scam","category":"family_emergency","channel":"sms","text":"Hi Nana, I lost my phone so this is a new number. I need you to cover a $780 bill for me today, I'll explain later.","context":{"actionType":"transfer","amountCents":78000,"payee":"new number"},"difficulty":"tricky","author":"team-v2","notes":"new number, no emergency words"} +{"id":"u-011","split":"test2","label":"scam","category":"prize_lottery","channel":"sms","text":"Congratulations! You have been selected to receive a $50,000 cash award. To release the funds, send $200 for the courier insurance.","difficulty":"easy","author":"team-v2","notes":"advance fee"} +{"id":"u-012","split":"test2","label":"scam","category":"prize_lottery","channel":"email","text":"Your email address won our monthly draw. Reply with your bank details and a $35 handling fee to collect your reward.","difficulty":"tricky","author":"team-v2","notes":"'draw' and 'reward' instead of lottery/prize"} +{"id":"u-013","split":"test2","label":"scam","category":"verification_code","channel":"sms","text":"Hi, I'm selling a sofa on the marketplace and I accidentally sent a code to your phone. Can you tell me what it says? It is just a mistake.","difficulty":"tricky","author":"team-v2","notes":"marketplace code-forwarding scam"} +{"id":"u-014","split":"test2","label":"scam","category":"delivery_fee","channel":"sms","text":"Your package is held at the depot. Pay a $3.50 customs charge today or it will be sent back.","context":{"actionType":"payment","amountCents":350,"payee":"depot"},"difficulty":"easy","author":"team-v2","notes":"small delivery fee"} +{"id":"u-015","split":"test2","label":"scam","category":"wire_crypto","channel":"email","text":"Our trading desk guarantees you will double your money in a week. Move $5,000 into the account we set up and we do the rest.","context":{"actionType":"transfer","amountCents":500000,"payee":"trading desk"},"difficulty":"tricky","author":"team-v2","notes":"investment promise without crypto words"} +{"id":"u-016","split":"test2","label":"scam","category":"romance_investment","channel":"sms","text":"My love, I have been stuck at the port for days. The customs agent wants $2,000 before they release my equipment. You are the only one I trust.","context":{"actionType":"transfer","amountCents":200000,"payee":"customs agent"},"difficulty":"tricky","author":"team-v2","notes":"romance, no urgency word"} +{"id":"u-017","split":"test2","label":"scam","category":"wire_crypto","channel":"email","text":"Your landlord has changed his bank details. Please transfer this month's rent to the new account number I have sent and ignore the old one.","context":{"actionType":"transfer","amountCents":140000,"payee":"landlord new account"},"difficulty":"tricky","author":"team-v2","notes":"changed payee details"} +{"id":"u-018","split":"test2","label":"scam","category":"government_impersonation","channel":"call","text":"This is your electric company. Your power will be cut in thirty minutes unless you pay $410 now with a prepaid card.","context":{"actionType":"payment","amountCents":41000,"payee":"electric company"},"difficulty":"easy","author":"team-v2","notes":"utility shut-off threat"} +{"id":"u-019","split":"test2","label":"scam","category":"wire_crypto","channel":"email","text":"Remote assistant job, $900 a week. We will send you a cheque; deposit it and forward $1,200 to our supplier by wire.","context":{"actionType":"transfer","amountCents":120000,"payee":"supplier"},"difficulty":"tricky","author":"team-v2","notes":"fake cheque job"} +{"id":"u-020","split":"test2","label":"scam","category":"family_emergency","channel":"call","text":"Grandpa, it's your grandson. I'm at the police station and need $2,500 for bail. The lawyer will call you. Please don't tell anyone.","context":{"actionType":"payment","amountCents":250000,"payee":"lawyer"},"difficulty":"easy","author":"team-v2","notes":"classic bail"} +{"id":"u-021","split":"test2","label":"benign","category":"bank_alert_genuine","channel":"sms","text":"Alert from your bank: a purchase of $62.10 at a grocery store was declined. If this was you, no action is needed. We will never ask for your PIN or full card number.","difficulty":"tricky","author":"team-v2","notes":"genuine alert with a warning sentence"} +{"id":"u-022","split":"test2","label":"benign","category":"bank_alert_genuine","channel":"email","text":"Your bank will never call you to ask for your password. If you get such a call, hang up and use the number on the back of your card.","difficulty":"tricky","author":"team-v2","notes":"pure warning"} +{"id":"u-023","split":"test2","label":"benign","category":"bank_alert_genuine","channel":"email","text":"Your monthly statement is ready. Sign in to the app to view it.","difficulty":"easy","author":"team-v2","notes":"plain notice"} +{"id":"u-024","split":"test2","label":"benign","category":"pharmacy","channel":"sms","text":"Your refill of metformin is ready for pickup. Your copay is $8. We are open until 8 pm.","difficulty":"easy","author":"team-v2","notes":"refill"} +{"id":"u-025","split":"test2","label":"benign","category":"pharmacy","channel":"call","text":"Please pick up your prescription today, otherwise it goes back into stock in seven days.","difficulty":"tricky","author":"team-v2","notes":"today + deadline"} +{"id":"u-026","split":"test2","label":"benign","category":"utility_bill","channel":"email","text":"Your gas bill of $74.20 is due on the 15th. You can pay by direct debit, online or at the post office.","context":{"actionType":"payment","amountCents":7420,"payee":"City Gas","trusted":true},"difficulty":"tricky","author":"team-v2","notes":"trusted utility"} +{"id":"u-027","split":"test2","label":"benign","category":"utility_bill","channel":"sms","text":"Heads up: water works on your street tomorrow between 9 and noon. No action is needed.","difficulty":"easy","author":"team-v2","notes":"notice"} +{"id":"u-028","split":"test2","label":"benign","category":"delivery_genuine","channel":"sms","text":"Your parcel is out for delivery today. If nobody is home we will leave a card with a pick-up code at the depot.","difficulty":"tricky","author":"team-v2","notes":"today + code + depot"} +{"id":"u-029","split":"test2","label":"benign","category":"delivery_genuine","channel":"email","text":"Your order has shipped and should arrive by Friday.","difficulty":"easy","author":"team-v2","notes":"plain shipping"} +{"id":"u-030","split":"test2","label":"benign","category":"family_chat","channel":"voice","text":"Can you read me the recipe for the lemon cake on the back of the flour packet?","difficulty":"tricky","author":"team-v2","notes":"read me ... back"} +{"id":"u-031","split":"test2","label":"benign","category":"family_chat","channel":"voice","text":"Tell Mum I will send her forty dollars for the groceries tonight.","context":{"actionType":"transfer","amountCents":4000,"payee":"Mum","trusted":true},"difficulty":"tricky","author":"team-v2","notes":"trusted family payee"} +{"id":"u-032","split":"test2","label":"benign","category":"family_chat","channel":"sms","text":"Don't tell Dad about the surprise party, it's a secret till Saturday!","difficulty":"tricky","author":"team-v2","notes":"innocent secrecy"} +{"id":"u-033","split":"test2","label":"benign","category":"family_chat","channel":"sms","text":"I'm at the hospital with Grandpa. His knee surgery went well and he is resting now.","difficulty":"tricky","author":"team-v2","notes":"hospital news, nobody asks for money"} +{"id":"u-034","split":"test2","label":"benign","category":"subscription_receipt","channel":"email","text":"Thanks for renewing your subscription. $11.99 was charged today to your card ending 4411. You can manage it any time in settings.","difficulty":"tricky","author":"team-v2","notes":"receipt with an amount"} +{"id":"u-035","split":"test2","label":"benign","category":"subscription_receipt","channel":"email","text":"Your free trial ends in three days. Cancel any time from your account.","difficulty":"easy","author":"team-v2","notes":"trial notice"} +{"id":"u-036","split":"test2","label":"benign","category":"appointment_reminder","channel":"sms","text":"This is the clinic. Please bring your insurance card and a photo ID to your appointment tomorrow.","difficulty":"tricky","author":"team-v2","notes":"photo ID"} +{"id":"u-037","split":"test2","label":"benign","category":"appointment_reminder","channel":"sms","text":"Reminder: dentist on Thursday at 2:30 pm. Reply C to confirm.","difficulty":"easy","author":"team-v2","notes":"reminder"} +{"id":"u-038","split":"test2","label":"benign","category":"bank_alert_genuine","channel":"sms","text":"We noticed a sign-in from a new device. If this was you, ignore this message. If not, call the number printed on your card.","difficulty":"tricky","author":"team-v2","notes":"genuine alert, unsolicited wording"} +{"id":"u-039","split":"test2","label":"benign","category":"family_chat","channel":"voice","text":"Can you transfer $200 to Maya for her school trip? Here are the details she gave me.","context":{"actionType":"transfer","amountCents":20000,"payee":"Maya","trusted":true},"difficulty":"tricky","author":"team-v2","notes":"trusted payee transfer"} +{"id":"u-040","split":"test2","label":"benign","category":"delivery_genuine","channel":"email","text":"Your gift card order from the store will arrive in two days. Gift cards are sent in a plain envelope.","difficulty":"tricky","author":"team-v2","notes":"mentions gift cards, asks for nothing"} diff --git a/schema/script.schema.json b/schema/script.schema.json index 6ca62d2..bd0a094 100644 --- a/schema/script.schema.json +++ b/schema/script.schema.json @@ -6,8 +6,8 @@ "additionalProperties": false, "required": ["id", "split", "label", "category", "channel", "text", "difficulty", "author"], "properties": { - "id": { "type": "string", "pattern": "^[dt]-\\d{3}$" }, - "split": { "enum": ["dev", "test"] }, + "id": { "type": "string", "pattern": "^[dtu]-\\d{3}$" }, + "split": { "enum": ["dev", "test", "test2"] }, "label": { "enum": ["scam", "benign"] }, "category": { "enum": [ diff --git a/scripts/validate.mjs b/scripts/validate.mjs index a6ff8b3..b26309f 100644 --- a/scripts/validate.mjs +++ b/scripts/validate.mjs @@ -11,6 +11,9 @@ const root = join(dirname(fileURLToPath(import.meta.url)), '..'); const PRIVATE_DATA = /https?:\/\/|www\./i; const PHONE = /\b\d{3}[-.\s]\d{3}[-.\s]\d{4}\b/; const NEAR_DUPLICATE = 0.8; +const PREFIX = { dev: 'd-', test: 't-', test2: 'u-' }; +/** Records per label in each split. */ +const PER_LABEL = { dev: 15, test: 15, test2: 20 }; /** * Validate one or more splits. Returns a list of human-readable problems (empty when everything is fine). @@ -34,7 +37,7 @@ export function validate(files, { schemaPath = join(root, 'schema', 'script.sche continue; } if (r.split !== split) problems.push(`${where}: split "${r.split}" does not match file "${split}"`); - if (!r.id.startsWith(split === 'dev' ? 'd-' : 't-')) problems.push(`${where}: id prefix does not match the split`); + if (!r.id.startsWith(PREFIX[split])) problems.push(`${where}: id prefix does not match the split`); if (ids.has(r.id)) problems.push(`${where}: duplicate id`); ids.add(r.id); if (PRIVATE_DATA.test(r.text) || PHONE.test(r.text)) problems.push(`${where}: contains a URL or phone number (use invented, generic text)`); @@ -44,18 +47,23 @@ export function validate(files, { schemaPath = join(root, 'schema', 'script.sche const scams = rows.filter((r) => r.label === 'scam'); const benign = rows.filter((r) => r.label === 'benign'); - if (scams.length !== 15) problems.push(`${split}: expected 15 scam records, found ${scams.length}`); - if (benign.length !== 15) problems.push(`${split}: expected 15 benign records, found ${benign.length}`); + const want = PER_LABEL[split] ?? 15; + if (scams.length !== want) problems.push(`${split}: expected ${want} scam records, found ${scams.length}`); + if (benign.length !== want) problems.push(`${split}: expected ${want} benign records, found ${benign.length}`); const tricky = benign.filter((r) => r.difficulty === 'tricky').length; if (tricky < Math.ceil(benign.length * 0.4)) problems.push(`${split}: at least 40% of benign records must be tricky (found ${tricky}/${benign.length})`); const cats = new Set(scams.map((r) => r.category)); if (cats.size < 6) problems.push(`${split}: at least 6 distinct scam categories required (found ${cats.size})`); } - if (bySplit.dev && bySplit.test) { - const devAuthors = new Set(bySplit.dev.map((r) => r.author)); - const testAuthors = new Set(bySplit.test.map((r) => r.author)); - for (const a of devAuthors) if (testAuthors.has(a)) problems.push(`author label "${a}" appears in both dev and test`); + const splits = Object.keys(bySplit); + for (let i = 0; i < splits.length; i++) { + for (let j = i + 1; j < splits.length; j++) { + const a = new Set(bySplit[splits[i]].map((r) => r.author)); + for (const x of new Set(bySplit[splits[j]].map((r) => r.author))) { + if (a.has(x)) problems.push(`author label "${x}" appears in both ${splits[i]} and ${splits[j]}`); + } + } } const all = Object.entries(bySplit).flatMap(([split, rows]) => rows.map((r) => ({ split, r, sh: shingles(r.text ?? '') }))); @@ -71,7 +79,7 @@ export function validate(files, { schemaPath = join(root, 'schema', 'script.sche if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { const files = {}; - for (const split of ['dev', 'test']) { + for (const split of ['dev', 'test', 'test2']) { const p = join(root, 'data', `${split}.jsonl`); if (existsSync(p)) files[split] = p; } diff --git a/test/bench.test.mjs b/test/bench.test.mjs index c6cf3e0..9bbe72c 100644 --- a/test/bench.test.mjs +++ b/test/bench.test.mjs @@ -131,3 +131,11 @@ test('similarity helper: identical texts match, different texts do not', () => { assert.equal(jaccard(shingles('buy gift cards today now'), shingles('buy gift cards today now')), 1); assert.ok(jaccard(shingles('buy gift cards today now'), shingles('your appointment is on thursday')) < 0.1); }); + +test('test2 has 20 scam and 20 benign records with hard cases, and validates', () => { + const rows = readJsonl(join(root, 'data', 'test2.jsonl')); + assert.equal(rows.length, 40); + assert.equal(rows.filter((r) => r.label === 'scam').length, 20); + assert.ok(rows.filter((r) => r.label === 'benign' && r.difficulty === 'tricky').length >= 8); + assert.deepEqual(validate({ test2: join(root, 'data', 'test2.jsonl') }), []); +});