From 0ada4d8fc0a8835b22fa4a17a6a26da42c9c071e Mon Sep 17 00:00:00 2001 From: im-rk Date: Fri, 9 Oct 2026 22:00:43 +0530 Subject: [PATCH] feat: 60 synthetic scripts (dev and test), JSON Schema, validator, scorer and CI --- .gitattributes | 1 + .github/workflows/ci.yml | 22 +++++++ data/dev.jsonl | 30 +++++++++ data/test.jsonl | 30 +++++++++ docs/dataset-spec.md | 2 +- package-lock.json | 77 ++++++++++++++++++++++ package.json | 17 +++++ schema/script.schema.json | 35 ++++++++++ scripts/lib.mjs | 37 +++++++++++ scripts/score.mjs | 107 ++++++++++++++++++++++++++++++ scripts/validate.mjs | 85 ++++++++++++++++++++++++ test/bench.test.mjs | 133 ++++++++++++++++++++++++++++++++++++++ 12 files changed, 575 insertions(+), 1 deletion(-) create mode 100644 .gitattributes create mode 100644 .github/workflows/ci.yml create mode 100644 data/dev.jsonl create mode 100644 data/test.jsonl create mode 100644 package-lock.json create mode 100644 package.json create mode 100644 schema/script.schema.json create mode 100644 scripts/lib.mjs create mode 100644 scripts/score.mjs create mode 100644 scripts/validate.mjs create mode 100644 test/bench.test.mjs diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..6313b56 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +* text=auto eol=lf diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..263e0e5 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,22 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + - run: npm ci + - run: npm run validate + - run: npm test diff --git a/data/dev.jsonl b/data/dev.jsonl new file mode 100644 index 0000000..a3f9b2e --- /dev/null +++ b/data/dev.jsonl @@ -0,0 +1,30 @@ +{"id":"d-001","split":"dev","label":"scam","category":"gift_card","channel":"voice","text":"The bank says I have to buy five $200 gift cards today or my account is frozen.","context":{"actionType":"purchase","amountCents":100000,"payee":"gift cards"},"difficulty":"easy","author":"team-dev","notes":"classic gift card + bank impersonation + urgency"} +{"id":"d-002","split":"dev","label":"scam","category":"government_impersonation","channel":"sms","text":"Hello, this is Mark from the Social Security office. Your benefits are suspended. To reactivate, purchase Google Play cards worth $500 and read me the codes.","difficulty":"easy","author":"team-dev","notes":"agency + prepaid cards"} +{"id":"d-003","split":"dev","label":"scam","category":"wire_crypto","channel":"call","text":"Your account has been compromised. Move all your savings to this secure Bitcoin wallet right now before the hackers drain it.","difficulty":"easy","author":"team-dev","notes":"safe-account crypto transfer"} +{"id":"d-004","split":"dev","label":"scam","category":"verification_code","channel":"call","text":"This is the fraud department. We detected a suspicious charge of $890. To cancel it, I need the six digit code we just sent to your phone.","difficulty":"easy","author":"team-dev","notes":"asks for OTP"} +{"id":"d-005","split":"dev","label":"scam","category":"tech_support_remote","channel":"call","text":"A pop-up says my computer has a virus. The man said to install AnyDesk so he can remove it and I should pay $299 by gift card.","difficulty":"easy","author":"team-dev","notes":"remote access + gift card"} +{"id":"d-006","split":"dev","label":"scam","category":"family_emergency","channel":"voice","text":"Grandma, it's me. I'm in jail after an accident and need $2,000 for bail. Please don't tell Mom and Dad.","context":{"actionType":"transfer","amountCents":200000,"payee":"bail"},"difficulty":"easy","author":"team-dev","notes":"grandparent scam + secrecy"} +{"id":"d-007","split":"dev","label":"scam","category":"prize_lottery","channel":"sms","text":"Congratulations! You won the national sweepstakes jackpot. To claim your prize, pay the $150 processing fee today.","difficulty":"easy","author":"team-dev","notes":"prize with a fee"} +{"id":"d-008","split":"dev","label":"scam","category":"delivery_fee","channel":"sms","text":"Your parcel could not be delivered. Pay a $2.99 redelivery fee at the link to avoid return.","difficulty":"tricky","author":"team-dev","notes":"small fee, few keywords"} +{"id":"d-009","split":"dev","label":"scam","category":"romance_investment","channel":"email","text":"I've made so much on this crypto platform, my love. Deposit just $1,000 and I'll show you how to double it by Friday. Keep this between us.","difficulty":"tricky","author":"team-dev","notes":"romance + crypto + secrecy"} +{"id":"d-010","split":"dev","label":"scam","category":"government_impersonation","channel":"call","text":"This is the IRS. You owe $3,400 in back taxes and a warrant will be issued unless you pay immediately with a prepaid card.","difficulty":"easy","author":"team-dev","notes":"agency + threat + prepaid card"} +{"id":"d-011","split":"dev","label":"scam","category":"family_emergency","channel":"sms","text":"Hi Grandma, I'm travelling and my card got blocked. Could you wire me $400 to this account? I'll pay you back, love you.","context":{"actionType":"transfer","amountCents":40000,"payee":"unknown account"},"difficulty":"tricky","author":"team-dev","notes":"polite impersonation, no scam vocabulary"} +{"id":"d-012","split":"dev","label":"scam","category":"family_emergency","channel":"call","text":"I'm a lawyer representing your grandson. He needs $5,000 for bail today. He is embarrassed and asked that we not call his parents.","context":{"actionType":"transfer","amountCents":500000,"payee":"lawyer"},"difficulty":"tricky","author":"team-dev","notes":"lawyer intermediary"} +{"id":"d-013","split":"dev","label":"scam","category":"tech_support_remote","channel":"call","text":"Microsoft support here. We see errors on your PC. Please give me remote access so I can fix it.","difficulty":"easy","author":"team-dev","notes":"remote access request"} +{"id":"d-014","split":"dev","label":"scam","category":"bank_impersonation","channel":"email","text":"Your account is locked. Verify your identity within 24 hours by sending a photo of your debit card and PIN.","difficulty":"tricky","author":"team-dev","notes":"credential phishing"} +{"id":"d-015","split":"dev","label":"scam","category":"wire_crypto","channel":"voice","text":"The investment manager says to wire $8,000 to the new account number today or I lose the guaranteed returns.","context":{"actionType":"transfer","amountCents":800000,"payee":"investment manager"},"difficulty":"easy","author":"team-dev","notes":"guaranteed returns + wire"} +{"id":"d-016","split":"dev","label":"benign","category":"pharmacy","channel":"sms","text":"Your prescription is ready for pickup at Greenleaf Pharmacy.","difficulty":"easy","author":"team-dev","notes":"refill notice"} +{"id":"d-017","split":"dev","label":"benign","category":"bank_alert_genuine","channel":"sms","text":"Your bank noticed a suspicious charge on your card. If this wasn't you, call the number on the back of your card. We will never ask for your PIN or a code.","difficulty":"tricky","author":"team-dev","notes":"genuine fraud alert with scary words"} +{"id":"d-018","split":"dev","label":"benign","category":"utility_bill","channel":"voice","text":"Pay City Power & Light $86.40 for the October bill.","context":{"actionType":"payment","amountCents":8640,"payee":"City Power & Light","trusted":true},"difficulty":"easy","author":"team-dev","notes":"routine bill to a trusted payee"} +{"id":"d-019","split":"dev","label":"benign","category":"delivery_genuine","channel":"sms","text":"Your package will arrive today between 2 and 4 PM.","difficulty":"easy","author":"team-dev","notes":"delivery window"} +{"id":"d-020","split":"dev","label":"benign","category":"family_chat","channel":"voice","text":"Can you send me the soup recipe you made on Sunday?","difficulty":"easy","author":"team-dev","notes":"family chat"} +{"id":"d-021","split":"dev","label":"benign","category":"subscription_receipt","channel":"email","text":"Thanks for your payment of $9.99 to Daily News. Your next renewal is on November 3.","difficulty":"easy","author":"team-dev","notes":"receipt"} +{"id":"d-022","split":"dev","label":"benign","category":"appointment_reminder","channel":"sms","text":"Reminder: your appointment with Dr. Lee is Thursday at 10 AM.","difficulty":"easy","author":"team-dev","notes":"appointment"} +{"id":"d-023","split":"dev","label":"benign","category":"bank_alert_genuine","channel":"sms","text":"Alert: a $450 payment to Acme Hardware was approved. If you don't recognise it, contact us through the app or the number on your statement.","difficulty":"tricky","author":"team-dev","notes":"genuine alert with an amount"} +{"id":"d-024","split":"dev","label":"benign","category":"family_chat","channel":"voice","text":"Mom, I'm at the shop. Do you want me to buy some gift wrap or just ribbon for Aunt Rita's present?","difficulty":"tricky","author":"team-dev","notes":"gift wrap, not gift cards"} +{"id":"d-025","split":"dev","label":"benign","category":"bank_alert_genuine","channel":"email","text":"Reminder from your credit union: we will never ask you to pay with gift cards. Stay alert for scams.","difficulty":"tricky","author":"team-dev","notes":"warning about gift cards"} +{"id":"d-026","split":"dev","label":"benign","category":"pharmacy","channel":"voice","text":"Pay Greenleaf Pharmacy $24 for my refill.","context":{"actionType":"payment","amountCents":2400,"payee":"Greenleaf Pharmacy","trusted":true},"difficulty":"easy","author":"team-dev","notes":"trusted pharmacy payment"} +{"id":"d-027","split":"dev","label":"benign","category":"delivery_genuine","channel":"email","text":"Your order has shipped and arrives Friday. You do not need to do anything.","difficulty":"tricky","author":"team-dev","notes":"reassuring delivery notice"} +{"id":"d-028","split":"dev","label":"benign","category":"family_chat","channel":"voice","text":"Dad says he'll wire the money for the deck repair after the contractor sends the invoice, no rush.","difficulty":"tricky","author":"team-dev","notes":"innocent \"wire the money\""} +{"id":"d-029","split":"dev","label":"benign","category":"utility_bill","channel":"email","text":"Your water bill of $42.10 is due on the 20th. You can pay online or by check.","difficulty":"easy","author":"team-dev","notes":"bill reminder"} +{"id":"d-030","split":"dev","label":"benign","category":"appointment_reminder","channel":"call","text":"This is the dental office calling to confirm your cleaning today at 3 PM. Please call us back if you need to reschedule.","difficulty":"tricky","author":"team-dev","notes":"unsolicited call with \"today\""} diff --git a/data/test.jsonl b/data/test.jsonl new file mode 100644 index 0000000..ecebf1f --- /dev/null +++ b/data/test.jsonl @@ -0,0 +1,30 @@ +{"id":"t-001","split":"test","label":"scam","category":"gift_card","channel":"call","text":"Six Walmart gift cards at $100 each. The officer said my son will be released as soon as I read him the numbers.","context":{"actionType":"purchase","amountCents":60000,"payee":"gift cards"},"difficulty":"easy","author":"team-test","notes":"gift cards + bail pressure"} +{"id":"t-002","split":"test","label":"scam","category":"bank_impersonation","channel":"call","text":"Hello ma'am, I'm calling from your bank's security team. Someone is trying to take $2,300 from your account. Stay on the line and do not tell any of the staff at your branch.","difficulty":"easy","author":"team-test","notes":"impersonation + secrecy"} +{"id":"t-003","split":"test","label":"scam","category":"verification_code","channel":"call","text":"We texted you a one time passcode. Please read it back to me so I can stop the transfer.","difficulty":"easy","author":"team-test","notes":"asks to read OTP"} +{"id":"t-004","split":"test","label":"scam","category":"tech_support_remote","channel":"call","text":"Your router has been hacked. Download TeamViewer so our technician can secure your network today.","difficulty":"easy","author":"team-test","notes":"remote access"} +{"id":"t-005","split":"test","label":"scam","category":"government_impersonation","channel":"call","text":"Medicare is issuing new cards. To avoid losing your coverage, confirm your number and pay the $25 activation fee now.","difficulty":"tricky","author":"team-test","notes":"agency + fee"} +{"id":"t-006","split":"test","label":"scam","category":"prize_lottery","channel":"email","text":"You've been selected for a $10,000 grocery giveaway. Pay $60 in taxes with an Apple card to receive it.","difficulty":"tricky","author":"team-test","notes":"prize + prepaid card taxes"} +{"id":"t-007","split":"test","label":"scam","category":"romance_investment","channel":"email","text":"Darling, the gold shipment is stuck in customs. I just need $3,500 sent to my agent via Western Union and then we can finally meet.","context":{"actionType":"transfer","amountCents":350000,"payee":"agent"},"difficulty":"tricky","author":"team-test","notes":"romance + Western Union"} +{"id":"t-008","split":"test","label":"scam","category":"wire_crypto","channel":"call","text":"Take cash out of your account and put it in the bitcoin machine at the gas station. The code on screen is how we protect your money.","difficulty":"easy","author":"team-test","notes":"crypto ATM"} +{"id":"t-009","split":"test","label":"scam","category":"delivery_fee","channel":"sms","text":"We tried to deliver your parcel but the address is incomplete. Update it and pay the small fee of $1.95 to schedule redelivery.","difficulty":"tricky","author":"team-test","notes":"delivery fee"} +{"id":"t-010","split":"test","label":"scam","category":"family_emergency","channel":"voice","text":"Gran, I'm so sorry to call late. I had a car accident and I'm at the hospital. The doctor needs a deposit of $900 right away. Please don't worry Mom.","context":{"actionType":"transfer","amountCents":90000,"payee":"hospital"},"difficulty":"tricky","author":"team-test","notes":"accident + secrecy"} +{"id":"t-011","split":"test","label":"scam","category":"romance_investment","channel":"email","text":"Hello, this is Priya's landlord. She asked me to collect the deposit for the apartment by tomorrow. Could you send $1,200 to me directly? I can give you my account number.","context":{"actionType":"transfer","amountCents":120000,"payee":"landlord"},"difficulty":"tricky","author":"team-test","notes":"third-party payment request, no scam vocabulary"} +{"id":"t-012","split":"test","label":"scam","category":"delivery_fee","channel":"call","text":"Your electricity will be disconnected within the hour due to an unpaid balance. Pay now with a prepaid card to keep your service.","difficulty":"easy","author":"team-test","notes":"utility shut-off threat"} +{"id":"t-013","split":"test","label":"scam","category":"gift_card","channel":"email","text":"I'm your boss Alan. I'm in a meeting and need you to quietly pick up some iTunes cards for a client, I'll reimburse you. Send me the card numbers when you have them.","difficulty":"tricky","author":"team-test","notes":"boss impersonation + secrecy + cards"} +{"id":"t-014","split":"test","label":"scam","category":"government_impersonation","channel":"call","text":"You missed jury duty and the sheriff has issued a warrant. You can settle the fine today by phone.","difficulty":"tricky","author":"team-test","notes":"warrant + fine"} +{"id":"t-015","split":"test","label":"scam","category":"prize_lottery","channel":"sms","text":"Our system refunded you $400 too much. Please send $350 back by wire and keep the rest for your trouble.","context":{"actionType":"transfer","amountCents":35000,"payee":"refund desk"},"difficulty":"tricky","author":"team-test","notes":"refund overpayment"} +{"id":"t-016","split":"test","label":"benign","category":"bank_alert_genuine","channel":"sms","text":"Your card ending 4821 was used at a gas station for $62.14. Not you? Call the number printed on your card.","difficulty":"tricky","author":"team-test","notes":"genuine card alert"} +{"id":"t-017","split":"test","label":"benign","category":"pharmacy","channel":"call","text":"Hi, it's Greenleaf Pharmacy. Your blood pressure refill is ready. Pickup is open until 6 PM today.","difficulty":"tricky","author":"team-test","notes":"unsolicited pharmacy call with \"today\""} +{"id":"t-018","split":"test","label":"benign","category":"family_chat","channel":"voice","text":"Can you pay Mr. Evans next door $20 for mowing the lawn when you see him? I'll transfer it to you tonight.","context":{"actionType":"payment","amountCents":2000,"payee":"Mr. Evans"},"difficulty":"tricky","author":"team-test","notes":"small neighbour payment"} +{"id":"t-019","split":"test","label":"benign","category":"utility_bill","channel":"email","text":"Your gas bill statement for October is now available in your online account.","difficulty":"easy","author":"team-test","notes":"statement ready"} +{"id":"t-020","split":"test","label":"benign","category":"appointment_reminder","channel":"call","text":"This is your clinic. Your lab results are ready. Please call today between 9 and 5 to schedule a visit.","difficulty":"tricky","author":"team-test","notes":"clinic call with \"today\""} +{"id":"t-021","split":"test","label":"benign","category":"subscription_receipt","channel":"email","text":"Receipt: $12.99 monthly plan renewed. To cancel, open the app and go to account settings.","difficulty":"easy","author":"team-test","notes":"receipt"} +{"id":"t-022","split":"test","label":"benign","category":"delivery_genuine","channel":"sms","text":"Out for delivery: your order arrives today. Signature not required.","difficulty":"easy","author":"team-test","notes":"delivery"} +{"id":"t-023","split":"test","label":"benign","category":"bank_alert_genuine","channel":"email","text":"We will never call you to ask for your password or a security code. If someone does, hang up and call us.","difficulty":"tricky","author":"team-test","notes":"bank warning about codes"} +{"id":"t-024","split":"test","label":"benign","category":"family_chat","channel":"voice","text":"Bought a $50 gift card for Aunt Rita's birthday. Do you think she'd like the bookstore one?","difficulty":"tricky","author":"team-test","notes":"innocent gift card talk"} +{"id":"t-025","split":"test","label":"benign","category":"family_chat","channel":"voice","text":"Your brother called, he wants to know if Sunday lunch is still on.","difficulty":"easy","author":"team-test","notes":"family chat"} +{"id":"t-026","split":"test","label":"benign","category":"pharmacy","channel":"voice","text":"Pay Greenleaf Pharmacy $31 for the vitamins.","context":{"actionType":"payment","amountCents":3100,"payee":"Greenleaf Pharmacy","trusted":true},"difficulty":"easy","author":"team-test","notes":"trusted pharmacy payment"} +{"id":"t-027","split":"test","label":"benign","category":"delivery_genuine","channel":"sms","text":"Package held at the front desk. Pick it up with ID before Friday.","difficulty":"tricky","author":"team-test","notes":"held package, genuine"} +{"id":"t-028","split":"test","label":"benign","category":"appointment_reminder","channel":"email","text":"Your car inspection is due today. Book a slot online or visit any station.","difficulty":"tricky","author":"team-test","notes":"urgency word, genuine"} +{"id":"t-029","split":"test","label":"benign","category":"bank_alert_genuine","channel":"sms","text":"Your transfer of $200 to Daniel Reyes was completed. If you did not make this transfer, call our fraud line on the back of your card.","difficulty":"tricky","author":"team-test","notes":"genuine transfer confirmation"} +{"id":"t-030","split":"test","label":"benign","category":"family_chat","channel":"voice","text":"Grandpa, the plumber said the fix would be $240. I'll pay him myself, don't worry about it.","difficulty":"tricky","author":"team-test","notes":"reassuring family message with an amount"} diff --git a/docs/dataset-spec.md b/docs/dataset-spec.md index c234838..e6cb606 100644 --- a/docs/dataset-spec.md +++ b/docs/dataset-spec.md @@ -30,7 +30,7 @@ Benign: `bank_alert_genuine`, `pharmacy`, `utility_bill`, `delivery_genuine`, `f - `test` is frozen after the first scoring run; later edits go in a new file `test-v2.jsonl` with a changelog. ## Authorship and independence (read this before quoting a number) -The first release (60 scripts) was written by the project team with AI assistance, in two separate passes: `dev` first, `test` second without opening the detector's rules. That is **not** an independent benchmark: the people who wrote the detector also wrote the examples, so results are a *development yardstick*, likely optimistic. The honest way to strengthen it is to have someone who has never seen the detector write more `test` scripts (`data/test-v2.jsonl`) and to add real, consented, de-identified examples from partner organisations. Contributions of independently written scripts are welcome. +The first release (60 scripts) was written in one working session by the same people (with AI assistance) who built the detector it was made for, **with knowledge of how that detector works**. The `test` split is held out in the narrow sense that the detector is tuned on `dev` only and run on `test` once, with the result recorded; it is **not** blind. That makes this a *development yardstick*, not an independent benchmark, and the numbers are likely optimistic. The honest way to strengthen it is to have someone who has never seen the detector write more `test` scripts (`data/test-v2.jsonl`) and to add real, consented, de-identified examples from partner organisations. Contributions of independently written scripts are welcome. ## Hard cases on purpose Each split deliberately contains: genuine bank fraud alerts that use scary words; family messages that mention "gift card" or "wire the money" innocently; and scams with **no** scam vocabulary (a polite landlord or "boss" asking for a payment). Detectors that only match keywords will fail on these, and the failures are reported. diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..da717eb --- /dev/null +++ b/package-lock.json @@ -0,0 +1,77 @@ +{ + "name": "scam-script-bench", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "scam-script-bench", + "version": "0.1.0", + "license": "MIT", + "devDependencies": { + "ajv": "^8.17.1" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-uri": { + "version": "3.1.8", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.8.tgz", + "integrity": "sha512-GZMtZUTNRpOVIECoXwLNZS5xUGE+mVNbTB8h/7Rwh2TFWcBQiPzTgyZi05BF9UMZKkLJv8XBRJTlU7zg8+ZfMg==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true, + "license": "MIT" + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..4d05a9f --- /dev/null +++ b/package.json @@ -0,0 +1,17 @@ +{ + "name": "scam-script-bench", + "version": "0.1.0", + "description": "A small, open, synthetic benchmark for payment-pressure scam detection, with deliberately hard benign and scam cases.", + "license": "MIT", + "type": "module", + "private": false, + "engines": { "node": ">=20" }, + "scripts": { + "validate": "node scripts/validate.mjs", + "score": "node scripts/score.mjs", + "test": "node --test \"test/**/*.test.mjs\"" + }, + "devDependencies": { + "ajv": "^8.17.1" + } +} diff --git a/schema/script.schema.json b/schema/script.schema.json new file mode 100644 index 0000000..6ca62d2 --- /dev/null +++ b/schema/script.schema.json @@ -0,0 +1,35 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/Monolithic-Dev/scam-script-bench/schema/script.schema.json", + "title": "scam-script-bench record", + "type": "object", + "additionalProperties": false, + "required": ["id", "split", "label", "category", "channel", "text", "difficulty", "author"], + "properties": { + "id": { "type": "string", "pattern": "^[dt]-\\d{3}$" }, + "split": { "enum": ["dev", "test"] }, + "label": { "enum": ["scam", "benign"] }, + "category": { + "enum": [ + "gift_card", "wire_crypto", "bank_impersonation", "government_impersonation", "tech_support_remote", + "family_emergency", "prize_lottery", "verification_code", "delivery_fee", "romance_investment", + "bank_alert_genuine", "pharmacy", "utility_bill", "delivery_genuine", "family_chat", "subscription_receipt", "appointment_reminder" + ] + }, + "channel": { "enum": ["call", "sms", "email", "voice", "unknown"] }, + "text": { "type": "string", "minLength": 1, "maxLength": 600 }, + "context": { + "type": "object", + "additionalProperties": false, + "properties": { + "actionType": { "enum": ["purchase", "payment", "transfer", "none"] }, + "amountCents": { "type": "integer", "minimum": 0, "maximum": 10000000 }, + "payee": { "type": "string", "maxLength": 100 }, + "trusted": { "type": "boolean" } + } + }, + "difficulty": { "enum": ["easy", "tricky"] }, + "author": { "type": "string", "minLength": 1, "maxLength": 40 }, + "notes": { "type": "string", "maxLength": 200 } + } +} diff --git a/scripts/lib.mjs b/scripts/lib.mjs new file mode 100644 index 0000000..f293219 --- /dev/null +++ b/scripts/lib.mjs @@ -0,0 +1,37 @@ +import { readFileSync } from 'node:fs'; + +export const SCAM_CATEGORIES = [ + 'gift_card', 'wire_crypto', 'bank_impersonation', 'government_impersonation', 'tech_support_remote', + 'family_emergency', 'prize_lottery', 'verification_code', 'delivery_fee', 'romance_investment', +]; + +/** Read a JSONL file into records; throws with the offending line number on bad JSON. */ +export function readJsonl(path) { + const rows = []; + const lines = readFileSync(path, 'utf8').split(/\r?\n/); + lines.forEach((line, i) => { + if (!line.trim()) return; + try { + rows.push(JSON.parse(line)); + } catch (e) { + throw new Error(`${path}:${i + 1}: invalid JSON (${e.message})`); + } + }); + return rows; +} + +/** Lowercase word 3-grams, for near-duplicate detection. */ +export function shingles(text) { + const words = text.toLowerCase().replace(/[^a-z0-9$\s]/g, ' ').split(/\s+/).filter(Boolean); + const out = new Set(); + for (let i = 0; i + 2 < words.length; i++) out.add(words.slice(i, i + 3).join(' ')); + if (out.size === 0) out.add(words.join(' ')); + return out; +} + +export function jaccard(a, b) { + if (a.size === 0 && b.size === 0) return 0; + let inter = 0; + for (const x of a) if (b.has(x)) inter++; + return inter / (a.size + b.size - inter); +} diff --git a/scripts/score.mjs b/scripts/score.mjs new file mode 100644 index 0000000..f81b0b9 --- /dev/null +++ b/scripts/score.mjs @@ -0,0 +1,107 @@ +#!/usr/bin/env node +import { writeFileSync } from 'node:fs'; +import { pathToFileURL } from 'node:url'; +import { readJsonl } from './lib.mjs'; + +const VERDICTS = new Set(['allow', 'caution', 'hold']); + +function ratio(n, d) { + return d === 0 ? null : n / d; +} + +function confusion(labels, verdicts, positive) { + let tp = 0, fp = 0, tn = 0, fn = 0; + for (const r of labels) { + const predicted = positive(verdicts.get(r.id)); + const actual = r.label === 'scam'; + if (predicted && actual) tp++; + else if (predicted && !actual) fp++; + else if (!predicted && !actual) tn++; + else fn++; + } + const precision = ratio(tp, tp + fp); + const recall = ratio(tp, tp + fn); + const f1 = precision === null || recall === null || precision + recall === 0 ? null : (2 * precision * recall) / (precision + recall); + return { tp, fp, tn, fn, precision, recall, f1, fpr: ratio(fp, fp + tn) }; +} + +/** + * Score predictions against labels at two operating points: + * hold: positive = verdict "hold"; flag: positive = "hold" or "caution". Positive class = scam. + * Missing predictions count as "allow" and are reported. Division by zero yields null, never 0. + */ +export function score(labels, predictions) { + const verdicts = new Map(); + const problems = []; + for (const p of predictions) { + if (!VERDICTS.has(p.verdict)) problems.push(`prediction ${p.id}: unknown verdict "${p.verdict}"`); + else verdicts.set(p.id, p.verdict); + } + const known = new Set(labels.map((l) => l.id)); + for (const p of predictions) if (!known.has(p.id)) problems.push(`prediction ${p.id}: no such script`); + const missing = labels.filter((l) => !verdicts.has(l.id)).map((l) => l.id); + for (const id of missing) verdicts.set(id, 'allow'); + + const hold = confusion(labels, verdicts, (v) => v === 'hold'); + const flag = confusion(labels, verdicts, (v) => v === 'hold' || v === 'caution'); + + const perCategory = {}; + for (const r of labels.filter((l) => l.label === 'scam')) { + const c = (perCategory[r.category] ??= { total: 0, caughtAtHold: 0, caughtAtFlag: 0 }); + c.total++; + const v = verdicts.get(r.id); + if (v === 'hold') c.caughtAtHold++; + if (v === 'hold' || v === 'caution') c.caughtAtFlag++; + } + + const misclassified = []; + for (const r of labels) { + const v = verdicts.get(r.id); + const holdWrong = (v === 'hold') !== (r.label === 'scam'); + const flagWrong = (v !== 'allow') !== (r.label === 'scam'); + if (holdWrong || flagWrong) { + misclassified.push({ + id: r.id, label: r.label, category: r.category, difficulty: r.difficulty, verdict: v, + error: r.label === 'scam' ? (v === 'allow' ? 'missed scam' : 'only flagged, not held') : v === 'hold' ? 'false hold' : 'false flag', + }); + } + } + return { n: labels.length, hold, flag, perCategory, misclassified, missingPredictions: missing, problems }; +} + +const pct = (x) => (x === null ? 'n/a' : `${(x * 100).toFixed(1)}%`); + +export function toMarkdown(result, title = 'Results') { + const row = (name, m) => + `| ${name} | ${pct(m.recall)} | ${pct(m.precision)} | ${pct(m.f1)} | ${pct(m.fpr)} | ${m.tp} / ${m.fp} / ${m.tn} / ${m.fn} |`; + const lines = [ + `### ${title} (${result.n} scripts)`, + '', + '| Operating point | Scam recall | Precision | F1 | Benign false-positive rate | TP / FP / TN / FN |', + '|---|---|---|---|---|---|', + row('hold (blocks the action)', result.hold), + row('flag (hold or caution)', result.flag), + '', + '**Recall by scam category** (held / flagged / total)', + '', + ...Object.entries(result.perCategory).map(([c, v]) => `- ${c}: ${v.caughtAtHold} / ${v.caughtAtFlag} / ${v.total}`), + ]; + if (result.misclassified.length) { + lines.push('', '**Misclassified**', '', '| id | label | category | difficulty | verdict | error |', '|---|---|---|---|---|---|'); + for (const m of result.misclassified) lines.push(`| ${m.id} | ${m.label} | ${m.category} | ${m.difficulty} | ${m.verdict} | ${m.error} |`); + } + if (result.missingPredictions.length) lines.push('', `Missing predictions (counted as allow): ${result.missingPredictions.join(', ')}`); + if (result.problems.length) lines.push('', `Problems: ${result.problems.join('; ')}`); + return lines.join('\n') + '\n'; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + const args = Object.fromEntries(process.argv.slice(2).reduce((acc, a, i, all) => (a.startsWith('--') ? [...acc, [a.slice(2), all[i + 1]]] : acc), [])); + if (!args.labels || !args.pred) { + console.error('usage: node scripts/score.mjs --labels data/test.jsonl --pred predictions.jsonl [--json results.json]'); + process.exit(2); + } + const result = score(readJsonl(args.labels), readJsonl(args.pred)); + if (args.json) writeFileSync(args.json, JSON.stringify(result, null, 2) + '\n'); + process.stdout.write(toMarkdown(result, args.labels)); +} diff --git a/scripts/validate.mjs b/scripts/validate.mjs new file mode 100644 index 0000000..a6ff8b3 --- /dev/null +++ b/scripts/validate.mjs @@ -0,0 +1,85 @@ +#!/usr/bin/env node +import { createRequire } from 'node:module'; +import { existsSync, readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { jaccard, readJsonl, SCAM_CATEGORIES, shingles } from './lib.mjs'; + +const require = createRequire(import.meta.url); +const root = join(dirname(fileURLToPath(import.meta.url)), '..'); + +const PRIVATE_DATA = /https?:\/\/|www\./i; +const PHONE = /\b\d{3}[-.\s]\d{3}[-.\s]\d{4}\b/; +const NEAR_DUPLICATE = 0.8; + +/** + * Validate one or more splits. Returns a list of human-readable problems (empty when everything is fine). + * `files` is { dev: path, test: path }; any split may be omitted. + */ +export function validate(files, { schemaPath = join(root, 'schema', 'script.schema.json') } = {}) { + const Ajv = require('ajv/dist/2020').default; + const ajv = new Ajv({ allErrors: true, strict: true }); + const check = ajv.compile(JSON.parse(readFileSync(schemaPath, 'utf8'))); + const problems = []; + const bySplit = {}; + + for (const [split, path] of Object.entries(files)) { + const rows = readJsonl(path); + bySplit[split] = rows; + const ids = new Set(); + for (const r of rows) { + const where = `${split}/${r?.id ?? '?'}`; + if (!check(r)) { + for (const e of check.errors) problems.push(`${where}: ${e.instancePath || '(record)'} ${e.message}`); + continue; + } + if (r.split !== split) problems.push(`${where}: split "${r.split}" does not match file "${split}"`); + if (!r.id.startsWith(split === 'dev' ? 'd-' : 't-')) problems.push(`${where}: id prefix does not match the split`); + if (ids.has(r.id)) problems.push(`${where}: duplicate id`); + ids.add(r.id); + if (PRIVATE_DATA.test(r.text) || PHONE.test(r.text)) problems.push(`${where}: contains a URL or phone number (use invented, generic text)`); + const isScamCat = SCAM_CATEGORIES.includes(r.category); + if ((r.label === 'scam') !== isScamCat) problems.push(`${where}: category "${r.category}" does not belong to label "${r.label}"`); + } + + const scams = rows.filter((r) => r.label === 'scam'); + const benign = rows.filter((r) => r.label === 'benign'); + if (scams.length !== 15) problems.push(`${split}: expected 15 scam records, found ${scams.length}`); + if (benign.length !== 15) problems.push(`${split}: expected 15 benign records, found ${benign.length}`); + const tricky = benign.filter((r) => r.difficulty === 'tricky').length; + if (tricky < Math.ceil(benign.length * 0.4)) problems.push(`${split}: at least 40% of benign records must be tricky (found ${tricky}/${benign.length})`); + const cats = new Set(scams.map((r) => r.category)); + if (cats.size < 6) problems.push(`${split}: at least 6 distinct scam categories required (found ${cats.size})`); + } + + if (bySplit.dev && bySplit.test) { + const devAuthors = new Set(bySplit.dev.map((r) => r.author)); + const testAuthors = new Set(bySplit.test.map((r) => r.author)); + for (const a of devAuthors) if (testAuthors.has(a)) problems.push(`author label "${a}" appears in both dev and test`); + } + + const all = Object.entries(bySplit).flatMap(([split, rows]) => rows.map((r) => ({ split, r, sh: shingles(r.text ?? '') }))); + for (let i = 0; i < all.length; i++) { + for (let j = i + 1; j < all.length; j++) { + const a = all[i]; + const b = all[j]; + if (jaccard(a.sh, b.sh) >= NEAR_DUPLICATE) problems.push(`near-duplicate texts: ${a.split}/${a.r.id} and ${b.split}/${b.r.id}`); + } + } + return problems; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + const files = {}; + for (const split of ['dev', 'test']) { + const p = join(root, 'data', `${split}.jsonl`); + if (existsSync(p)) files[split] = p; + } + const problems = validate(files); + if (problems.length) { + console.error(`${problems.length} problem(s):\n- ${problems.join('\n- ')}`); + process.exit(1); + } + const counts = Object.entries(files).map(([s, p]) => `${s}: ${readJsonl(p).length}`).join(', '); + console.log(`OK (${counts})`); +} diff --git a/test/bench.test.mjs b/test/bench.test.mjs new file mode 100644 index 0000000..c6cf3e0 --- /dev/null +++ b/test/bench.test.mjs @@ -0,0 +1,133 @@ +import assert from 'node:assert/strict'; +import { mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { test } from 'node:test'; +import { jaccard, readJsonl, shingles } from '../scripts/lib.mjs'; +import { score, toMarkdown } from '../scripts/score.mjs'; +import { validate } from '../scripts/validate.mjs'; + +const root = join(dirname(fileURLToPath(import.meta.url)), '..'); +const real = { dev: join(root, 'data', 'dev.jsonl'), test: join(root, 'data', 'test.jsonl') }; + +test('the shipped dataset passes every validation rule', () => { + assert.deepEqual(validate(real), []); +}); + +test('each split has 15 scam and 15 benign records, with hard cases on both sides', () => { + for (const [split, path] of Object.entries(real)) { + const rows = readJsonl(path); + assert.equal(rows.length, 30, split); + assert.equal(rows.filter((r) => r.label === 'scam').length, 15, split); + assert.ok(rows.filter((r) => r.label === 'benign' && r.difficulty === 'tricky').length >= 6, `${split} tricky benign`); + assert.ok(rows.filter((r) => r.label === 'scam' && r.difficulty === 'tricky').length >= 3, `${split} tricky scams`); + assert.ok(new Set(rows.filter((r) => r.label === 'scam').map((r) => r.category)).size >= 6, `${split} categories`); + } +}); + +function tmpFile(rows) { + const dir = mkdtempSync(join(tmpdir(), 'ssb-')); + const p = join(dir, 'x.jsonl'); + writeFileSync(p, rows.map((r) => JSON.stringify(r)).join('\n') + '\n'); + return p; +} + +test('validation catches the mistakes it exists to catch', () => { + const rows = readJsonl(real.dev); + const problemsFor = (mutate) => validate({ dev: tmpFile(mutate(structuredClone(rows))) }).join(' | '); + + assert.match(problemsFor((r) => { r[1].id = r[0].id; return r; }), /duplicate id/); + assert.match(problemsFor((r) => { r[0].split = 'test'; return r; }), /does not match file/); + assert.match(problemsFor((r) => { r[0].text += ' see https://example.com'; return r; }), /URL or phone number/); + assert.match(problemsFor((r) => { r[0].text += ' call 555-123-4567'; return r; }), /URL or phone number/); + assert.match(problemsFor((r) => { r[0].category = 'pharmacy'; return r; }), /does not belong to label/); + assert.match(problemsFor((r) => { r[0].channel = 'carrier-pigeon'; return r; }), /channel/); + assert.match(problemsFor((r) => { delete r[0].difficulty; return r; }), /difficulty/); + assert.match(problemsFor((r) => r.slice(1)), /expected 15 scam/); + assert.match(problemsFor((r) => { for (const x of r) if (x.label === 'benign') x.difficulty = 'easy'; return r; }), /at least 40%/); + assert.match(problemsFor((r) => { for (const x of r) if (x.label === 'scam') x.category = 'gift_card'; return r; }), /distinct scam categories/); + assert.match(problemsFor((r) => { r[1].text = r[0].text; return r; }), /near-duplicate/); + assert.match(problemsFor((r) => { r[0].extra = 1; return r; }), /additional properties/); +}); + +test('dev and test must come from different author groups', () => { + const dev = readJsonl(real.dev); + const test2 = readJsonl(real.test).map((r) => ({ ...r, author: dev[0].author })); + assert.match(validate({ dev: real.dev, test: tmpFile(test2) }).join(' | '), /appears in both dev and test/); +}); + +test('a malformed line reports its line number', () => { + const dir = mkdtempSync(join(tmpdir(), 'ssb-')); + const p = join(dir, 'bad.jsonl'); + writeFileSync(p, '{"id":"d-001"}\n{oops\n'); + assert.throws(() => readJsonl(p), /bad\.jsonl:2: invalid JSON/); +}); + +const labels = [ + { id: 'a', label: 'scam', category: 'gift_card', difficulty: 'easy' }, + { id: 'b', label: 'scam', category: 'gift_card', difficulty: 'easy' }, + { id: 'c', label: 'scam', category: 'wire_crypto', difficulty: 'tricky' }, + { id: 'd', label: 'benign', category: 'pharmacy', difficulty: 'easy' }, + { id: 'e', label: 'benign', category: 'family_chat', difficulty: 'tricky' }, + { id: 'f', label: 'benign', category: 'utility_bill', difficulty: 'easy' }, +]; + +test('scoring: hold and flag operating points are computed by hand-checked arithmetic', () => { + const pred = [ + { id: 'a', verdict: 'hold' }, // caught + { id: 'b', verdict: 'caution' }, // flagged only + { id: 'c', verdict: 'allow' }, // missed + { id: 'd', verdict: 'allow' }, + { id: 'e', verdict: 'caution' }, // false flag + { id: 'f', verdict: 'hold' }, // false hold + ]; + const r = score(labels, pred); + assert.deepEqual([r.hold.tp, r.hold.fp, r.hold.tn, r.hold.fn], [1, 1, 2, 2]); + assert.equal(r.hold.recall, 1 / 3); + assert.equal(r.hold.precision, 1 / 2); + assert.equal(r.hold.fpr, 1 / 3); + assert.ok(Math.abs(r.hold.f1 - 0.4) < 1e-12); + assert.deepEqual([r.flag.tp, r.flag.fp, r.flag.tn, r.flag.fn], [2, 2, 1, 1]); + assert.equal(r.flag.recall, 2 / 3); + assert.deepEqual(r.perCategory, { gift_card: { total: 2, caughtAtHold: 1, caughtAtFlag: 2 }, wire_crypto: { total: 1, caughtAtHold: 0, caughtAtFlag: 0 } }); + const errors = Object.fromEntries(r.misclassified.map((m) => [m.id, m.error])); + assert.deepEqual(errors, { b: 'only flagged, not held', c: 'missed scam', e: 'false flag', f: 'false hold' }); +}); + +test('scoring: perfect and empty detectors, divide-by-zero is null, missing predictions count as allow', () => { + const perfect = score(labels, labels.map((l) => ({ id: l.id, verdict: l.label === 'scam' ? 'hold' : 'allow' }))); + assert.equal(perfect.hold.recall, 1); + assert.equal(perfect.hold.precision, 1); + assert.equal(perfect.hold.fpr, 0); + assert.deepEqual(perfect.misclassified, []); + + const none = score(labels, []); + assert.equal(none.hold.recall, 0); + assert.equal(none.hold.precision, null, 'no positive predictions: precision is undefined, not 0'); + assert.equal(none.hold.f1, null); + assert.equal(none.missingPredictions.length, 6); + + const allBenign = score(labels.filter((l) => l.label === 'benign'), []); + assert.equal(allBenign.hold.recall, null, 'no scams in the labels: recall is undefined'); +}); + +test('scoring: unknown verdicts and unknown ids are reported, not silently accepted', () => { + const r = score(labels, [{ id: 'a', verdict: 'maybe' }, { id: 'zzz', verdict: 'hold' }]); + assert.equal(r.problems.length, 2); + assert.match(r.problems.join(' '), /unknown verdict/); + assert.match(r.problems.join(' '), /no such script/); +}); + +test('the markdown report states both operating points and lists the failures', () => { + const md = toMarkdown(score(labels, [{ id: 'a', verdict: 'hold' }]), 'demo'); + assert.match(md, /hold \(blocks the action\)/); + assert.match(md, /flag \(hold or caution\)/); + assert.match(md, /Misclassified/); + assert.match(md, /missed scam/); +}); + +test('similarity helper: identical texts match, different texts do not', () => { + assert.equal(jaccard(shingles('buy gift cards today now'), shingles('buy gift cards today now')), 1); + assert.ok(jaccard(shingles('buy gift cards today now'), shingles('your appointment is on thursday')) < 0.1); +});