From b7f3885119d6d2dda8e7470e7c867797d3e6d33a Mon Sep 17 00:00:00 2001 From: yashaskn8 Date: Fri, 30 Oct 2026 01:59:46 +0530 Subject: [PATCH 1/3] Add Colab-safe question bank integrity and filtering utility --- tools/colab_question_bank_review.py | 69 +++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 tools/colab_question_bank_review.py diff --git a/tools/colab_question_bank_review.py b/tools/colab_question_bank_review.py new file mode 100644 index 0000000..09832e7 --- /dev/null +++ b/tools/colab_question_bank_review.py @@ -0,0 +1,69 @@ +"""Review a question-only Servy benchmark JSON in Colab or locally. + +Does not access company databases, send network requests, run inference, or +treat hypothetical questions as verified ground truth. +""" +from __future__ import annotations + +import argparse +from collections import Counter +from hashlib import sha256 +import json +from pathlib import Path + + +def load_bank(path: Path) -> tuple[dict, list[dict]]: + payload = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict) or not isinstance(payload.get("questions"), list): + raise ValueError("Expected a JSON object containing a 'questions' list") + metadata = payload.get("metadata", {}) + questions = payload["questions"] + ids = set() + for n, item in enumerate(questions, start=1): + if not isinstance(item, dict): + raise ValueError(f"Question {n} is not an object") + for key in ("id", "product", "module", "question", "question_sha256", "asset_id", "coverage_status"): + if key not in item: + raise ValueError(f"Question {n} has no {key}") + if item["id"] in ids: + raise ValueError(f"Duplicate question ID: {item['id']}") + ids.add(item["id"]) + if not isinstance(item["question"], str) or not item["question"].strip(): + raise ValueError(f"Blank question: {item['id']}") + actual = sha256(item["question"].encode("utf-8")).hexdigest() + if item["question_sha256"] != actual: + raise ValueError(f"Full-text checksum mismatch: {item['id']}") + return metadata, questions + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--input", type=Path, required=True, help="Locally uploaded question-only JSON") + parser.add_argument("--asset-id", type=int, help="Filter to one asset (100 questions)") + parser.add_argument("--module", choices=["installation_and_commissioning", "preventive_maintenance", "troubleshooting"]) + parser.add_argument("--export", type=Path, help="Export the filtered question-only JSON; excludes actual answers") + parser.add_argument("--show", type=int, default=0, help="Show the first N full questions") + args = parser.parse_args() + + metadata, rows = load_bank(args.input) + totals = Counter((r["asset_id"], r["product"]) for r in rows) + print(f"Validated {len(rows)} unique questions with complete SHA-256 checksums") + for (asset_id, product), count in sorted(totals.items()): + print(f" {asset_id}: {product} — {count}") + print("WARNING: Questions are unreviewed probes, not certified answerable cases or model accuracy results.") + print("WARNING: Do not upload confidential manuals, answers, or company data to personal Colab.") + print("WARNING: Do not run on an asset without explicit authorization and approved source handling.") + + selected = [r for r in rows if (args.asset_id is None or r["asset_id"] == args.asset_id) + and (args.module is None or r["module"] == args.module)] + print(f"Selected {len(selected)} questions") + for item in selected[:max(0, args.show)]: + print(f"\n{item['id']} [{item['module']}, {item['coverage_status']}]\n{item['question']}") + if args.export: + args.export.write_text(json.dumps({"metadata": metadata, "questions": selected}, + ensure_ascii=False, indent=2), encoding="utf-8") + print(f"Exported {len(selected)} questions to {args.export}") + + +if __name__ == "__main__": + main() From 89dff57f53409172a8c490017c6985282963b847 Mon Sep 17 00:00:00 2001 From: yashaskn8 Date: Fri, 30 Oct 2026 01:59:50 +0530 Subject: [PATCH 2/3] Document Colab-safe question bank review and access safeguards --- tools/COLAB_QUESTION_BANK_REVIEW.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 tools/COLAB_QUESTION_BANK_REVIEW.md diff --git a/tools/COLAB_QUESTION_BANK_REVIEW.md b/tools/COLAB_QUESTION_BANK_REVIEW.md new file mode 100644 index 0000000..611f36d --- /dev/null +++ b/tools/COLAB_QUESTION_BANK_REVIEW.md @@ -0,0 +1,27 @@ +# Servy — Colab question-only review + +This tool only checks and filters a **locally uploaded, synthetic question-only JSON**. It does not connect to Servy's Django database, access private manuals, upload results, train models or calculate technical accuracy. + +## Use + +Obtain \`servy_colab_1200_questions.json\` separately from the person who generated it. The JSON must contain a top-level \`questions\` list with \`id\`, \`asset_id\`, \`product\`, \`module\`, \`question\`, \`question_sha256\`, and \`coverage_status\` for each record. + +Run (locally or in Google Colab after cloning this **public** GitHub repository and uploading only the question-only JSON): + +\`\`\`bash +python tools/colab_question_bank_review.py --input servy_colab_1200_questions.json +python tools/colab_question_bank_review.py --input servy_colab_1200_questions.json --asset-id 60 --show 5 +python tools/colab_question_bank_review.py --input servy_colab_1200_questions.json --asset-id 60 --module troubleshooting --export vrf_troubleshooting_questions.json +\`\`\` + +Each question's full SHA-256 is verified to detect the previous 200-character question-truncation error. + +## Scope and limitations + +The proposed bank covers 12 registered assets with 100 candidate questions per asset, split into installation/commissioning (30), maintenance (30), and troubleshooting (40). These are **hypothetical adversarial question prompts** based only on equipment names, not gold answers produced from company manuals. Based on the last local inventory, only assets 59, 60, and 61 had knowledge-base documents; most asset questions must therefore be handled as unsupported or pending documentation. + +**Do not use the 1,200 prompts as 1,200 verified answerable examples.** Validate ground truth against approved documentation. Obtain authorization for each tested customer/asset; the earlier selected scope covered asset 60 only. The test-labeled asset without a customer must not contribute to an equipment accuracy metric. Keep question banks used for training separate from held-out evaluation cases. + +**GitHub repository is public:** do not commit source-derived answer keys, proprietary documents, customer details, actual RAG responses, credentials or unredacted evaluation reports. Do not upload company data to personal Google Colab without explicit cloud-processing permission. + +The utility does not modify Servy's retrieval model or RAG behavior. Make and validate any future production changes against the latest local uncommitted source files, not a stale GitHub snapshot. From 8c861ca699d1cac8f6014a14c38c29573502ea84 Mon Sep 17 00:00:00 2001 From: yashaskn8 Date: Fri, 30 Oct 2026 02:00:25 +0530 Subject: [PATCH 3/3] Fix Markdown formatting in Colab question-bank instructions --- tools/COLAB_QUESTION_BANK_REVIEW.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/COLAB_QUESTION_BANK_REVIEW.md b/tools/COLAB_QUESTION_BANK_REVIEW.md index 611f36d..b3c66ba 100644 --- a/tools/COLAB_QUESTION_BANK_REVIEW.md +++ b/tools/COLAB_QUESTION_BANK_REVIEW.md @@ -4,15 +4,15 @@ This tool only checks and filters a **locally uploaded, synthetic question-only ## Use -Obtain \`servy_colab_1200_questions.json\` separately from the person who generated it. The JSON must contain a top-level \`questions\` list with \`id\`, \`asset_id\`, \`product\`, \`module\`, \`question\`, \`question_sha256\`, and \`coverage_status\` for each record. +Obtain `servy_colab_1200_questions.json` separately from the person who generated it. The JSON must contain a top-level `questions` list with `id`, `asset_id`, `product`, `module`, `question`, `question_sha256`, and `coverage_status` for each record. Run (locally or in Google Colab after cloning this **public** GitHub repository and uploading only the question-only JSON): -\`\`\`bash +```bash python tools/colab_question_bank_review.py --input servy_colab_1200_questions.json python tools/colab_question_bank_review.py --input servy_colab_1200_questions.json --asset-id 60 --show 5 python tools/colab_question_bank_review.py --input servy_colab_1200_questions.json --asset-id 60 --module troubleshooting --export vrf_troubleshooting_questions.json -\`\`\` +``` Each question's full SHA-256 is verified to detect the previous 200-character question-truncation error.