diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f0ebd420..7a8e2a36 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -132,10 +132,14 @@ jobs: run: | task_maven_repo=$(mktemp -d "$RUNNER_TEMP/ubnfc-parser.XXXXXX") echo "MAVEN_REPO=$task_maven_repo" >> "$GITHUB_ENV" + - name: Set up Rust 1.85 for the production facade comparison + run: bash .github/scripts/setup-rust.sh 1.85.0 # Declarative lexical modules require the pinned 3.3.0-SNAPSHOT generator; # published 3.1.1 cannot read this grammar. Build into the job-local repository. - name: Install unlaxer from source if it is not on Maven Central yet run: bash scripts/ci/install-unlaxer-if-unpublished.sh + env: + UNLAXER_PROVENANCE_FILE: ${{ github.workspace }}/target/fraud-acceptance/parser-dependency.json # Grammar sha256 + every vendored file against UBNFC_PIN. ubnfc is private, so the full # grammar -> IR -> Java regeneration runs where it is readable (UBNFC_DIR); see the script. - name: Verify strict layout compatibility adapter @@ -149,7 +153,28 @@ jobs: - name: Parity, differential fuzz and engine mode matrix run: >- mvn -B test -Dgpg.skip=true -Dtinyexpression.skipRailroad=true -Dmaven.repo.local="$MAVEN_REPO" - -Dtest=DeclarativeLexicalConformanceTest,UbnfcParityTest,UbnfcDifferentialFuzzTest,P4EngineModeMatrixTest,P4ParserEngineTest,P4DslJavaCodeCalculatorTest,P4DslJavaCodeCalculatorClassicEngineTest,P4AstEvaluatorCalculatorTest,P4AstEvaluatorCalculatorClassicEngineTest,P4PreferredAstMapperDiagnosticsTest,P4SourceMappingTest + -Dtest=DeclarativeLexicalConformanceTest,UbnfcParityTest,UbnfcDifferentialFuzzTest,P4EngineModeMatrixTest,P4ParserEngineTest,P4DslJavaCodeCalculatorTest,P4DslJavaCodeCalculatorClassicEngineTest,P4AstEvaluatorCalculatorTest,P4AstEvaluatorCalculatorClassicEngineTest,P4PreferredAstMapperDiagnosticsTest,P4SourceMappingTest,P4PackratFraudFormulaTest,FraudAcceptanceTest + - name: Require Rust production fraud acceptance and collector negative gates + env: + PYTHONDONTWRITEBYTECODE: '1' + run: | + cargo test --locked --manifest-path rust/Cargo.toml -p tinyexpression-rs --test fraud_acceptance -- --nocapture + python3 -m unittest discover -s scripts/tests -p test_fraud_production.py -v + python3 -m unittest discover -s scripts/tests -p test_unlaxer_provenance.py -v + - name: Require all 30 Java/Rust production observations + if: always() + run: | + python3 scripts/check-fraud-production.py --parser-m2 "$MAVEN_REPO" \ + --parser-provenance target/fraud-acceptance/parser-dependency.json \ + --output target/fraud-production-evidence + - uses: actions/upload-artifact@v6 + if: always() + with: + name: fraud-production-acceptance + path: | + target/fraud-production-evidence/ + target/fraud-acceptance/ + if-no-files-found: error - uses: actions/upload-artifact@v6 if: always() with: diff --git a/docs/quality/fraud-production-acceptance.md b/docs/quality/fraud-production-acceptance.md new file mode 100644 index 00000000..0b60f796 --- /dev/null +++ b/docs/quality/fraud-production-acceptance.md @@ -0,0 +1,140 @@ +# Fraud production acceptance and timing (Tiny #257) + +The previous `P4PackratFraudFormulaTest` caught every `RuntimeException` and accepted elapsed +values below **two** seconds while promising one second. A fast failure therefore passed. It +now requires successful parse+map, a non-null `IfExpr`, complete source CP range, valid nested +node ranges and unchanged AST across first-call/warm-up/warm calls. Every sample must take +strictly less than **1,000,000,000 ns**. Exceptions and failed assertions propagate; the 90-second +JUnit timeout is a hang guard, not an alternative performance threshold. + +Both languages read the unchanged five inputs from +`rust/tinyexpression-rs/tests/fixtures/fraud-alert.tsv`; the adjacent `fraud-acceptance.json` +pins input SHA-256, CP length and manually expected conditional counts. The comparison checks +numeric then/else branch ASTs independently (fraud 1–4: 1/0; fraud 5: then 1,1,5,5,5), followed +by complete cross-language AST/CP equality. Null, partial input, invalid span, altered branch, +missing/duplicate observations, wrong entry, cross-language differences and the exact one-second +boundary are negative gates. + +## Production entries and representation + +Java calls `P4PreferredAstMapper.parseDetailed(source, ExpressionTypes._float)` with a scoped +`P4ParserEngine.UBNFC` override. Rust calls public `tinyexpression_rs::parse(source)` once. Tests +add no alternate-entry parse/retry. The facades' existing internal root-selection behavior +remains production behavior, including the retries tracked separately in Tiny #256. + +Java selects the numeric `IfExpr`; Rust can return the enclosing `FormulaExpr`. The Rust test +retains its **raw AST** and selects the conditional for observation only. It accepts only +whole-input Formula wrappers without imports/declarations/methods/code blocks, Expression +wrappers, and Binary wrappers with an AST operand and empty operators/right operands. Anything +else fails. This projection is outside timing and changes no production parse. + +The other existing public shape difference is explicit in Rust `generated/compat.rs`: ubnfc's +synthetic operand `BinaryExpr(left=null, op=[text], right=[])` is folded to `AstValue::Text`. +Java retains that node. The Python comparator folds only this exact shape (including the +existing single-quote lexical normalization). Raw Java AST and raw/selected Rust AST remain +in the reports. All actual node CP spans are independently bounds-checked; the comparator +compares the spans of common nodes and does **not** claim identical publication of text-value +spans absent from Rust canonical JSON. No arbitrary source reparse or text fallback is used. + +Successful public facade calls expose no failure diagnostic. Reports explicitly use +`diagnostic:null`; this does not claim absence/equality of private speculative failures. Java +production currently asks ubnfc for diagnostics and occurrences, while Rust defaults both off; +reports retain that difference. Invalid/trailing input must produce an explicit public failure. +Memo is ubnfc's production default packrat in both cases. The Classic-only +`tinyexpression.p4.memoize` property is neither set nor cited as enabling this production gate. + +## Reproduce semantic and loaded-host regression checks + +Use Java 21 and Rust 1.85. Build against the intended parser artifact in an isolated Maven +repository, explicitly overriding Tiny's default dependency; do not install over a published +artifact in the ordinary user Maven repository. + +```sh +RUSTUP_TOOLCHAIN=1.85.0 mvn -B -Dmaven.repo.local=/path/to/isolated-m2 \ + -Dunlaxer.version=3.3.0 -Dtinyexpression.skipRailroad=true \ + -Dtest=P4PackratFraudFormulaTest,FraudAcceptanceTest test +RUSTUP_TOOLCHAIN=1.85.0 cargo test --manifest-path rust/Cargo.toml --locked \ + -p tinyexpression-rs --test fraud_acceptance -- --nocapture +RUSTUP_TOOLCHAIN=1.85.0 python3 -m unittest discover -s scripts/tests -p test_fraud_production.py -v +RUSTUP_TOOLCHAIN=1.85.0 python3 scripts/check-fraud-production.py \ + --parser-m2 /path/to/isolated-m2 --parser-commit FULL_40_HEX_PARSER_SHA \ + --output /path/to/evidence +``` + +The two tests truncate stale reports before starting and persist successful samples before +checking the time limit, so an over-budget observation is retained. The final comparator +requires all five inputs × three phases × two languages; incomplete/error runs cannot pass. +The output contains raw reports, source commit/dirty state, input identities, grammar/pin +hashes, exact parser JAR hashes, toolchain versions, native platform and verdict. Archive the +Maven/Cargo logs and dependency resolution alongside these files. Use a new output directory +for each run and preserve failed-run evidence. A dirty-tree record is development evidence; +repeat on the final clean source revision before citing its acceptance. + +`cold` means the **first call for this input in this test process**. Other tests in the same JVM or Rust test binary can already have parsed inputs, and later +inputs can use warmed runtime/compiler state. `pid` and `first_observation_in_process` identify +observations, without claiming that the parser or process was otherwise cold. `warmup` is a separate second +call; `warm` is the third call. Do not aggregate these phases or relabel all first-input calls +as fresh-process latency. Parsing and AST conversion are timed; validation, canonical JSON, +input I/O, report I/O, JVM/process startup and build time are excluded. Public Rust `parse` +includes its owned AST conversion; Java includes its facade selection/conversion. + +## Final performance acceptance protocol (not attested by these checks) + +This correction establishes actual acceptance and a one-second regression guard. Its report +always says `measurement_eligible:false`. It does not close parser #197/#184, prove Classic SAFE +performance, or replace those issues' root-selection/retry requirements. + +For final dedicated-host measurement: + +1. Pin the clean Tiny/parser commits, grammar and toolchains; build before timing, hash binaries + and dependency JARs, verify semantic checks, save exact commands/options and native resource + limits. Preserve the parser #652/#653 finite-budget fixes for any separate Classic study. +2. Reserve an idle host: no CI/builds/JFR/sampling profiler or competing benchmark. Record CPU, + memory, OS, governor/container/cgroup limits, runtime heap/stack flags, load before/after and + an explicit truthful exclusivity attestation. Shared-host runs stay exploratory evidence. +3. Run at least five fresh processes per input/language. To select one case in a fresh JVM use + `-Dtinyexpression.fraud.id=fraud#1 -Dtinyexpression.fraud.output=/absolute/java-1.jsonl` + with only `-Dtest=P4PackratFraudFormulaTest`; invoke each case in its own Maven/Surefire process. + For Rust use `FRAUD_ID=fraud#1 FRAUD_OUTPUT=/absolute/rust-1.jsonl` and run + `cargo test --manifest-path rust/Cargo.toml --locked -p tinyexpression-rs --test fraud_acceptance + fraud_formulas_are_accepted_and_mapped_within_one_second -- --exact --nocapture` with + `RUSTUP_TOOLCHAIN=1.85.0` (omit other tests so they cannot pre-warm the process). Repeat for ids 1–5. Run language processes sequentially, not concurrently. Retain distinct + PIDs, all first-call/warm-up/warm samples and failed samples; never choose only a fast run. +4. Compare complete AST/source-position/public-diagnostic observations before evaluating + latency. Require every measured parse+map call < 1 second; do not average away a miss or + raise it to two seconds. Report first-call and warm distributions independently, including + maxima and process-startup exclusion. Keep a hard process hang deadline and count timeout, + exception, partial/null result or unexpected diagnostic as failure. +5. Archive commands, environment, input/source/build hashes, raw times and semantic verdicts. + Separate ubnfc production, Classic SAFE and legacy evidence by backend/API. The ordinary + semantic comparator cannot promote records into a dedicated-host attestation. + +The protocol is preparation only. No dedicated/exclusive-host final measurement was performed +by this change. Concurrent parser CI/build activity existed during development verification. + +## Mandatory ordinary CI gate and dependency identity + +The existing `ubnfc-parser` job runs Java acceptance alongside its parity suite, Rust's production +fraud integration and the collector/provenance negative tests, then explicitly requires all 30 +observations. These steps have no `continue-on-error`; an exception, missing observation, +comparison difference or negative-gate failure makes the job fail. An `always()` collector step +writes a failed summary even when an earlier test failed, and an `always()` artifact upload +preserves `fraud-production-acceptance` raw reports, input/source/JAR hashes, provenance, +metadata and verdict. The existing Java Surefire artifact remains separate. CI evidence remains +shared-host verification with `measurement_eligible:false`. + +This job's dependency route is **Tiny's declared Maven version**, currently `3.3.0-SNAPSHOT`, +not the locally tested candidate `3.3.0`. Its existing installer builds the fixed +`.github/unlaxer-source-pin` (`fc75fe1e25e538d47908f05ea2039f585aef696e`) into an empty job-local +Maven repository when that version is unavailable on Central. The installer's optional +`UNLAXER_PROVENANCE_FILE` records the actual checked-out commit after successful installation. +If the declared version becomes available on Central, it instead records `maven-central` with +no inferred source commit. Failed installation clears stale success provenance. The collector +uses the recorded version to hash the actual dependency JARs; it never substitutes the local +candidate SHA or assumes the source pin produced a published artifact. + +Local runs can keep `--parser-commit FULL_SHA --parser-version 3.3.0` (the latter defaults to +3.3.0); those records are explicitly labeled `reported-local-source`. CI instead passes +`--parser-provenance target/fraud-acceptance/parser-dependency.json`. The two representations +are mutually exclusive, so evidence cannot silently label a published GAV as a source build. +No parser version, source pin, ubnfc generation pin or production behavior changes in this wiring. diff --git a/rust/tinyexpression-rs/tests/fixtures/fraud-acceptance.json b/rust/tinyexpression-rs/tests/fixtures/fraud-acceptance.json new file mode 100644 index 00000000..55b1e02c --- /dev/null +++ b/rust/tinyexpression-rs/tests/fixtures/fraud-acceptance.json @@ -0,0 +1,37 @@ +[ + { + "id": "fraud#1", + "sha256": "3b69c9eb95cdea2ba20197530b042791b437d2c2fee05bc3b684308a5da7711c", + "cp": 404, + "if_count": 1, + "root": "IfExpr" + }, + { + "id": "fraud#2", + "sha256": "065b7e78a0cbc470f945da584b8a36a34c2ae1a4de4da399e6ac49e444de3163", + "cp": 209, + "if_count": 1, + "root": "IfExpr" + }, + { + "id": "fraud#3", + "sha256": "3917e9283333ef813cd17de5f9b6a26cac6cef97bda148d1b0928fce24556c47", + "cp": 68, + "if_count": 1, + "root": "IfExpr" + }, + { + "id": "fraud#4", + "sha256": "1167d25450f1b142b86e6891394419ed29c818fc39b397e8ab67cf03bf4e137d", + "cp": 707, + "if_count": 1, + "root": "IfExpr" + }, + { + "id": "fraud#5", + "sha256": "4bcdc00f795d398d678a7ba7f35e49ca53de7ae9702892f8b917711917a56c35", + "cp": 459, + "if_count": 5, + "root": "IfExpr" + } +] diff --git a/rust/tinyexpression-rs/tests/fraud_acceptance.rs b/rust/tinyexpression-rs/tests/fraud_acceptance.rs new file mode 100644 index 00000000..e7275a8c --- /dev/null +++ b/rust/tinyexpression-rs/tests/fraud_acceptance.rs @@ -0,0 +1,113 @@ +//! Production facade acceptance and strict sub-second regression guard (Tiny #257). +//! This is ubnfc production evidence, not unlaxer Classic SAFE calibration. +use std::fmt::Write; +use std::fs; +use std::time::Instant; +use tinyexpression_rs::generated::ast::AstValue; +use tinyexpression_rs::{parse, Ast, FrontendError, Span}; + +const INPUTS: &str = include_str!("fixtures/fraud-alert.tsv"); +const LENGTHS: [usize; 5] = [404, 209, 68, 707, 459]; + +fn conditional_view<'a>(source: &str, raw: &'a Ast) -> &'a Ast { + let full = Span { + start: 0, + end: source.chars().count(), + }; + let mut ast = raw; + loop { + assert_eq!( + ast.span(), + full, + "transparent wrapper must own the complete input" + ); + ast = match ast { + Ast::IfExpr { .. } => return ast, + Ast::FormulaExpr { imports, declarations, expression, methods, codeBlocks, .. } + if imports.is_empty() && declarations.is_empty() && methods.is_empty() && codeBlocks.is_empty() => expression, + Ast::ExpressionExpr { value, .. } => value, + Ast::BinaryExpr { left: AstValue::Node(node), op, right, .. } + if op.is_empty() && right.is_empty() => node, + _ => panic!("production result is not a complete conditional or its documented transparent wrappers"), + }; + } +} + +#[test] +fn fraud_formulas_are_accepted_and_mapped_within_one_second() { + let output = std::env::var_os("FRAUD_OUTPUT") + .map(std::path::PathBuf::from) + .unwrap_or_else(|| { + std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../target/fraud-acceptance/rust.jsonl") + }); + fs::create_dir_all(output.parent().unwrap()).unwrap(); + // Truncate first: stale green output must not survive a failed run. + fs::write(&output, "").unwrap(); + let inputs: Vec<_> = INPUTS + .lines() + .filter(|line| !line.is_empty() && !line.starts_with('#')) + .collect(); + assert_eq!(inputs.len(), 5); + let selected_id = std::env::var("FRAUD_ID").ok(); + if let Some(id) = &selected_id { + assert!( + inputs + .iter() + .any(|line| line.starts_with(&format!("{id}\t"))), + "unknown fraud id" + ); + } + let mut records = String::new(); + for (index, row) in inputs.into_iter().enumerate() { + let (id, source) = row.split_once('\t').expect("id/formula"); + if selected_id + .as_deref() + .is_some_and(|selected| selected != id) + { + continue; + } + assert_eq!(source.chars().count(), LENGTHS[index]); + let mut previous = None; + for phase in ["cold", "warmup", "warm"] { + // Serialization, validation and I/O do not enter parse+map timing. + let start = Instant::now(); + let ast = parse(source).expect("fraud production parse+map must succeed"); + let elapsed = start.elapsed().as_nanos(); + let selected = conditional_view(source, &ast); + let canonical = selected.canonical_json(); + if let Some(previous) = &previous { + assert_eq!(previous, &canonical, "phase changed AST/spans"); + } + previous = Some(canonical.clone()); + let first_observation_in_process = records.is_empty(); + let pid = std::process::id(); + let input_hex = source + .as_bytes() + .iter() + .fold(String::new(), |mut out, byte| { + write!(out, "{byte:02x}").unwrap(); + out + }); + records.push_str(&format!( + "{{\"id\":\"{id}\",\"phase\":\"{phase}\",\"backend\":\"rust-ubnfc\",\"pid\":{pid},\"first_observation_in_process\":{first_observation_in_process},\"entry\":\"tinyexpression_rs::parse\",\"memo\":\"ubnfc-default-packrat\",\"input_utf8_hex\":\"{input_hex}\",\"observations\":{{\"diagnostics\":false,\"occurrences\":false}},\"elapsed_ns\":{elapsed},\"cp\":{},\"accepted\":true,\"diagnostic\":null,\"ast\":{canonical},\"raw_ast\":{}}}\n", LENGTHS[index], ast.canonical_json())); + fs::write(&output, &records).unwrap(); + eprintln!("fraud production {id} {phase} {elapsed} ns"); + assert!( + elapsed < 1_000_000_000, + "{id} {phase}: expected < 1 second, got {elapsed} ns" + ); + } + } +} + +#[test] +fn rejected_and_partial_inputs_cannot_count_as_fast_success() { + let source = "if(true){1}else{0}"; + conditional_view(source, &parse(source).unwrap()); + assert!(matches!( + parse(&format!("{source}@")), + Err(FrontendError::Parse(_)) + )); + assert!(matches!(parse("@"), Err(FrontendError::Parse(_)))); +} diff --git a/scripts/check-fraud-production.py b/scripts/check-fraud-production.py new file mode 100644 index 00000000..ede6f191 --- /dev/null +++ b/scripts/check-fraud-production.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python3 +"""Compare actual production facade observations, never turn quick failure into success.""" +import argparse +import hashlib +import json +from pathlib import Path +import platform +import re +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] +LENGTHS = [404, 209, 68, 707, 459] +PHASES = {"cold", "warmup", "warm"} +LIMIT_NS = 1_000_000_000 + + +def cases(): + expected = json.loads((ROOT / "rust/tinyexpression-rs/tests/fixtures/fraud-acceptance.json").read_text()) + rows = [] + for line in (ROOT / "rust/tinyexpression-rs/tests/fixtures/fraud-alert.tsv").read_text().splitlines(): + if line and not line.startswith("#"): + identifier, source = line.split("\t") + rows.append({"id": identifier, "source": source, "sha256": hashlib.sha256(source.encode()).hexdigest(), + "cp": len(source), "if_count": 5 if identifier == "fraud#5" else 1}) + if len(expected) != 5 or len(rows) != 5 or [r["cp"] for r in rows] != LENGTHS: + raise ValueError("original five fraud inputs changed") + for row, pinned in zip(rows, expected): + if any(row[key] != pinned[key] for key in ("id", "sha256", "cp", "if_count")): + raise ValueError("pinned fraud identity changed") + return {row["id"]: row for row in rows} + + +def nodes(value): + if isinstance(value, dict): + if isinstance(value.get("type"), str) and "fields" in value: + yield value + for child in value.values(): + yield from nodes(child) + elif isinstance(value, list): + for child in value: + yield from nodes(child) + + +def comparison_view(value): + """Only the pre-existing Rust compat.rs synthetic operand conversion; no source fallback.""" + if isinstance(value, list): + return [comparison_view(child) for child in value] + if not isinstance(value, dict): + return value + fields = value.get("fields") + if value.get("type") == "BinaryExpr" and isinstance(fields, dict): + if fields.get("left") is None and fields.get("right") == []: + text = fields.get("op") + if isinstance(text, list) and len(text) == 1 and isinstance(text[0], str): + return text[0][1:-1] if text[0].startswith("'") and text[0].endswith("'") else text[0] + return {key: comparison_view(child) for key, child in value.items()} + + +def branch_literal(node): + while isinstance(node, dict): + fields = node["fields"] + if node["type"] == "BranchExpressionExpr": + node = fields["value"] + elif node["type"] == "BinaryExpr" and fields["op"] == [] and fields["right"] == []: + node = fields["left"] + else: + break + return node + + +def selected_conditional(raw, cp): + node = raw + while isinstance(node, dict) and node.get("span") == [0, cp]: + kind, fields = node.get("type"), node.get("fields", {}) + if kind == "IfExpr": + return node + if kind == "FormulaExpr" and all(fields.get(key) == [] for key in ("imports", "declarations", "methods", "codeBlocks")): + node = fields.get("expression") + elif kind == "ExpressionExpr": + node = fields.get("value") + elif kind == "BinaryExpr" and fields.get("op") == [] and fields.get("right") == []: + node = fields.get("left") + else: + break + raise ValueError("invalid raw AST conditional projection") + + +def validate_row(row, case, backend): + if row.get("backend") != backend or row.get("memo") != "ubnfc-default-packrat": + raise ValueError("wrong production backend/memo policy") + entry = "P4PreferredAstMapper.parseDetailed(_float)" if backend == "java-ubnfc" else "tinyexpression_rs::parse" + if row.get("entry") != entry: + raise ValueError("wrong production entry") + if row.get("accepted") is not True or "diagnostic" not in row or row["diagnostic"] is not None: + raise ValueError("not an explicit diagnostic-free success") + actual_hash = row.get("sha256") + if "input_utf8_hex" in row: + actual_hash = hashlib.sha256(bytes.fromhex(row["input_utf8_hex"])).hexdigest() + if row.get("cp") != case["cp"] or actual_hash != case["sha256"]: + raise ValueError("input identity mismatch") + elapsed = row.get("elapsed_ns") + if type(elapsed) is not int or not 0 <= elapsed < LIMIT_NS: + raise ValueError("parse+map did not complete in strictly less than one second") + ast = row.get("ast") + if not isinstance(ast, dict) or ast.get("type") != "IfExpr" or ast.get("span") != [0, case["cp"]]: + raise ValueError("null, partial or wrong AST root") + if backend == "rust-ubnfc" and selected_conditional(row.get("raw_ast"), case["cp"]) != ast: + raise ValueError("selected conditional differs from raw production AST") + for tree in [ast, row.get("raw_ast", ast)]: + for node in nodes(tree): + span = node.get("span") + if (not isinstance(span, list) or len(span) != 2 or any(type(n) is not int for n in span) + or not 0 <= span[0] <= span[1] <= case["cp"]): + raise ValueError("invalid code-point node span") + view = comparison_view(ast) + conditionals = [node for node in nodes(view) if node["type"] == "IfExpr"] + if len(conditionals) != case["if_count"]: + raise ValueError("conditional structure changed") + expected_then = ["1", "1", "5", "5", "5"] if case["id"] == "fraud#5" else ["1"] + if [branch_literal(node["fields"]["thenExpr"]) for node in conditionals] != expected_then: + raise ValueError("numeric branch AST changed") + if case["id"] != "fraud#5" and branch_literal(view["fields"]["elseExpr"]) != "0": + raise ValueError("numeric else AST changed") + return view + + +def compare(java, rust, corpus): + observed = [] + for backend, rows in [("java-ubnfc", java), ("rust-ubnfc", rust)]: + keyed = {} + for row in rows: + key = row.get("id"), row.get("phase") + if key in keyed or key[0] not in corpus or key[1] not in PHASES: + raise ValueError("duplicate or unknown observation") + keyed[key] = validate_row(row, corpus[key[0]], backend) + expected = {(identifier, phase) for identifier in corpus for phase in PHASES} + if set(keyed) != expected: + raise ValueError("missing production observation") + for identifier in corpus: + if not all(keyed[identifier, phase] == keyed[identifier, "cold"] for phase in PHASES): + raise ValueError("AST/spans changed across phases") + observed.append(keyed) + if observed[0] != observed[1]: + raise ValueError("Java/Rust production AST or source spans differ") + return {"status": "passed", "observations": len(java) + len(rust), "measurement_eligible": False, + "timing_kind": "shared-host-regression", "limit_ns": LIMIT_NS} + + +def command(*args): + return subprocess.check_output(args, cwd=ROOT, text=True, stderr=subprocess.STDOUT).strip() + + +def dependency_provenance(version, commit, provenance_path): + if provenance_path is not None: + value = json.loads(provenance_path.read_text()) + if not isinstance(value, dict) or value.get("origin") not in {"pinned-source", "maven-central"}: + raise ValueError("unknown parser dependency origin") + else: + value = {"version": version, "origin": "reported-local-source", "source_commit": commit} + if not isinstance(value.get("version"), str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._+-]*", value["version"]): + raise ValueError("invalid parser Maven version") + actual_commit = value.get("source_commit") + if value["origin"] == "maven-central": + if actual_commit is not None: + raise ValueError("published Maven artifact must not claim an unverified source commit") + elif not isinstance(actual_commit, str) or not re.fullmatch(r"[0-9a-f]{40}", actual_commit): + raise ValueError("parser commit must be a full lowercase hexadecimal SHA") + return value + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--java-report", type=Path, default=ROOT / "target/fraud-acceptance/java.jsonl") + parser.add_argument("--rust-report", type=Path, default=ROOT / "target/fraud-acceptance/rust.jsonl") + parser.add_argument("--output", type=Path, required=True) + provenance = parser.add_mutually_exclusive_group(required=True) + provenance.add_argument("--parser-commit", help="reported local source SHA installed into the isolated Maven repository") + provenance.add_argument("--parser-provenance", type=Path, help="actual CI installer provenance (source pin or Maven Central)") + parser.add_argument("--parser-version", default="3.3.0", help="local candidate Maven version, used with --parser-commit") + parser.add_argument("--parser-m2", type=Path, required=True, help="isolated Maven repo used for the Java build") + args = parser.parse_args() + args.output.mkdir(parents=True, exist_ok=True) + summary = {"status": "failed", "measurement_eligible": False} + metadata = {} + try: + dependency = dependency_provenance(args.parser_version, args.parser_commit, args.parser_provenance) + corpus = cases() + metadata = {"commit": command("git", "rev-parse", "HEAD"), + "parser_dependency": dependency, + "dirty": bool(command("git", "status", "--porcelain")), + "platform": platform.platform(), "hostname": platform.node(), "java": command("java", "-version"), + "rustc": command("rustc", "--version"), "maven": command("mvn", "-version"), + "concurrent_load": "uncontrolled; not exclusive-host attestation", + "cold_definition": "first observed call for this input; normal test-suite execution does not guarantee process-cold state", + "cases": list(corpus.values()), "artifacts": {}} + for artifact in ("unlaxer-common", "unlaxer-dsl"): + version = dependency["version"] + path = args.parser_m2 / "org/unlaxer" / artifact / version / f"{artifact}-{version}.jar" + metadata["artifacts"][artifact] = hashlib.sha256(path.read_bytes()).hexdigest() + for path in ["src/main/java/org/unlaxer/tinyexpression/p4/ubnfc/UBNFC_PIN", "rust/ubnfc-pin.txt", + "tools/tinyexpression-p4-lsp-vscode/grammar/tinyexpression-p4.ubnf"]: + metadata.setdefault("source_identities", {})[path] = hashlib.sha256((ROOT / path).read_bytes()).hexdigest() + reports = [] + for backend, path in [("java", args.java_report), ("rust", args.rust_report)]: + raw = path.read_text() + (args.output / f"{backend}.jsonl").write_text(raw) + reports.append([json.loads(line) for line in raw.splitlines()]) + summary = compare(*reports, corpus) + except (ValueError, KeyError, TypeError, OSError, subprocess.CalledProcessError) as failure: + summary["error"] = str(failure) + (args.output / "metadata.json").write_text(json.dumps(metadata, ensure_ascii=False, indent=2) + "\n") + (args.output / "summary.json").write_text(json.dumps(summary, indent=2) + "\n") + print(json.dumps(summary)) + return 0 if summary["status"] == "passed" else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ci/install-unlaxer-if-unpublished.sh b/scripts/ci/install-unlaxer-if-unpublished.sh index d6fecbd3..6f53fd67 100755 --- a/scripts/ci/install-unlaxer-if-unpublished.sh +++ b/scripts/ci/install-unlaxer-if-unpublished.sh @@ -12,6 +12,9 @@ set -euo pipefail repo=$(cd "$(dirname "$0")/../.." && pwd) : "${MAVEN_REPO:?set MAVEN_REPO to the job-local Maven repository}" +if [[ -n "${UNLAXER_PROVENANCE_FILE:-}" ]]; then + rm -f -- "$UNLAXER_PROVENANCE_FILE" # an unsuccessful rerun must not retain old success provenance +fi version=$(sed -n 's:.*\(.*\).*:\1:p' "$repo/pom.xml" | head -1) if [[ -z "$version" ]]; then @@ -19,6 +22,24 @@ if [[ -z "$version" ]]; then exit 1 fi +# Optional machine-readable provenance for consumers comparing the actual CI dependency. +# Other callers keep their existing behavior when this variable is unset. +record_provenance() { + if [[ -z "${UNLAXER_PROVENANCE_FILE:-}" ]]; then + return + fi + python3 - "$UNLAXER_PROVENANCE_FILE" "$version" "$1" "${2:-}" <<'PYPROVENANCE' +import json +from pathlib import Path +import sys +path, version, origin, commit = sys.argv[1:] +output = Path(path) +output.parent.mkdir(parents=True, exist_ok=True) +output.write_text(json.dumps({"version": version, "origin": origin, + "source_commit": commit or None}, indent=2) + "\n") +PYPROVENANCE +} + central=https://repo1.maven.org/maven2/org/unlaxer published=true for artifact in unlaxer-common unlaxer-dsl; do @@ -28,6 +49,7 @@ for artifact in unlaxer-common unlaxer-dsl; do done if $published; then echo "unlaxer $version is on Maven Central; nothing to install." + record_provenance maven-central exit 0 fi @@ -50,6 +72,7 @@ fi echo "unlaxer $version is not on Maven Central; installing it from unlaxer-parser $commit" mvn -B -q -f "$work/pom.xml" -pl .,unlaxer-common,unlaxer-dsl \ install -DskipTests -Dgpg.skip=true -Dmaven.repo.local="$MAVEN_REPO" +record_provenance pinned-source "$(git -C "$work" rev-parse HEAD)" if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then echo "unlaxer $version: not on Maven Central yet, built from unlaxer-parser \`$commit\`" >> "$GITHUB_STEP_SUMMARY" fi diff --git a/scripts/tests/test_fraud_production.py b/scripts/tests/test_fraud_production.py new file mode 100644 index 00000000..c71e0db3 --- /dev/null +++ b/scripts/tests/test_fraud_production.py @@ -0,0 +1,98 @@ +import copy +import importlib.util +from pathlib import Path +import unittest +import json +import tempfile + +spec = importlib.util.spec_from_file_location("fraud", Path(__file__).parents[1] / "check-fraud-production.py") +fraud = importlib.util.module_from_spec(spec) +spec.loader.exec_module(fraud) + + +def leaf(text): + return {"type": "BinaryExpr", "span": [1, 2], "fields": {"left": None, "op": [text], "right": []}} + + +def ast(): + return {"type": "IfExpr", "span": [0, 5], "fields": { + "condition": "true", "thenExpr": leaf("1"), "elseExpr": leaf("0")}} + + +class ProductionEvidenceTest(unittest.TestCase): + def setUp(self): + self.case = {"id": "fraud#1", "cp": 5, "sha256": "input-hash", "if_count": 1} + self.row = {"id": "fraud#1", "phase": "cold", "backend": "java-ubnfc", + "entry": "P4PreferredAstMapper.parseDetailed(_float)", "memo": "ubnfc-default-packrat", + "sha256": "input-hash", "elapsed_ns": 10, "cp": 5, "accepted": True, "diagnostic": None, "ast": ast()} + + def test_one_second_is_strict_and_no_negative_duration(self): + for elapsed in [-1, 1_000_000_000, 2_000_000_000, None, True]: + with self.subTest(elapsed=elapsed), self.assertRaises(ValueError): + fraud.validate_row(dict(self.row, elapsed_ns=elapsed), self.case, "java-ubnfc") + + def test_failure_null_partial_wrong_span_and_wrong_branch_fail(self): + for change in [{"accepted": False}, {"diagnostic": {}}, {"ast": None}, {"cp": 4}]: + with self.subTest(change=change), self.assertRaises(ValueError): + fraud.validate_row(dict(self.row, **change), self.case, "java-ubnfc") + for change in ["root", "nested", "branch"]: + row = copy.deepcopy(self.row) + if change == "root": row["ast"]["span"] = [0, 4] + if change == "nested": row["ast"]["fields"]["thenExpr"]["span"] = [1, 6] + if change == "branch": row["ast"]["fields"]["thenExpr"]["fields"]["op"] = ["9"] + with self.subTest(change=change), self.assertRaises(ValueError): + fraud.validate_row(row, self.case, "java-ubnfc") + + def test_compat_projection_only_folds_exact_synthetic_operand(self): + self.assertEqual("1", fraud.comparison_view(leaf("1"))) + wrong = leaf("1") + wrong["fields"]["op"] = ["1", "+"] + self.assertIsInstance(fraud.comparison_view(wrong), dict) + + def test_raw_projection_cannot_hide_declarations_or_a_different_ast(self): + valid = {"type": "FormulaExpr", "span": [0, 5], "fields": { + "imports": [], "declarations": [], "methods": [], "codeBlocks": [], "expression": ast()}} + self.assertEqual(ast(), fraud.selected_conditional(valid, 5)) + valid["fields"]["declarations"] = ["unexpected"] + with self.assertRaises(ValueError): fraud.selected_conditional(valid, 5) + row = dict(self.row, backend="rust-ubnfc", entry="tinyexpression_rs::parse", raw_ast=None) + with self.assertRaises(ValueError): fraud.validate_row(row, self.case, "rust-ubnfc") + + def test_ci_provenance_distinguishes_snapshot_pin_from_published_and_local(self): + commit = "a" * 40 + self.assertEqual("reported-local-source", fraud.dependency_provenance("3.3.0", commit, None)["origin"]) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "dependency.json" + for value in [ + {"version": "3.3.0-SNAPSHOT", "origin": "pinned-source", "source_commit": commit}, + {"version": "3.3.0", "origin": "maven-central", "source_commit": None}, + ]: + path.write_text(json.dumps(value)) + self.assertEqual(value, fraud.dependency_provenance("ignored", None, path)) + for value in [ + [], + {"version": "../3.3.0", "origin": "pinned-source", "source_commit": commit}, + {"version": "3.3.0", "origin": "maven-central", "source_commit": commit}, + {"version": "3.3.0", "origin": "pinned-source", "source_commit": "short"}, + {"version": "3.3.0", "origin": "unknown", "source_commit": None}, + ]: + path.write_text(json.dumps(value)) + with self.subTest(value=value), self.assertRaises(ValueError): + fraud.dependency_provenance("ignored", None, path) + + def test_missing_duplicate_and_cross_language_difference_fail(self): + java = [dict(self.row, phase=phase) for phase in sorted(fraud.PHASES)] + rust = [dict(row, backend="rust-ubnfc", entry="tinyexpression_rs::parse", raw_ast=row["ast"]) for row in java] + corpus = {"fraud#1": self.case} + self.assertEqual("passed", fraud.compare(java, rust, corpus)["status"]) + for rows in [rust[:2], rust + rust[:1]]: + with self.assertRaises(ValueError): fraud.compare(java, rows, corpus) + changed = copy.deepcopy(rust) + for row in changed: + row["ast"]["fields"]["condition"] = "false" + row["raw_ast"] = row["ast"] + with self.assertRaises(ValueError): fraud.compare(java, changed, corpus) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_unlaxer_provenance.py b/scripts/tests/test_unlaxer_provenance.py new file mode 100644 index 00000000..a50940ba --- /dev/null +++ b/scripts/tests/test_unlaxer_provenance.py @@ -0,0 +1,75 @@ +"""Exercise installer provenance branches without network/Maven or shared repository writes.""" +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import unittest + +SCRIPT = Path(__file__).resolve().parents[1] / "ci/install-unlaxer-if-unpublished.sh" +PIN = "f" * 40 + + +class InstallerProvenanceTest(unittest.TestCase): + def run_installer(self, central, failed=False): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + repo = root / "project" + script = repo / "scripts/ci" / SCRIPT.name + script.parent.mkdir(parents=True) + shutil.copy2(SCRIPT, script) + (repo / ".github").mkdir() + (repo / ".github/unlaxer-source-pin").write_text(f"commit={PIN}\n") + version = "3.3.0" if central else "3.3.0-SNAPSHOT" + (repo / "pom.xml").write_text(f"{version}\n") + stubs = root / "bin" + stubs.mkdir() + stub = f'''#!{sys.executable} +import os +from pathlib import Path +import sys +name=Path(sys.argv[0]).name +args=sys.argv[1:] +if name == 'curl': sys.exit(0 if os.environ['MOCK_CENTRAL'] == 'yes' else 22) +if name == 'mvn': sys.exit(int(os.environ['MOCK_MVN_EXIT'])) +if name == 'git': + if args[0] == 'init': Path(args[-1]).mkdir(parents=True,exist_ok=True) + elif args[2] == 'fetch': (Path(args[1])/'pom.xml').write_text('3.3.0-SNAPSHOT') + elif args[2] == 'rev-parse': print('{PIN}') +''' + for name in ("curl", "git", "mvn"): + path = stubs / name + path.write_text(stub) + path.chmod(0o755) + runner_temp = root / "runner" + runner_temp.mkdir() + provenance = root / "evidence/dependency.json" + provenance.parent.mkdir(parents=True) + provenance.write_text('{"origin":"stale-success"}\n') + environment = dict(os.environ, PATH=str(stubs) + os.pathsep + os.environ["PATH"], + MAVEN_REPO=str(root / "m2"), RUNNER_TEMP=str(runner_temp), + UNLAXER_PROVENANCE_FILE=str(provenance), MOCK_CENTRAL="yes" if central else "no", + MOCK_MVN_EXIT="1" if failed else "0", GITHUB_STEP_SUMMARY="") + result = subprocess.run(["bash", str(script)], env=environment, capture_output=True, text=True) + return result.returncode, json.loads(provenance.read_text()) if provenance.exists() else None + + def test_published_artifacts_do_not_claim_the_source_pin(self): + code, provenance = self.run_installer(True) + self.assertEqual(0, code) + self.assertEqual({"version": "3.3.0", "origin": "maven-central", "source_commit": None}, provenance) + + def test_unpublished_snapshot_records_actual_checked_out_commit(self): + code, provenance = self.run_installer(False) + self.assertEqual(0, code) + self.assertEqual({"version": "3.3.0-SNAPSHOT", "origin": "pinned-source", "source_commit": PIN}, provenance) + + def test_failed_install_cannot_publish_success_provenance(self): + code, provenance = self.run_installer(False, failed=True) + self.assertNotEqual(0, code) + self.assertIsNone(provenance) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/test/java/org/unlaxer/tinyexpression/evaluator/p4/P4PackratFraudFormulaTest.java b/src/test/java/org/unlaxer/tinyexpression/evaluator/p4/P4PackratFraudFormulaTest.java index be3d8762..579c0498 100644 --- a/src/test/java/org/unlaxer/tinyexpression/evaluator/p4/P4PackratFraudFormulaTest.java +++ b/src/test/java/org/unlaxer/tinyexpression/evaluator/p4/P4PackratFraudFormulaTest.java @@ -1,62 +1,17 @@ package org.unlaxer.tinyexpression.evaluator.p4; -import static org.junit.Assert.assertTrue; - import org.junit.Test; -import org.unlaxer.tinyexpression.p4.P4PreferredAstMapper; -import org.unlaxer.tinyexpression.parser.ExpressionTypes; +import org.unlaxer.tinyexpression.p4.ubnfc.FraudAcceptance; /** - * End-to-end check for unlaxer-parser #40 (packrat memoization) on the real fraud-detection - * formulas from issue #19 — the deeply nested boolean/if expressions that previously hung the - * generated P4 parser for 53+ minutes via exponential backtracking. - * - *

With {@code -Dtinyexpression.p4.memoize=true} (wired into {@link P4PreferredAstMapper}), each - * formula must parse in well under a second. The {@link Test#timeout()} is the hard guard: if - * memoization regressed, the exponential returns and the test times out rather than passing. + * Current production ubnfc facade: the original five fraud formulas must actually parse and map + * to complete conditional ASTs in strictly less than one second per call. Exceptions fail. + * Cold (first call for this input), warm-up and warm samples are reported separately. + * This is a loaded-host regression guard, not a dedicated-host Classic SAFE performance attestation. */ public class P4PackratFraudFormulaTest { - - private static final String[] FRAUD_FORMULAS = { - "if((isPresent($countryCode)&$countryCode!=\"JP\")&((isPresent($osGroup)&toLowerCase($osGroup).in(\"ios\"))&(isPresent($browserGroup)&toLowerCase($browserGroup).contains(\"safari\")))&(((isPresent($timezone)&$timezone=='+9')&(isPresent($priorityLanguage)¬($priorityLanguage.contains('ja'))))|((isPresent($timezone)&$timezone!='+9')&(isPresent($priorityLanguage)&$priorityLanguage.contains('ja'))))){1}else{0}", - "if((isPresent($calculated_BlackIPAddressInOtherSites)&$calculated_BlackIPAddressInOtherSites>0.0)|(isPresent($calculated_BlackCaulisCookieInOtherSites)&$calculated_BlackCaulisCookieInOtherSites>0.0)){1}else{0}", - "if(isPresent($calculated_TorNode)&$calculated_TorNode>0.0){1}else{0}", - "if((isPresent($userCountGroupedByCookieOnThisSite)&$userCountGroupedByCookieOnThisSite>=2)&((isPresent($os)&(not(toLowerCase($os).contains(\"linux\"))|not(toLowerCase($os).contains(\"Fire OS\"))))|(isPresent($number_accountCreationCountByIpAddress)&isPresent($userCountGroupedByCookieOnThisSite)¬($number_accountCreationCountByIpAddress - $userCountGroupedByCookieOnThisSite>=1))|(isPresent($userCountGroupedByCookieOnAllSite)&isPresent($userCountGroupedByCookieOnThisSite)&isPresent($userCountGroupedByCookieOnThisSiteOn12H)&(not($userCountGroupedByCookieOnAllSite - $userCountGroupedByCookieOnThisSite>=1)¬($userCountGroupedByCookieOnThisSite - $userCountGroupedByCookieOnThisSiteOn12H==0))))){1}else{0}", - "if(not(isPresent($calculated_FirstAccessUserHash))){1}else{if($ForcedRelativeSuspiciousValue1){1}else{if($ForcedRelativeSuspiciousValue5){5}else{if($default_RelativeSuspiciousValue==5){5}else{if(($POST_PROCESS_OriginalSpec_CountryIsNotJapan>0.0)|($POST_PROCESS_OriginalSpec_BlackListOnOtherSites>0.0)|($POST_PROCESS_OriginalSpec_SuspiciousProvider>0.0)|($POST_PROCESS_OriginalSpec_OneUserAccessToMultiAccount>0.0)){5}else{$default_RelativeSuspiciousValue}}}}}" - }; - - private long parseMillis(String formula) { - long start = System.nanoTime(); - try { - P4PreferredAstMapper.parse(formula, ExpressionTypes._float); - } catch (RuntimeException tolerated) { - // The point of this test is parse TIME, not the parse/typing verdict; a fast failure is fine. - } - return (System.nanoTime() - start) / 1_000_000; - } - - /** - * Formulas #1–4 are the boolean / parenthesis-ambiguity shape whose exponential backtracking - * packrat memoization collapses — they parse in well under a second (measured ~0.02–0.5s vs the - * 53-min hang in #19). Formula #5 is the deeply nested-{@code if} shape. Packrat (parse-phase) - * memoization alone left it at 12–30s; #49 found the real bottleneck was the mapping phase, not - * parsing: {@code findBestMappedToken} re-constructed the same subtrees ~10M times (each via - * per-token reflection + an unbounded {@code IdentityHashMap} of source spans). Memoizing - * {@code mapToken} by token identity and dropping the reflection (unlaxer-dsl MapperGenerator, - * 3.0.11) brings #5 to ~30ms — so it is now held to the same strict sub-second bound as the rest. - */ @Test(timeout = 90_000) - public void fraudFormulasParseFastWithMemoization() { - System.setProperty("tinyexpression.p4.memoize", "true"); - try { - for (int index = 0; index < FRAUD_FORMULAS.length; index++) { - long elapsed = parseMillis(FRAUD_FORMULAS[index]); - System.out.println("fraud formula #" + (index + 1) + " parsed in " + elapsed + "ms (memoize on)"); - assertTrue("formula #" + (index + 1) + " should parse in well under a second with " - + "memoization (was " + elapsed + "ms)", elapsed < 2_000); - } - } finally { - System.clearProperty("tinyexpression.p4.memoize"); - } + public void fraudFormulasParseFastWithMemoization() throws Exception { + FraudAcceptance.run(); } } diff --git a/src/test/java/org/unlaxer/tinyexpression/p4/ubnfc/FraudAcceptance.java b/src/test/java/org/unlaxer/tinyexpression/p4/ubnfc/FraudAcceptance.java new file mode 100644 index 00000000..c29d86e8 --- /dev/null +++ b/src/test/java/org/unlaxer/tinyexpression/p4/ubnfc/FraudAcceptance.java @@ -0,0 +1,96 @@ +package org.unlaxer.tinyexpression.p4.ubnfc; + +import static org.junit.Assert.*; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.util.ArrayList; +import java.util.HexFormat; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.unlaxer.tinyexpression.generated.p4.TinyExpressionP4AST; +import org.unlaxer.tinyexpression.p4.P4ParserEngine; +import org.unlaxer.tinyexpression.p4.P4PreferredAstMapper; +import org.unlaxer.tinyexpression.p4.ubnfc.generated.api.Json; +import org.unlaxer.tinyexpression.parser.ExpressionTypes; + +/** Test-only production facade observer; never retries or catches parser failures. */ +public final class FraudAcceptance { + private FraudAcceptance() {} + public static final long LIMIT_NS = 1_000_000_000L; + private static final int[] LENGTHS = {404, 209, 68, 707, 459}; + + public static String validated(P4PreferredAstMapper.ParsedAst parsed, String source) { + assertNotNull("null parse result", parsed); + assertNotNull("null mapped AST", parsed.ast()); + assertEquals("expected numeric conditional", TinyExpressionP4AST.IfExpr.class, parsed.ast().getClass()); + assertArrayEquals("partial source span", new int[] {0, source.codePointCount(0, source.length())}, + parsed.sourceText().spanOf(parsed.ast()).orElseThrow()); + assertEquals(source, parsed.sourceText().text(parsed.ast())); + String canonical = CanonicalAst.json(parsed); + validateSpans(Json.read(canonical), source.codePointCount(0, source.length())); + return canonical; + } + + private static void validateSpans(Object value, int length) { + if (value instanceof Map map) { + if (map.containsKey("fields") && map.get("type") instanceof String) { + assertTrue("missing/invalid node span: " + map.get("type"), map.get("span") instanceof List); + var span = (List) map.get("span"); + int start = ((Number) span.get(0)).intValue(); + int end = ((Number) span.get(1)).intValue(); + assertTrue("out of source CP span", start >= 0 && start <= end && end <= length); + } + map.values().forEach(child -> validateSpans(child, length)); + } else if (value instanceof List list) { + list.forEach(child -> validateSpans(child, length)); + } + } + + public static void run() throws Exception { + var records = new ArrayList(); + var formulas = Files.readAllLines(Path.of("rust/tinyexpression-rs/tests/fixtures/fraud-alert.tsv"), StandardCharsets.UTF_8) + .stream().filter(line -> !line.isBlank() && !line.startsWith("#")).toList(); + assertEquals("exact original five inputs", 5, formulas.size()); + String selectedId = System.getProperty("tinyexpression.fraud.id"); + if (selectedId != null) assertTrue("unknown fraud id", formulas.stream().anyMatch(line -> line.startsWith(selectedId + "\t"))); + Path output = Path.of(System.getProperty("tinyexpression.fraud.output", "target/fraud-acceptance/java.jsonl")); + Files.createDirectories(output.getParent()); + Files.deleteIfExists(output); // no stale success record after a failed run + for (int index = 0; index < formulas.size(); index++) { + String[] columns = formulas.get(index).split("\t", -1); + String id = columns[0], source = columns[1]; + if (selectedId != null && !selectedId.equals(id)) continue; + assertEquals("pinned fraud CP length", LENGTHS[index], source.codePointCount(0, source.length())); + String expected = null; + for (String phase : List.of("cold", "warmup", "warm")) { + // Only the production parse+map call is timed. Validation, serialization and I/O are outside. + long start = System.nanoTime(); + var parsed = P4ParserEngine.with(P4ParserEngine.UBNFC, + () -> P4PreferredAstMapper.parseDetailed(source, ExpressionTypes._float)); + long elapsed = System.nanoTime() - start; + String canonical = validated(parsed, source); + if (expected != null) assertEquals("phase changed AST/spans", expected, canonical); + expected = canonical; + var row = new LinkedHashMap(); + row.put("id", id); row.put("phase", phase); + row.put("pid", ProcessHandle.current().pid()); + row.put("first_observation_in_process", records.isEmpty()); row.put("backend", "java-ubnfc"); + row.put("entry", "P4PreferredAstMapper.parseDetailed(_float)"); + row.put("memo", "ubnfc-default-packrat"); row.put("elapsed_ns", elapsed); + row.put("observations", Map.of("diagnostics", true, "occurrences", true)); + row.put("sha256", HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(source.getBytes(StandardCharsets.UTF_8)))); + row.put("cp", LENGTHS[index]); row.put("accepted", true); row.put("diagnostic", null); + row.put("selectionMode", parsed.selectionMode()); row.put("ast", Json.read(canonical)); + records.add(Json.write(row)); + // Persist even an over-budget sample; a slow/failed run is never a missing-data pass. + Files.write(output, records, StandardCharsets.UTF_8); + System.out.println("fraud production " + id + " " + phase + " " + elapsed + " ns"); + assertTrue(id + " " + phase + " must parse+map in < 1 second, was " + elapsed + " ns", elapsed < LIMIT_NS); + } + } + } +} diff --git a/src/test/java/org/unlaxer/tinyexpression/p4/ubnfc/FraudAcceptanceTest.java b/src/test/java/org/unlaxer/tinyexpression/p4/ubnfc/FraudAcceptanceTest.java new file mode 100644 index 00000000..564bd6b6 --- /dev/null +++ b/src/test/java/org/unlaxer/tinyexpression/p4/ubnfc/FraudAcceptanceTest.java @@ -0,0 +1,23 @@ +package org.unlaxer.tinyexpression.p4.ubnfc; + +import static org.junit.Assert.*; +import org.junit.Test; +import org.unlaxer.tinyexpression.p4.P4ParserEngine; +import org.unlaxer.tinyexpression.p4.P4PreferredAstMapper; +import org.unlaxer.tinyexpression.p4.P4SourceText; +import org.unlaxer.tinyexpression.parser.ExpressionTypes; + +public class FraudAcceptanceTest { + @Test public void nullPartialAndRejectedResultsCannotCountAsFastSuccess() { + assertThrows(AssertionError.class, () -> FraudAcceptance.validated(null, "x")); + assertThrows(AssertionError.class, () -> FraudAcceptance.validated( + new P4PreferredAstMapper.ParsedAst(null, "invalid", P4SourceText.lexicalOnly()), "x")); + var source = "if(true){1}else{0}"; + var parsed = P4ParserEngine.with(P4ParserEngine.UBNFC, + () -> P4PreferredAstMapper.parseDetailed(source, ExpressionTypes._float)); + FraudAcceptance.validated(parsed, source); + assertThrows(AssertionError.class, () -> FraudAcceptance.validated(parsed, source + "@")); + assertThrows(RuntimeException.class, () -> P4ParserEngine.with(P4ParserEngine.UBNFC, + () -> P4PreferredAstMapper.parseDetailed(source + "@", ExpressionTypes._float))); + } +}