From 00d2aaaf2bfc2c791f07174a65d896ef8d5865dc Mon Sep 17 00:00:00 2001 From: Dmitrii Vasilev Date: Wed, 7 Oct 2026 15:45:34 +0700 Subject: [PATCH 1/8] tri: test runs the tests, mutate plant names what a one-line mutant turns red (Closes #7369) `tri test ` ran `t27c test`, which only lists the tests, and then printed "tests passed" whatever they did. It now runs `t27c test-report `, prints the report and one summary line `tests: N, pass: P, fail: F (names)`, and exits non-zero when a test FAILs, when the spec is BLOCKED, or when no test ran. The verdict is read from the report's text: test-report exits 0 even when tests fail (#7370, not changed here). `tri mutate plant --file --line --from --to [--expect ]...` is one recorded mutation experiment: the mutant is a copy in the work dir `tri mutate spec` uses (so `use` still resolves), `` must occur exactly once on line N, the one-line diff is printed, a clean baseline copy must pass first, the failing tests are named, and the original's sha256 is compared before and after. With --expect it exits 0 iff exactly those tests fail; without, iff any test fails. Cards specs/tools/tri/test.t27 and specs/tools/tri/mutate.t27 follow the clap docs. main.rs and mutate.rs are already in tools/policy/foreign-exceptions.txt; owner-approved-foreign under the owner's standing rule (#7090, #7255, #7072). Census re-blessed, not moved by this change: tools/census/quiet.txt "named a path but not quiet" 167->181, "a tracked path, present" 10->12, "no path ANYWHERE in the step" 9->7. `tri census explain` puts all three on .github/workflows/auto-merge-ready-prs.yml, changed on master since the bless at 31ad9dfbf; the pre-commit gate refuses any commit until it is re-blessed. Co-Authored-By: Claude Opus 5.5 --- cli/tri/src/main.rs | 21 +- cli/tri/src/mutate.rs | 497 ++++++++++++++++++++++++++++++++++++- specs/tools/tri/mutate.t27 | 7 +- specs/tools/tri/test.t27 | 15 +- tools/census/quiet.txt | 6 +- 5 files changed, 528 insertions(+), 18 deletions(-) diff --git a/cli/tri/src/main.rs b/cli/tri/src/main.rs index 21430047e3..f89605e4d6 100644 --- a/cli/tri/src/main.rs +++ b/cli/tri/src/main.rs @@ -113,8 +113,17 @@ enum Commands { Gen { spec_path: String, }, + /// Run a .t27 spec's tests with `t27c test-report` and fail unless every + /// test passed and at least one ran. + /// + /// Prints the report, then one line `tests: N, pass: P, fail: F (names)`. + /// Exits non-zero when a test FAILs, when the spec is BLOCKED, or when no + /// test ran. Test { spec_path: String, + /// The t27c binary; default target/release/t27c, then t27c on PATH. + #[arg(long)] + t27c: Option, }, Verdict { #[arg(long)] @@ -879,10 +888,12 @@ fn cmd_gen(spec_path: &str) -> Result<()> { Ok(()) } -fn cmd_test(spec_path: &str) -> Result<()> { - run_t27c(&["test", spec_path])?; - println!("tests passed: {}", spec_path); - Ok(()) +/// `t27c test` only LISTS a spec's tests; this used to run it and then print +/// "tests passed" whatever the tests did (#7369). The tests run in +/// `t27c test-report`, whose exit code is 0 even when they FAIL (#7370), so +/// the verdict is read from its text. +fn cmd_test(spec_path: &str, t27c: Option<&str>) -> Result<()> { + mutate::test_spec(Path::new(spec_path), t27c) } fn cmd_verdict(toxic: bool) -> Result<()> { @@ -1056,7 +1067,7 @@ fn main() -> Result<()> { } } Commands::Gen { spec_path } => cmd_gen(spec_path)?, - Commands::Test { spec_path } => cmd_test(spec_path)?, + Commands::Test { spec_path, t27c } => cmd_test(spec_path, t27c.as_deref())?, Commands::Verdict { toxic } => cmd_verdict(*toxic)?, Commands::Experience { action } => { let root = find_trinity_root()?; diff --git a/cli/tri/src/mutate.rs b/cli/tri/src/mutate.rs index 4aceac4982..8fefd9d30c 100644 --- a/cli/tri/src/mutate.rs +++ b/cli/tri/src/mutate.rs @@ -68,6 +68,31 @@ pub enum MutateCmd { #[arg(long)] t27c: Option, }, + /// Plant one hand-written mutant in a .t27 spec (--from becomes --to on + /// --line) and name the spec's tests that go red on it. + Plant { + /// The .t27 spec. It is never edited: the mutant is a copy in the work + /// dir `tri mutate spec` uses, and the original's sha256 is compared + /// before and after the run. + #[arg(long)] + file: String, + /// The 1-based line to change. + #[arg(long)] + line: usize, + /// The text to replace. It must occur exactly once on --line. + #[arg(long, allow_hyphen_values = true)] + from: String, + /// The text that replaces it, on the same line. + #[arg(long, allow_hyphen_values = true)] + to: String, + /// A test the mutant should fail; repeat for each. With it, exit 0 only + /// when exactly these tests fail; without it, when any test fails. + #[arg(long)] + expect: Vec, + /// The t27c binary; default target/release/t27c, then t27c on PATH. + #[arg(long)] + t27c: Option, + }, } pub fn run(cmd: &MutateCmd) -> Result<()> { @@ -88,6 +113,14 @@ pub fn run(cmd: &MutateCmd) -> Result<()> { *timeout, t27c.as_deref(), ), + MutateCmd::Plant { + file, + line, + from, + to, + expect, + t27c, + } => plant(Path::new(file), *line, from, to, expect, t27c.as_deref()), } } @@ -1187,7 +1220,7 @@ fn spec_report(file: &Path, ran: &[(&SpecMutant, Fate)]) -> String { out } -fn resolve_t27c(explicit: Option<&str>) -> String { +pub(crate) fn resolve_t27c(explicit: Option<&str>) -> String { if let Some(p) = explicit { return p.to_string(); } @@ -1346,6 +1379,290 @@ fn mutate_spec( Ok(()) } +/// What `t27c test-report ` said about a spec's tests, read from its +/// text. Never from its exit code: test-report exits 0 when tests FAIL and +/// when the spec is BLOCKED (#7370). `tri test` used to run `t27c test`, which +/// only lists the tests, and then printed "tests passed" (#7369). +#[derive(Debug, Default, PartialEq)] +pub(crate) struct TestReport { + /// test-report's `BLOCKED `: the spec never built, so no test ran. + pub blocked: Option, + pub total: usize, + pub passed: usize, + pub failed: usize, + /// The ` FAIL ` lines, in report order. + pub fails: Vec, + /// The ` pass ` lines; test-report prints them only with --verbose. + pub passes: Vec, + /// Invariants proved at compile time. They are not tests. + pub invariants: usize, +} + +impl TestReport { + /// The one greppable line: `tests: N, pass: P, fail: F (names...)`. + pub fn summary(&self) -> String { + let mut s = format!("tests: {}, pass: {}, fail: {}", self.total, self.passed, self.failed); + if let Some(why) = &self.blocked { + s.push_str(&format!(", BLOCKED: {}", why.lines().next().unwrap_or(""))); + } else if !self.fails.is_empty() { + s.push_str(&format!(" ({})", self.fails.join(", "))); + } + s + } + + /// Green means: it built, at least one test ran, and none failed. + pub fn green(&self, spec: &Path) -> Result<()> { + if let Some(why) = &self.blocked { + bail!("{} is BLOCKED, so no test ran: {why}", spec.display()); + } + if self.failed > 0 { + bail!( + "{} of {} test(s) FAIL in {}: {}", + self.failed, + self.total, + spec.display(), + self.fails.join(", ") + ); + } + if self.total == 0 { + let inv = match self.invariants { + 0 => String::new(), + n => format!(" ({n} invariant(s) proved at compile time; an invariant is not a test)"), + }; + bail!("no test ran in {}: it has no `test` block{inv}", spec.display()); + } + Ok(()) + } +} + +/// Read test-report's text. Fails closed: a report with no `tests N` line +/// that is not BLOCKED, or whose FAIL count disagrees with the FAIL names it +/// listed, is an error, not a pass. +pub(crate) fn parse_test_report(out: &str) -> Result { + // Totals pad the word to a column (` tests 4`); a name line has + // exactly two spaces (` FAIL negz`); the per-test assert lines after + // the totals start with a number or `-`, never with `pass` or `FAIL`. + let blocked = regex::Regex::new(r"^ BLOCKED (.*)$").unwrap(); + let totals = regex::Regex::new(r"^ (tests|pass|FAIL) {3,}(\d+)\s*$").unwrap(); + let named = regex::Regex::new(r"^ (FAIL|pass) (\S.*?)\s*$").unwrap(); + let invariants = regex::Regex::new(r"^ invariants\s+(\d+)").unwrap(); + let mut r = TestReport::default(); + let (mut total, mut passed, mut failed) = (None, None, None); + for l in out.lines() { + if let Some(c) = blocked.captures(l) { + r.blocked = Some(c[1].trim().to_string()); + return Ok(r); + } + if let Some(c) = totals.captures(l) { + let slot = match &c[1] { + "tests" => &mut total, + "pass" => &mut passed, + _ => &mut failed, + }; + if slot.is_none() { + *slot = c[2].parse::().ok(); + } + } else if let Some(c) = named.captures(l) { + let list = if &c[1] == "FAIL" { &mut r.fails } else { &mut r.passes }; + list.push(c[2].to_string()); + } else if let Some(c) = invariants.captures(l) { + r.invariants = c[1].parse().unwrap_or(0); + } + } + let (Some(t), Some(p), Some(f)) = (total, passed, failed) else { + bail!("t27c test-report printed no `tests` / `pass` / `FAIL` totals and no BLOCKED line:\n{out}"); + }; + if p + f != t { + bail!("t27c test-report's totals do not add up: tests {t}, pass {p}, FAIL {f}"); + } + if r.fails.len() != f { + bail!( + "t27c test-report counts {f} FAIL but names {}: {}", + r.fails.len(), + r.fails.join(", ") + ); + } + (r.total, r.passed, r.failed) = (t, p, f); + Ok(r) +} + +/// Run ` test-report ` and read it. Returns the text too, so a +/// caller can print exactly what t27c said. +pub(crate) fn test_report(t27c: &str, spec: &Path, verbose: bool) -> Result<(String, TestReport)> { + let mut cmd = Command::new(t27c); + cmd.arg("test-report").arg(spec); + if verbose { + cmd.arg("--verbose"); + } + let out = cmd + .output() + .with_context(|| format!("cannot start `{t27c} test-report`"))?; + let text = String::from_utf8_lossy(&out.stdout).into_owned(); + if !out.status.success() { + bail!( + "`{t27c} test-report {}` exited {:?}: {}", + spec.display(), + out.status.code(), + first_error(&String::from_utf8_lossy(&out.stderr)) + ); + } + let r = parse_test_report(&text)?; + Ok((text, r)) +} + +/// `tri test `: run the spec's tests with `t27c test-report`, print the +/// report and the summary line, and fail unless every test passed and at +/// least one ran. +pub(crate) fn test_spec(spec: &Path, t27c: Option<&str>) -> Result<()> { + if !spec.is_file() { + bail!("not a file: {}", spec.display()); + } + let t27c = resolve_t27c(t27c); + let (text, r) = test_report(&t27c, spec, false)?; + print!("{text}"); + println!("{}", r.summary()); + r.green(spec) +} + +/// The mutant text: `from` replaced by `to` on 1-based line `line` only. +/// Returns (old line, new line, whole mutant). Refuses anything but exactly +/// one occurrence of `from` on that line, so the experiment is the one named. +pub(crate) fn plant_line(text: &str, line: usize, from: &str, to: &str) -> Result<(String, String, String)> { + if from.is_empty() { + bail!("--from is empty"); + } + if from == to { + bail!("--from and --to are the same text, so nothing would change"); + } + if to.contains('\n') || to.contains('\r') { + bail!("--to holds a line break: a planted mutant changes one line into one line"); + } + let mut lines: Vec<&str> = text.split('\n').collect(); + let count = if text.ends_with('\n') { lines.len() - 1 } else { lines.len() }; + if line == 0 || line > count { + bail!("--line {line}: the file has {count} line(s), numbered from 1"); + } + let old = lines[line - 1]; + let hits = old.char_indices().filter(|(i, _)| old[*i..].starts_with(from)).count(); + if hits != 1 { + bail!( + "--from {from:?} occurs {hits} time(s) on line {line}; it must occur exactly once\n {line}: {}", + old.trim_end_matches('\r') + ); + } + let new = old.replacen(from, to, 1); + lines[line - 1] = &new; + let mutant = lines.join("\n"); + Ok(( + old.trim_end_matches('\r').to_string(), + new.trim_end_matches('\r').to_string(), + mutant, + )) +} + +fn sha256_hex(bytes: &[u8]) -> String { + use sha2::{Digest, Sha256}; + format!("{:x}", Sha256::digest(bytes)) +} + +/// `tri mutate plant`: one named experiment. The original is hashed before +/// and after, and a changed original is an error whatever the tests said. +fn plant( + file: &Path, + line: usize, + from: &str, + to: &str, + expect: &[String], + t27c: Option<&str>, +) -> Result<()> { + let bytes = std::fs::read(file).with_context(|| format!("cannot read {}", file.display()))?; + let before = sha256_hex(&bytes); + let text = String::from_utf8(bytes).with_context(|| format!("{} is not UTF-8", file.display()))?; + let (old, new, mutant) = plant_line(&text, line, from, to)?; + println!("{}:{line}", file.display()); + println!("- {old}"); + println!("+ {new}"); + + let dir = spec_work_dir(file, std::process::id()); + let verdict = plant_run(&resolve_t27c(t27c), file, &dir, &text, &mutant, expect); + let _ = std::fs::remove_dir_all(&dir); + + let after = std::fs::read(file) + .map(|b| sha256_hex(&b)) + .unwrap_or_else(|e| format!("unreadable ({e})")); + if after != before { + let ran = match &verdict { + Ok(()) => "the verdict above would have been a pass".to_string(), + Err(e) => format!("the run also said: {e:#}"), + }; + bail!( + "ORIGINAL CHANGED: {} was sha256 {before} before the run and is {after} after it. \ + Restore it from git before anything else; {ran}", + file.display() + ); + } + println!("original unchanged: {} sha256 {before}", file.display()); + verdict +} + +/// Baseline first, then the mutant, then the verdict. +fn plant_run(t27c: &str, file: &Path, dir: &Path, original: &str, mutant: &str, expect: &[String]) -> Result<()> { + let name = file.file_name().context("--file has no file name")?; + let base = dir.join("base").join(name); + let mutated = dir.join("mutant").join(name); + for (path, text) in [(&base, original), (&mutated, mutant)] { + std::fs::create_dir_all(path.parent().unwrap())?; + std::fs::write(path, text)?; + } + + // Against a red baseline every mutant "is killed" and says nothing. The + // copy sits in the work dir like the mutant, so a `use` it cannot resolve + // shows up here, not as a kill. + let (_, b) = test_report(t27c, &base, true)?; + println!("baseline: {}", b.summary()); + b.green(file).context("the unmutated spec is not green, so the mutant would mean nothing")?; + for e in expect { + if !b.passes.iter().any(|p| p == e) { + bail!("--expect {e}: the spec has no test of that name (its tests: {})", b.passes.join(", ")); + } + } + + let (_, m) = test_report(t27c, &mutated, false)?; + println!("mutant: {}", m.summary()); + if let Some(why) = &m.blocked { + bail!("the mutant does not build, so no test judged it (unviable, not killed): {why}"); + } + for f in &m.fails { + println!(" FAIL {f}"); + } + let mut got: Vec<&str> = m.fails.iter().map(String::as_str).collect(); + got.sort_unstable(); + got.dedup(); + if expect.is_empty() { + if got.is_empty() { + bail!("SURVIVED: no test failed on the mutant"); + } + println!("KILLED by: {}", got.join(", ")); + return Ok(()); + } + let mut want: Vec<&str> = expect.iter().map(String::as_str).collect(); + want.sort_unstable(); + want.dedup(); + if got != want { + let missing: Vec<&str> = want.iter().filter(|w| !got.contains(w)).copied().collect(); + let extra: Vec<&str> = got.iter().filter(|g| !want.contains(g)).copied().collect(); + bail!( + "expected FAIL {{{}}}, got FAIL {{{}}} (expected but passed: {}; failed but not expected: {})", + want.join(", "), + got.join(", "), + if missing.is_empty() { "none".into() } else { missing.join(", ") }, + if extra.is_empty() { "none".into() } else { extra.join(", ") } + ); + } + println!("KILLED as expected by: {}", got.join(", ")); + Ok(()) +} + #[cfg(test)] mod tests { use super::*; @@ -2095,4 +2412,182 @@ mod tests { "nothing listed when everything was killed" ); } + + // ---- tri test / tri mutate plant (#7369) ---- + + /// test-report's real text for specs/ternary/gft_relu.t27 with line 12 + /// planted, as `run_test_report` prints it (bootstrap/src/main.rs). + const REPORT_NEGZ: &str = concat!( + "--- test report: specs/ternary/gft_relu.t27 ---\n", + " FAIL negz\n", + "\n", + " tests 4\n", + " pass 3\n", + " FAIL 1\n", + " invariants 2 proved -- comptime, so compiling IS the check\n", + " rate 75.0%\n", + "\n", + " runtime asserts executed, per test (#6509; a pass with 0 is vacuous, T730):\n", + " 1 pos\n", + " - negz (failed, not counted)\n", + " 1 zero\n", + " 1 pos2\n", + " vacuous passes 0 of 3 (passed with 0 runtime asserts executed)\n", + ); + + #[test] + fn the_report_is_read_from_its_text() { + let r = parse_test_report(REPORT_NEGZ).unwrap(); + assert_eq!((r.total, r.passed, r.failed, r.invariants), (4, 3, 1, 2)); + assert_eq!(r.fails, vec!["negz"]); + assert!(r.passes.is_empty(), "no `pass ` line without --verbose"); + assert_eq!(r.summary(), "tests: 4, pass: 3, fail: 1 (negz)"); + + let verbose = "--- test report: s.t27 ---\n pass pos\n pass zero\n\n \ + tests 2\n pass 2\n FAIL 0\n rate 100.0%\n"; + let r = parse_test_report(verbose).unwrap(); + assert_eq!(r.passes, vec!["pos", "zero"]); + assert_eq!(r.summary(), "tests: 2, pass: 2, fail: 0"); + + let blocked = "--- test report: s.t27 ---\n BLOCKED zig: error: expected type 'i32'\n\n \ + A blocked spec is not a failing one.\n"; + let r = parse_test_report(blocked).unwrap(); + assert_eq!(r.blocked.as_deref(), Some("zig: error: expected type 'i32'")); + assert!(r.summary().contains("BLOCKED"), "{}", r.summary()); + } + + /// A report the reader does not recognise is an error, never a pass. + #[test] + fn a_report_that_does_not_add_up_is_an_error() { + assert!(parse_test_report("").is_err(), "no totals at all"); + assert!(parse_test_report("--- test report: s.t27 ---\n listed 4 tests\n").is_err()); + let miscount = " FAIL a\n\n tests 2\n pass 0\n FAIL 2\n"; + assert!(parse_test_report(miscount).is_err(), "2 FAIL counted, 1 named"); + let sum = "\n tests 3\n pass 1\n FAIL 0\n"; + assert!(parse_test_report(sum).is_err(), "1 + 0 is not 3"); + } + + /// `tri test`'s verdict: green only when it built, a test ran and none + /// failed. The old command printed "tests passed" for all four. + #[test] + fn tri_test_is_green_only_when_every_test_passed_and_one_ran() { + let spec = Path::new("s.t27"); + let ok = TestReport { total: 4, passed: 4, ..Default::default() }; + assert!(ok.green(spec).is_ok()); + let red = parse_test_report(REPORT_NEGZ).unwrap(); + let e = format!("{:#}", red.green(spec).unwrap_err()); + assert!(e.contains("1 of 4 test(s) FAIL") && e.contains("negz"), "{e}"); + let none = TestReport { invariants: 2, ..Default::default() }; + let e = format!("{:#}", none.green(spec).unwrap_err()); + assert!(e.contains("no test ran") && e.contains("2 invariant"), "{e}"); + let blocked = TestReport { blocked: Some("zig: error: x".into()), ..Default::default() }; + assert!(format!("{:#}", blocked.green(spec).unwrap_err()).contains("BLOCKED")); + } + + #[test] + fn a_plant_changes_one_line_and_only_one_occurrence() { + let src = "module m;\nfn f(x: i32) -> i32 {\n if (x < 0) { return 0; } // negative -> 0\n return x;\n}\n"; + let (old, new, m) = plant_line(src, 3, "{ return 0; }", "{ return x; }").unwrap(); + assert_eq!(old, " if (x < 0) { return 0; } // negative -> 0"); + assert_eq!(new, " if (x < 0) { return x; } // negative -> 0"); + assert_eq!(m, src.replace("{ return 0; }", "{ return x; }")); + assert_eq!(m.lines().count(), src.lines().count()); + assert!(m.ends_with("}\n"), "the trailing newline is kept"); + + let err = |line, from: &str, to: &str| format!("{:#}", plant_line(src, line, from, to).unwrap_err()); + assert!(err(4, "{ return 0; }", "x").contains("0 time(s) on line 4"), "absent on that line"); + assert!(err(2, "i32", "u32").contains("2 time(s)"), "twice on the line"); + assert!(err(3, "00", "1").contains("0 time(s)")); + assert!(err(0, "m", "n").contains("numbered from 1")); + assert!(err(6, "m", "n").contains("5 line(s)"), "line past the end"); + assert!(err(3, "", "x").contains("empty")); + assert!(err(3, "x", "x").contains("same text")); + assert!(err(3, "return 0;", "a\nb").contains("line break")); + // Overlapping occurrences count: `aa` sits twice in `aaa`. + assert!(format!("{:#}", plant_line("aaa\n", 1, "aa", "b").unwrap_err()).contains("2 time(s)")); + } + + /// A tree with `specs/` so the work dir lands under its `target/`, and a + /// stub t27c: test `t` FAILs when the spec it is given holds `kills-t`. + /// `extra` runs first, for a stub that also does something it must not. + fn plant_tree(name: &str, extra: &str) -> (PathBuf, PathBuf, String) { + let root = std::env::temp_dir().join(format!("tri-plant-{}-{name}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(root.join("specs")).unwrap(); + let spec = root.join("specs").join("g.t27"); + std::fs::write(&spec, "module g;\nfn f(x: i32) -> i32 {\n if (x < 0) { return 0; }\n return x;\n}\n").unwrap(); + let stub = root.join("t27c"); + let script = format!( + "#!/bin/sh\n{extra}\nspec=\"$2\"\np=0\nf=0\necho \"--- test report: $spec ---\"\n\ + for t in pos negz zero pos2; do\n \ + if grep -q \"kills-$t\" \"$spec\"; then echo \" FAIL $t\"; f=$((f+1));\n \ + else p=$((p+1)); [ \"$3\" = \"--verbose\" ] && echo \" pass $t\"; fi\n\ + done\necho\necho \" tests 4\"\necho \" pass $p\"\necho \" FAIL $f\"\n" + ); + std::fs::write(&stub, script).unwrap(); + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&stub, std::fs::Permissions::from_mode(0o755)).unwrap(); + let s = stub.to_string_lossy().into_owned(); + (root, spec, s) + } + + fn plant_with(stub: &str, spec: &Path, to: &str, expect: &[&str]) -> Result<()> { + let expect: Vec = expect.iter().map(|s| s.to_string()).collect(); + plant(spec, 3, "{ return 0; }", to, &expect, Some(stub)) + } + + #[test] + fn plant_passes_on_exactly_the_expected_failures() { + let (root, spec, stub) = plant_tree("expect", ""); + let before = std::fs::read(&spec).unwrap(); + let negz = "{ return x; } // kills-negz"; + assert!(plant_with(&stub, &spec, negz, &["negz"]).is_ok()); + assert!(plant_with(&stub, &spec, negz, &[]).is_ok(), "no --expect: any failure is a kill"); + let e = format!("{:#}", plant_with(&stub, &spec, negz, &["pos"]).unwrap_err()); + assert!(e.contains("expected but passed: pos") && e.contains("failed but not expected: negz"), "{e}"); + let e = format!("{:#}", plant_with(&stub, &spec, negz, &["negz", "zero"]).unwrap_err()); + assert!(e.contains("expected but passed: zero"), "a subset is not the set: {e}"); + let e = format!("{:#}", plant_with(&stub, &spec, "{ return 0; } // same", &[]).unwrap_err()); + assert!(e.contains("SURVIVED"), "{e}"); + let e = format!("{:#}", plant_with(&stub, &spec, negz, &["nosuch"]).unwrap_err()); + assert!(e.contains("no test of that name"), "{e}"); + assert_eq!(std::fs::read(&spec).unwrap(), before, "the original is never written"); + assert!(!root.join("target").join(format!("tri-mutate-spec-{}", std::process::id())).exists()); + let _ = std::fs::remove_dir_all(&root); + } + + /// A red baseline says nothing about the mutant, so it is refused. + #[test] + fn plant_refuses_a_spec_that_is_red_before_the_plant() { + let (root, spec, stub) = plant_tree("redbase", ""); + let src = std::fs::read_to_string(&spec).unwrap().replace("return x;", "return x; // kills-pos"); + std::fs::write(&spec, src).unwrap(); + let e = format!("{:#}", plant_with(&stub, &spec, "{ return x; } // kills-negz", &["negz"]).unwrap_err()); + assert!(e.contains("unmutated spec is not green"), "{e}"); + let _ = std::fs::remove_dir_all(&root); + } + + /// The original is hashed after the run whatever the verdict was: a run + /// that wrote to it fails even when its tests said exactly what was + /// expected. + #[test] + fn plant_fails_when_the_original_changed_during_the_run() { + let (root, spec, stub) = plant_tree("tamper", "echo '// touched' >> \"$(dirname \"$0\")/specs/g.t27\""); + let e = format!("{:#}", plant_with(&stub, &spec, "{ return x; } // kills-negz", &["negz"]).unwrap_err()); + assert!(e.contains("ORIGINAL CHANGED") && e.contains("would have been a pass"), "{e}"); + let _ = std::fs::remove_dir_all(&root); + } + + /// `tri test` on the stub: rc 0 on a clean spec, an error naming the + /// failing test once one fails. + #[test] + fn tri_test_fails_when_a_test_fails() { + let (root, spec, stub) = plant_tree("tritest", ""); + assert!(test_spec(&spec, Some(&stub)).is_ok()); + let src = std::fs::read_to_string(&spec).unwrap().replace("{ return 0; }", "{ return x; } // kills-negz"); + std::fs::write(&spec, src).unwrap(); + let e = format!("{:#}", test_spec(&spec, Some(&stub)).unwrap_err()); + assert!(e.contains("negz"), "{e}"); + let _ = std::fs::remove_dir_all(&root); + } } diff --git a/specs/tools/tri/mutate.t27 b/specs/tools/tri/mutate.t27 index 1a8fff604c..177c15f907 100644 --- a/specs/tools/tri/mutate.t27 +++ b/specs/tools/tri/mutate.t27 @@ -3,7 +3,8 @@ ; Source of truth for the tool card on t27.ai (#/tools?tool=tri/mutate). ; Extracted from the clap #[derive(Subcommand)] enums in cli/tri/src (witness source-parse: ; cargo was not run, the enum and its /// docs were read at commit 3cb7a4352492; the spec action -; was read from the branch that adds it, gHashTag/t27#6993, and its kinds from #7022). +; was read from the branch that adds it, gHashTag/t27#6993, and its kinds from #7022; the plant +; action was read from the branch that adds it, gHashTag/t27#7369). ; Layer 5 of the ladder Specs -> Skills -> Crons -> Agents -> Tools. ASCII only (L3). ; phi^2 + 1/phi^2 = 3 | TRINITY @@ -24,8 +25,8 @@ pub const ENTRY : str = "cli/tri/src/main.rs"; pub const ABOUT : str = "Find the constants in a checker that nothing actually checks."; pub const ABOUT_SOURCE : str = "clap /// doc on Commands::Mutate (cli/tri/src/main.rs; module cli/tri/src/mutate.rs)"; ; Nested subcommands (kebab-case as clap derives them) and their first /// paragraph ("" when none). -pub const ACTIONS : [2]str = ["run", "spec"]; -pub const ACTIONS_ABOUT : [2]str = ["Perturb each numeric literal in a file and report which ones the checker does not notice.", "Drop guards, flip operators and empty whole bodies inside the functions of a .t27 spec, and report the mutants its own tests do not notice."]; +pub const ACTIONS : [3]str = ["run", "spec", "plant"]; +pub const ACTIONS_ABOUT : [3]str = ["Perturb each numeric literal in a file and report which ones the checker does not notice.", "Drop guards, flip operators and empty whole bodies inside the functions of a .t27 spec, and report the mutants its own tests do not notice.", "Plant one hand-written mutant in a .t27 spec (--from becomes --to on --line) and name the spec's tests that go red on it."]; ; Top-level arguments/flags of the command itself, ": doc". pub const ARGS : [0]str = []; ; Agent letters that own this command, only where a source line names it (AGENTS_NOTE cites it). diff --git a/specs/tools/tri/test.t27 b/specs/tools/tri/test.t27 index 2d01e3f8e6..38ef194046 100644 --- a/specs/tools/tri/test.t27 +++ b/specs/tools/tri/test.t27 @@ -2,7 +2,10 @@ ; specs/tools/tri/test.t27 -- tool tri/test, the `tri test` command of the tri CLI ; Source of truth for the tool card on t27.ai (#/tools?tool=tri/test). ; Extracted from the clap #[derive(Subcommand)] enums in cli/tri/src (witness source-parse: -; cargo was not run, the enum and its /// docs were read at commit 3cb7a4352492). +; cargo was not run, the enum and its /// docs were read at commit 3cb7a4352492; the Test +; variant's doc, its --t27c flag and the test-report run were read from the branch that adds +; them, gHashTag/t27#7369. Before it, cmd_test ran `t27c test`, which only lists the tests, +; and printed "tests passed" whatever they did). ; Layer 5 of the ladder Specs -> Skills -> Crons -> Agents -> Tools. ASCII only (L3). ; phi^2 + 1/phi^2 = 3 | TRINITY @@ -19,19 +22,19 @@ pub const COMMAND : str = "tri test"; pub const VARIANT : str = "Commands::Test"; pub const SOURCE : str = "cli/tri/src/main.rs"; pub const ENTRY : str = "cli/tri/src/main.rs"; -; This variant carries no /// doc; ABOUT quotes the source named in ABOUT_SOURCE instead. -pub const ABOUT : str = "Runs tests for a .t27 spec file using t27c."; -pub const ABOUT_SOURCE : str = "tri-mcp tool tri_test description (cli/tri-mcp/src/main.rs); handler cmd_test runs t27c test (cli/tri/src/main.rs:870)"; +; The clap about text (first /// paragraph), as `tri --help` would print it. +pub const ABOUT : str = "Run a .t27 spec's tests with `t27c test-report` and fail unless every test passed and at least one ran."; +pub const ABOUT_SOURCE : str = "clap /// doc on Commands::Test (cli/tri/src/main.rs); handler cmd_test calls mutate::test_spec (cli/tri/src/mutate.rs), which runs t27c test-report and reads the verdict from its text, since test-report exits 0 when a test FAILs (gHashTag/t27#7370)"; ; Nested subcommands (kebab-case as clap derives them) and their first /// paragraph ("" when none). pub const ACTIONS : [0]str = []; pub const ACTIONS_ABOUT : [0]str = []; ; Top-level arguments/flags of the command itself, ": doc". -pub const ARGS : [1]str = [""]; +pub const ARGS : [2]str = ["", "--t27c: The t27c binary; default target/release/t27c, then t27c on PATH."]; ; Agent letters that own this command, only where a source line names it (AGENTS_NOTE cites it). pub const AGENTS : [2]str = ["T", "V"]; pub const AGENTS_NOTE : str = "docs/agents/AGENTS_ALPHABET.md \"Who holds a tool, and who holds a skill\" binds every tool to the Queen (T). Domain lead V: The spec's own AGENTS field is [\"T\",\"V\"] with AGENTS_NOTE citing .claude/agents/agent-v-verify.md:47 (\"tri test -- Run conformance tests\"); with T excluded by rule, V is the domain lead."; ; From the about text (long about when the doc has more than one paragraph); no marketing. -pub const WHEN_TO_USE : str = "Runs tests for a .t27 spec file using t27c."; +pub const WHEN_TO_USE : str = "Run a .t27 spec's tests with `t27c test-report` and fail unless every test passed and at least one ran. Prints the report, then one line `tests: N, pass: P, fail: F (names)`. Exits non-zero when a test FAILs, when the spec is BLOCKED, or when no test ran."; pub const WITNESS : str = "source-parse"; pub const ENABLED : bool = true; diff --git a/tools/census/quiet.txt b/tools/census/quiet.txt index ed26c31f43..e20d3baf23 100644 --- a/tools/census/quiet.txt +++ b/tools/census/quiet.txt @@ -2,7 +2,7 @@ GATE STEPS WHOSE PASS SURVIVES THE SUBJECT GOING MISSING workflow files read 65 steps in a quiet shape 30 - named a path but not quiet 167 (--excluded prints them) + named a path but not quiet 181 (--excluded prints them) by shape: failure branch passes 14 `… 2>/dev/null … || echo PASSED` @@ -12,10 +12,10 @@ GATE STEPS WHOSE PASS SURVIVES THE SUBJECT GOING MISSING and what can be said about the subject WITHOUT running the workflow: a tracked path, ABSENT 0 <- guarding nothing right now - a tracked path, present 10 + a tracked path, present 12 the run builds it 7 absence from a checkout says nothing a variable in the path 4 depends on the run - no path ANYWHERE in the step 9 cannot be checked -- NOT the same as absent + no path ANYWHERE in the step 7 cannot be checked -- NOT the same as absent No quiet gate names a tracked path that is missing today. That is a result, not an absence of one: the shapes are there and the subjects From abc2100a1c368d8917760712ee9ac0794ff6a68d Mon Sep 17 00:00:00 2001 From: Dmitrii Vasilev Date: Wed, 7 Oct 2026 16:10:55 +0700 Subject: [PATCH 2/8] tri mutate plant: an extra failure fails --expect; a red report with a non-zero exit is the red verdict (Refs #7369) Review of #7400, point by point: - A mutant of the exact-set check (`got != want` weakened to "every expected test is among the failures") survived all mutate:: tests: no test had the mutant fail MORE than --expect named. plant_fails_when_the_mutant_fails_more_than_expected plants a mutant that fails pos and pos2 with --expect pos and asserts the verdict fails and names "failed but not expected: pos2". - test_report treated any non-zero exit of `t27c test-report` as an error. #7370 makes test-report exit non-zero on FAIL, after which every killed planted mutant would read as an error and a red `tri test` would not print its report. The verdict is now read from the text whenever a report can be read: a red report (FAIL or BLOCKED) with a non-zero exit is that red verdict. A non-zero exit is an error when no report can be read, and when the report reads green (a run that disagrees with itself is not a pass). The stub t27c in the tests takes FAIL_RC / PASS_RC so both exits are exercised: a_non_zero_exit_with_a_red_report_is_read_as_the_red_verdict and a_non_zero_exit_without_a_red_report_is_an_error. Co-Authored-By: Claude Opus 5.5 --- cli/tri/src/mutate.rs | 107 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 95 insertions(+), 12 deletions(-) diff --git a/cli/tri/src/mutate.rs b/cli/tri/src/mutate.rs index 99b17f9fc3..ea2987425d 100644 --- a/cli/tri/src/mutate.rs +++ b/cli/tri/src/mutate.rs @@ -2345,9 +2345,10 @@ fn mutate_spec( } /// What `t27c test-report ` said about a spec's tests, read from its -/// text. Never from its exit code: test-report exits 0 when tests FAIL and -/// when the spec is BLOCKED (#7370). `tri test` used to run `t27c test`, which -/// only lists the tests, and then printed "tests passed" (#7369). +/// text. Never from its exit code alone: test-report exits 0 when tests FAIL +/// and when the spec is BLOCKED (#7370; `test_report` reads both exits). +/// `tri test` used to run `t27c test`, which only lists the tests, and then +/// printed "tests passed" (#7369). #[derive(Debug, Default, PartialEq)] pub(crate) struct TestReport { /// test-report's `BLOCKED `: the spec never built, so no test ran. @@ -2453,6 +2454,15 @@ pub(crate) fn parse_test_report(out: &str) -> Result { /// Run ` test-report ` and read it. Returns the text too, so a /// caller can print exactly what t27c said. +/// +/// The verdict is the text's. The exit code only says what an unreadable +/// report means: today test-report exits 0 on FAIL and on BLOCKED, and #7370 +/// makes it exit non-zero on both, so a red report with a non-zero exit is +/// that red verdict, not a tool error -- otherwise every killed mutant would +/// read as an error and a red `tri test` would not print its report. A +/// non-zero exit is an error when no report can be read, and when the report +/// reads green: a run that says every test passed and exits non-zero +/// disagrees with itself, and that is not a pass. pub(crate) fn test_report(t27c: &str, spec: &Path, verbose: bool) -> Result<(String, TestReport)> { let mut cmd = Command::new(t27c); cmd.arg("test-report").arg(spec); @@ -2463,16 +2473,26 @@ pub(crate) fn test_report(t27c: &str, spec: &Path, verbose: bool) -> Result<(Str .output() .with_context(|| format!("cannot start `{t27c} test-report`"))?; let text = String::from_utf8_lossy(&out.stdout).into_owned(); - if !out.status.success() { - bail!( - "`{t27c} test-report {}` exited {:?}: {}", - spec.display(), - out.status.code(), + let read = parse_test_report(&text); + if out.status.success() { + return read.map(|r| (text, r)); + } + let exited = format!( + "`{t27c} test-report {}` exited {}", + spec.display(), + out.status.code().map_or_else(|| "on a signal".to_string(), |c| c.to_string()) + ); + match read { + Ok(r) if r.blocked.is_some() || r.failed > 0 => Ok((text, r)), + Ok(r) => bail!( + "{exited}, yet its report reads green ({}); a run that disagrees with itself is not a pass", + r.summary() + ), + Err(e) => bail!( + "{exited}: {}\nand no report could be read from its output: {e:#}", first_error(&String::from_utf8_lossy(&out.stderr)) - ); + ), } - let r = parse_test_report(&text)?; - Ok((text, r)) } /// `tri test `: run the spec's tests with `t27c test-report`, print the @@ -3483,6 +3503,8 @@ mod tests { /// A tree with `specs/` so the work dir lands under its `target/`, and a /// stub t27c: test `t` FAILs when the spec it is given holds `kills-t`. /// `extra` runs first, for a stub that also does something it must not. + /// The stub exits 0 like today's test-report; `extra` may set FAIL_RC (the + /// exit after a FAIL, as #7370 proposes) or PASS_RC (the exit otherwise). fn plant_tree(name: &str, extra: &str) -> (PathBuf, PathBuf, String) { let root = std::env::temp_dir().join(format!("tri-plant-{}-{name}", std::process::id())); let _ = std::fs::remove_dir_all(&root); @@ -3495,7 +3517,8 @@ mod tests { for t in pos negz zero pos2; do\n \ if grep -q \"kills-$t\" \"$spec\"; then echo \" FAIL $t\"; f=$((f+1));\n \ else p=$((p+1)); [ \"$3\" = \"--verbose\" ] && echo \" pass $t\"; fi\n\ - done\necho\necho \" tests 4\"\necho \" pass $p\"\necho \" FAIL $f\"\n" + done\necho\necho \" tests 4\"\necho \" pass $p\"\necho \" FAIL $f\"\n\ + [ \"$f\" -gt 0 ] && exit ${{FAIL_RC:-0}}\nexit ${{PASS_RC:-0}}\n" ); std::fs::write(&stub, script).unwrap(); use std::os::unix::fs::PermissionsExt; @@ -3529,6 +3552,25 @@ mod tests { let _ = std::fs::remove_dir_all(&root); } + /// A mutant that fails MORE tests than --expect names is not the + /// experiment named: the verdict fails and names the extra test. A check + /// that only asked "is every expected test among the failures" let this + /// through (review of #7400). + #[test] + fn plant_fails_when_the_mutant_fails_more_than_expected() { + let (root, spec, stub) = plant_tree("extra", ""); + let both = "{ return x; } // kills-pos kills-pos2"; + assert!(plant_with(&stub, &spec, both, &["pos", "pos2"]).is_ok(), "the whole set is the set"); + let e = format!("{:#}", plant_with(&stub, &spec, both, &["pos"]).unwrap_err()); + assert!( + e.contains("got FAIL {pos, pos2}") + && e.contains("expected but passed: none") + && e.contains("failed but not expected: pos2"), + "{e}" + ); + let _ = std::fs::remove_dir_all(&root); + } + /// A red baseline says nothing about the mutant, so it is refused. #[test] fn plant_refuses_a_spec_that_is_red_before_the_plant() { @@ -3564,6 +3606,47 @@ mod tests { let _ = std::fs::remove_dir_all(&root); } + /// #7370 makes test-report exit non-zero on FAIL. A red report with a + /// non-zero exit is that red verdict, read from the text, not a tool + /// error: a planted mutant is still KILLED as expected, and `tri test` + /// still prints the report and names the failing test. + #[test] + fn a_non_zero_exit_with_a_red_report_is_read_as_the_red_verdict() { + let (root, spec, stub) = plant_tree("failrc", "FAIL_RC=1"); + assert!(test_spec(&spec, Some(&stub)).is_ok(), "a green run exits 0 here too"); + assert!(plant_with(&stub, &spec, "{ return x; } // kills-negz", &["negz"]).is_ok()); + let src = std::fs::read_to_string(&spec).unwrap().replace("{ return 0; }", "{ return x; } // kills-negz"); + std::fs::write(&spec, src).unwrap(); + let (text, r) = test_report(&stub, &spec, false).unwrap(); + assert!(text.contains(" FAIL negz\n") && text.contains(" FAIL 1\n"), "{text}"); + assert_eq!((r.total, r.failed, r.fails.clone()), (4, 1, vec!["negz".to_string()])); + let e = format!("{:#}", test_spec(&spec, Some(&stub)).unwrap_err()); + assert!(e.contains("1 of 4 test(s) FAIL") && e.contains("negz") && !e.contains("exited"), "{e}"); + let _ = std::fs::remove_dir_all(&root); + + let blocked = "echo '--- test report'; echo ' BLOCKED zig: error: x'; exit 2"; + let (root, spec, stub) = plant_tree("blockedrc", blocked); + let (_, r) = test_report(&stub, &spec, false).unwrap(); + assert_eq!(r.blocked.as_deref(), Some("zig: error: x")); + let _ = std::fs::remove_dir_all(&root); + } + + /// A non-zero exit is an error when no report can be read from the + /// output, and when the report reads green: neither is a pass. + #[test] + fn a_non_zero_exit_without_a_red_report_is_an_error() { + let (root, spec, stub) = plant_tree("norep", "echo 'zig: error: out of memory' >&2; exit 3"); + let e = format!("{:#}", test_report(&stub, &spec, false).unwrap_err()); + assert!(e.contains("exited 3") && e.contains("error: out of memory") && e.contains("no report"), "{e}"); + let _ = std::fs::remove_dir_all(&root); + + let (root, spec, stub) = plant_tree("passrc", "PASS_RC=1"); + let e = format!("{:#}", test_report(&stub, &spec, false).unwrap_err()); + assert!(e.contains("exited 1") && e.contains("reads green") && e.contains("tests: 4, pass: 4"), "{e}"); + assert!(test_spec(&spec, Some(&stub)).is_err()); + let _ = std::fs::remove_dir_all(&root); + } + /// A value in a spec's asserts: a bool or a count. #[derive(Debug, Clone, Copy, PartialEq)] enum V { From dbff4753cbe3b022df3d903d886947fe0c5382fb Mon Sep 17 00:00:00 2001 From: Dmitrii Vasilev Date: Wed, 7 Oct 2026 16:20:52 +0700 Subject: [PATCH 3/8] tri mutate tests: a child shell writes the stub t27c, so no exec hits ETXTBSY (Refs #7369) The plant/test stubs were written with std::fs::write in the test process. A test thread that forked in that window kept a write fd to the stub until its exec, and exec'ing the stub then failed with "Text file busy". On the lab, 4 of 40 runs of `mutate::` failed this way (and one more run failed tri_test_fails_when_a_test_fails). The stub is now written by a child `sh -c "printf '%s' ..."`, so this process never holds a write fd to it. Co-Authored-By: Claude Opus 5.5 --- cli/tri/src/mutate.rs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/cli/tri/src/mutate.rs b/cli/tri/src/mutate.rs index ea2987425d..d4dbf81ab0 100644 --- a/cli/tri/src/mutate.rs +++ b/cli/tri/src/mutate.rs @@ -3520,7 +3520,21 @@ mod tests { done\necho\necho \" tests 4\"\necho \" pass $p\"\necho \" FAIL $f\"\n\ [ \"$f\" -gt 0 ] && exit ${{FAIL_RC:-0}}\nexit ${{PASS_RC:-0}}\n" ); - std::fs::write(&stub, script).unwrap(); + // A child shell writes the stub; this process never opens it for + // writing. A test thread that forks while this process holds a write + // fd to the stub leaves that fd open in its child until the child + // execs, and exec'ing the stub inside that window fails with ETXTBSY + // ("Text file busy"): the parallel harness hit it in about one run + // in ten. `printf '%s'` writes the script byte for byte. + let wrote = Command::new("sh") + .arg("-c") + .arg("printf '%s' \"$1\" > \"$2\"") + .arg("sh") + .arg(&script) + .arg(&stub) + .status() + .unwrap(); + assert!(wrote.success(), "could not write the stub {}", stub.display()); use std::os::unix::fs::PermissionsExt; std::fs::set_permissions(&stub, std::fs::Permissions::from_mode(0o755)).unwrap(); let s = stub.to_string_lossy().into_owned(); From 4188a8ca18e581f25a7dda378c949e10135ce41e Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 8 Oct 2026 20:07:58 +0000 Subject: [PATCH 4/8] seal: re-seal specs/tools/tri/test.t27 (Refs #7369) coverage (check_seal_coverage.py) named tri_tool_tri_test.json stale: this branch changed the card. Re-sealed with `t27c seal --save` from t27c built at this tree, and `tri seals sync-twins` wrote nothing. check_seal_coverage.py: OK, 1643 seals, 1522 hold, 121 known-broken. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01ScHVrGSr6zZUdwkdC8DR9k --- .trinity/seals/tri_tool_tri_test.json | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.trinity/seals/tri_tool_tri_test.json b/.trinity/seals/tri_tool_tri_test.json index 106839892e..9776631340 100644 --- a/.trinity/seals/tri_tool_tri_test.json +++ b/.trinity/seals/tri_tool_tri_test.json @@ -1,14 +1,14 @@ { - "built_by": "t27c-bootstrap@0.5.0+1120d30ae", - "gen_hash_c": "sha256:a520429c1f5a5f857b0f6a5af608fae95ce633b513df47911c03211ba1641afb", - "gen_hash_rust": "sha256:ead428fb444f19cd261e3d3935c375369853b36329bbf77a4a9bf26c6873b1b6", - "gen_hash_verilog": "sha256:8208a8a06ca0f53a8f5fe30515b41c4935985431418665dcadcb2518c8ac5763", - "gen_hash_zig": "sha256:f4028242a4be4ce1c567bbf28867a5cfb0cd947db035e7552333b73e8366363d", + "built_by": "t27c-bootstrap@0.5.1+c0b721c83", + "gen_hash_c": "sha256:b0d19d780e941712a2142433890680bae25a7c409a4cd867621d203af3e84838", + "gen_hash_rust": "sha256:5f0ca35d68f6b3c1b0d25c28d30589152a07b0e45ac3094549282a216d1f3522", + "gen_hash_verilog": "sha256:d40f2ba9717438d0b0cc517c843c262deaf6f02ac3f700ddeff0b56144f8e13a", + "gen_hash_zig": "sha256:1fcb55ec4ef450e66ec403e62410f214642fc8f21395f245fb6abd7929cf8903", "module": "tool_tri_test", "ring": 12, - "sealed_at": "2026-10-07T23:12:48Z", - "sealed_by": "t27c-bootstrap@0.5.0", - "spec_hash": "sha256:47814809da3b7a7a1e4e7dd69364a553f71c9702c2be8ba31c571badcbf8ee39", + "sealed_at": "2026-10-08T20:07:22Z", + "sealed_by": "t27c-bootstrap@0.5.1", + "spec_hash": "sha256:3e1cf6fec9804dcbef2d213bd2788deccdd5155f0eb869ad3300dc54c7ed52fa", "spec_path": "specs/tools/tri/test.t27", "tests": { "failed": 0, From 8931e52fb4d7e8567ba44576c0c97489515539d3 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 8 Oct 2026 20:09:31 +0000 Subject: [PATCH 5/8] fix(specs-generate): port #7916 -- record #7436's switch refuse fixtures (Refs #7369) emit-bitexact is red on every PR since #7436; this is #7916's change and no-ops once it lands. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01ScHVrGSr6zZUdwkdC8DR9k --- tools/specs_generate_baseline.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/specs_generate_baseline.txt b/tools/specs_generate_baseline.txt index 18f3947fb2..a4d281815b 100644 --- a/tools/specs_generate_baseline.txt +++ b/tools/specs_generate_baseline.txt @@ -11,3 +11,7 @@ bootstrap/tests/fixtures/core_selfhost/refuse/e2_shift_left_assign.t27 | Error: # Not a debt: #7377's both-refuse fixtures, read by bootstrap/tests/core_selfhost.rs. gen-c must keep refusing them. bootstrap/tests/fixtures/core_selfhost/refuse/e1_pub_use.t27 | Error: Compile error: Unexpected top-level token: KwUse ('use') at line 4:5 bootstrap/tests/fixtures/core_selfhost/refuse/e2_pub_twice.t27 | Error: Compile error: Unexpected top-level token: KwPub ('pub') at line 3:5 +# Not a debt: #7436's both-refuse fixtures, read by bootstrap/tests/core_selfhost.rs. gen-c must keep refusing them. +bootstrap/tests/fixtures/core_selfhost/refuse/e2_switch_bool_label.t27 | Error: Compile error: parse error in fn 'k' near line 4: Expected RBrace, got KwTrue ('true') at line 4:25 +bootstrap/tests/fixtures/core_selfhost/refuse/e2_switch_multi_label.t27 | Error: Compile error: parse error in fn 'k' near line 4: Unexpected token in expression: Comma (',') at line 4:26 +bootstrap/tests/fixtures/core_selfhost/refuse/e2_switch_statement.t27 | Error: Compile error: parse error in fn 'k' near line 5: Unexpected token in expression: KwReturn ('return') at line 5:14 From d43d01ffb52df829b36de9656078fd66b3b073b6 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 09:04:05 +0000 Subject: [PATCH 6/8] tri test / tri mutate plant: every rule in t27, Rust is glue (Refs #7369) The own-language budget (owner rule 2026-10-07, #7371) refused this PR: 593 hand-written Rust lines in cli/tri/src/mutate.rs against a limit of 40 per file and 80 per PR. The decisions move into two specs, lowered by `t27c gen-rust` and mounted as #[path] modules, as `tri mutate census` does with specs/tri/mutate/census.t27: - specs/tri/test/report.t27 (gen/rust/tri/test/report.rs): reading `t27c test-report` text -- each line's kind, the first tests/pass/FAIL totals, the FAIL/pass names and the BLOCKED why by offset -- and the verdict: fail closed on no totals, totals that do not add up, or a FAIL count that disagrees with the FAIL names; a red report with a non-zero exit is that red verdict, a green one with a non-zero exit is not a pass (#7370). Also the --expect list checks (every --expect names a test; the FAIL names are exactly the --expect set). 13 tests. - specs/tri/mutate/plant.t27 (gen/rust/tri/mutate/plant.rs): the plant refusals (empty --from, --from == --to, a line break in --to, no such line, --from not exactly once on the line, overlaps counted), the outcome fold (red baseline, unknown --expect, unreadable or BLOCKED mutant, KILLED / KILLED as expected / SURVIVED / NOT as expected) and ORIGINAL CHANGED winning over any outcome. 8 tests. Glue: main.rs runs t27c and prints the summary line (37 added, one test of the summary over two report texts); mutate.rs reads the file, writes the two copies into the `mutate spec` work dir, hashes the original before and after, and prints (40 added). Budget: 77 of 80. The PR's 9 Rust tests are replaced by the specs' tests (parsing, green rule, plant line, expect set, red baseline, changed original); the hermetic stub-t27c runs were exercised by hand instead (a stub that appends to the original gives ORIGINAL CHANGED, rc 1). Cards: specs/tools/tri/test.t27 and mutate.t27 follow the new clap docs and are re-sealed; both new specs are sealed and ledgered as t27b pass. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01ScHVrGSr6zZUdwkdC8DR9k --- .trinity/seals/mutate_TriMutatePlant.json | 20 + .trinity/seals/test_TriTestReport.json | 20 + .trinity/seals/tri_tool_tri_mutate.json | 14 +- .trinity/seals/tri_tool_tri_test.json | 12 +- cli/tri/src/main.rs | 46 +- cli/tri/src/mutate.rs | 631 ++-------------------- docs/reports/t27b_expectations.json | 2 + gen/rust/tri/mutate/plant.rs | 235 ++++++++ gen/rust/tri/test/report.rs | 346 ++++++++++++ specs/tools/tri/mutate.t27 | 2 +- specs/tools/tri/test.t27 | 9 +- specs/tri/mutate/plant.t27 | 315 +++++++++++ specs/tri/test/report.t27 | 505 +++++++++++++++++ 13 files changed, 1535 insertions(+), 622 deletions(-) create mode 100644 .trinity/seals/mutate_TriMutatePlant.json create mode 100644 .trinity/seals/test_TriTestReport.json create mode 100644 gen/rust/tri/mutate/plant.rs create mode 100644 gen/rust/tri/test/report.rs create mode 100644 specs/tri/mutate/plant.t27 create mode 100644 specs/tri/test/report.t27 diff --git a/.trinity/seals/mutate_TriMutatePlant.json b/.trinity/seals/mutate_TriMutatePlant.json new file mode 100644 index 0000000000..34ddc5f095 --- /dev/null +++ b/.trinity/seals/mutate_TriMutatePlant.json @@ -0,0 +1,20 @@ +{ + "built_by": "t27c-bootstrap@0.5.1+c0b721c83", + "gen_hash_c": "sha256:00ec0fd9a5b0eee498cd770bd03fa5fc79f61c018d3b94ad663ab3661529cf55", + "gen_hash_rust": "sha256:bedfb86cc4d9c5b0fa972e11f4434d10f2c20cb68bf17afff3a3c572a081071a", + "gen_hash_verilog": "sha256:2270d197974d4609000b6ded806598a348ff1a56ff4b91a61f67de22c110772c", + "gen_hash_zig": "sha256:2ad2379438ea12d57aa48758e25ba3d2c6bd11ee144e1192e03bdff663c0b0b8", + "module": "TriMutatePlant", + "ring": 12, + "sealed_at": "2026-10-09T08:49:38Z", + "sealed_by": "t27c-bootstrap@0.5.1", + "spec_hash": "sha256:6619f0b6cafdc9cc5abe33c35f11b030bb8036671e14baa1e79936c579909df0", + "spec_path": "specs/tri/mutate/plant.t27", + "tests": { + "failed": 0, + "failing": [], + "forced": false, + "passed": 8, + "total": 8 + } +} \ No newline at end of file diff --git a/.trinity/seals/test_TriTestReport.json b/.trinity/seals/test_TriTestReport.json new file mode 100644 index 0000000000..e8a415e435 --- /dev/null +++ b/.trinity/seals/test_TriTestReport.json @@ -0,0 +1,20 @@ +{ + "built_by": "t27c-bootstrap@0.5.1+c0b721c83", + "gen_hash_c": "sha256:5727a9e7b4cace405dff3ea3023789ee5d5bf8697b991c2a937a61d6449167e1", + "gen_hash_rust": "sha256:97f5785cb929ad2499cec12d236e721d5973b837dd522b8f1fc5aa7b5947cdfc", + "gen_hash_verilog": "sha256:67d0c7871b80a0f058665154f604b3d255dcd32393a1620199a65fb329bdafbf", + "gen_hash_zig": "sha256:43c7dccfe44c23285ba074bab8197afaff1556015ea80889483f24c5ddad2d42", + "module": "TriTestReport", + "ring": 12, + "sealed_at": "2026-10-09T08:49:37Z", + "sealed_by": "t27c-bootstrap@0.5.1", + "spec_hash": "sha256:c5914b13c06cfd8ff1b10448106d616ef8e256f3720cbb4d8f8d2519667d142c", + "spec_path": "specs/tri/test/report.t27", + "tests": { + "failed": 0, + "failing": [], + "forced": false, + "passed": 13, + "total": 13 + } +} \ No newline at end of file diff --git a/.trinity/seals/tri_tool_tri_mutate.json b/.trinity/seals/tri_tool_tri_mutate.json index 4b1a1eaf94..54313e8482 100644 --- a/.trinity/seals/tri_tool_tri_mutate.json +++ b/.trinity/seals/tri_tool_tri_mutate.json @@ -1,14 +1,14 @@ { - "built_by": "t27c-bootstrap@0.5.1+bbb77f0c3", - "gen_hash_c": "sha256:676361a189ed2fd5ce33d7881275953ac023b844927ce135b6b41d57cab8be4a", - "gen_hash_rust": "sha256:6902973acdd641d0e561e3313f66c1a4cfbfaa26a3cd36a01563d0f0c2ac3a6d", - "gen_hash_verilog": "sha256:6bbc1330fe5a3c0ebae24dc9960cbc20ea5180c7dbb4f44ab22361f57a18421d", - "gen_hash_zig": "sha256:4d8de6cb7c8f8e8d163c9fd192983c92d6dd8b8eea944d5d5b9a862fb923d9e4", + "built_by": "t27c-bootstrap@0.5.1+c0b721c83", + "gen_hash_c": "sha256:fd736493cbd958d85fac82a65cbabba01a15ecb4af37c873b9fe23892a80ebc2", + "gen_hash_rust": "sha256:113ce838d9c54435a3047f4b695858d5e3515cd389cb7c4394b1ae77d7dc7e43", + "gen_hash_verilog": "sha256:030852712a0e7da935b8fdf3b42af7bfc024353a8ec41bedfa928f5dde4fb0c1", + "gen_hash_zig": "sha256:bfd01ceef1c25488fdba94116f28759af26e9bb9767f7038a6dde18fb6ad5651", "module": "tool_tri_mutate", "ring": 12, - "sealed_at": "2026-10-08T19:24:08Z", + "sealed_at": "2026-10-09T08:50:56Z", "sealed_by": "t27c-bootstrap@0.5.1", - "spec_hash": "sha256:2cdb91d73a7613e2081474820fc1737c8c9b21d0bf299f8b24930b99d866531b", + "spec_hash": "sha256:f705017b7001024557ca8344ff4ce63f0eb41d5e04d208bea1d832f69a3d2296", "spec_path": "specs/tools/tri/mutate.t27", "tests": { "failed": 0, diff --git a/.trinity/seals/tri_tool_tri_test.json b/.trinity/seals/tri_tool_tri_test.json index 9776631340..6186e87a9a 100644 --- a/.trinity/seals/tri_tool_tri_test.json +++ b/.trinity/seals/tri_tool_tri_test.json @@ -1,14 +1,14 @@ { "built_by": "t27c-bootstrap@0.5.1+c0b721c83", - "gen_hash_c": "sha256:b0d19d780e941712a2142433890680bae25a7c409a4cd867621d203af3e84838", - "gen_hash_rust": "sha256:5f0ca35d68f6b3c1b0d25c28d30589152a07b0e45ac3094549282a216d1f3522", - "gen_hash_verilog": "sha256:d40f2ba9717438d0b0cc517c843c262deaf6f02ac3f700ddeff0b56144f8e13a", - "gen_hash_zig": "sha256:1fcb55ec4ef450e66ec403e62410f214642fc8f21395f245fb6abd7929cf8903", + "gen_hash_c": "sha256:68dff38aee7eff7abf4e37051f955a3f4174ae9cbcd097468fdfa4977c836c27", + "gen_hash_rust": "sha256:bcc538c79a6a7b540535d34e94d2615e4673c5cc94195652e214493baf73a1fa", + "gen_hash_verilog": "sha256:ffe7f3ac5f623e6e83fb92681ac7ddb27c836abcae169787939e683e3a869c60", + "gen_hash_zig": "sha256:ec67f5540b168a6e2554f6747b0fc9c328b74bf5e00e67dbb7a06e588c3fa09a", "module": "tool_tri_test", "ring": 12, - "sealed_at": "2026-10-08T20:07:22Z", + "sealed_at": "2026-10-09T08:50:55Z", "sealed_by": "t27c-bootstrap@0.5.1", - "spec_hash": "sha256:3e1cf6fec9804dcbef2d213bd2788deccdd5155f0eb869ad3300dc54c7ed52fa", + "spec_hash": "sha256:d55a36d6f72de8d716592ab1b63b82b064dc3cd64ca5302729d88ccdfd792e77", "spec_path": "specs/tools/tri/test.t27", "tests": { "failed": 0, diff --git a/cli/tri/src/main.rs b/cli/tri/src/main.rs index f89605e4d6..d9f1848ee2 100644 --- a/cli/tri/src/main.rs +++ b/cli/tri/src/main.rs @@ -41,6 +41,8 @@ mod prcheck; mod prose; mod quant; mod red; +#[path = "../../../gen/rust/tri/test/report.rs"] #[allow(dead_code, unused_parens)] +mod report; // t27c gen-rust of specs/tri/test/report.t27 (#7369): every rule of reading `t27c test-report` mod renum; mod reseal; mod rtl; @@ -113,12 +115,7 @@ enum Commands { Gen { spec_path: String, }, - /// Run a .t27 spec's tests with `t27c test-report` and fail unless every - /// test passed and at least one ran. - /// - /// Prints the report, then one line `tests: N, pass: P, fail: F (names)`. - /// Exits non-zero when a test FAILs, when the spec is BLOCKED, or when no - /// test ran. + /// Run a .t27 spec's tests with `t27c test-report` and fail unless every test passed and at least one ran (rules: specs/tri/test/report.t27). Test { spec_path: String, /// The t27c binary; default target/release/t27c, then t27c on PATH. @@ -888,12 +885,37 @@ fn cmd_gen(spec_path: &str) -> Result<()> { Ok(()) } -/// `t27c test` only LISTS a spec's tests; this used to run it and then print -/// "tests passed" whatever the tests did (#7369). The tests run in -/// `t27c test-report`, whose exit code is 0 even when they FAIL (#7370), so -/// the verdict is read from its text. -fn cmd_test(spec_path: &str, t27c: Option<&str>) -> Result<()> { - mutate::test_spec(Path::new(spec_path), t27c) +/// `t27c test` only lists the tests (#7369); they run in `t27c test-report`, which exits 0 when one FAILs (#7370). +fn cmd_test(spec: &str, t27c: Option<&str>) -> Result<()> { + let (t, v) = test_report(&mutate::resolve_t27c(t27c), Path::new(spec), false)?; + println!("{t}{}", report_summary(t)); + anyhow::ensure!(report::passes(v), "{spec}: {}", report::why(v)); + Ok(()) +} + +/// Run ` test-report `; the verdict is report.t27's, from the text and whether t27c exited 0. +pub(crate) fn test_report(t27c: &str, spec: &Path, verbose: bool) -> Result<(&'static str, u8)> { + let out = Command::new(t27c).arg("test-report").arg(spec).args(verbose.then_some("--verbose")).stderr(std::process::Stdio::inherit()).output().with_context(|| format!("cannot start `{t27c} test-report`"))?; + let text: &'static str = String::from_utf8_lossy(&out.stdout).into_owned().leak(); // gen-rust's `string` is `&'static str` (#7449) + Ok((text, report::verdict(text, out.status.success()))) +} + +/// The report's names of one kind (FAIL, pass, or the BLOCKED why), joined. +pub(crate) fn report_names(t: &'static str, k: u8) -> String { + (0..report::count(t, k)).map(|n| &t[report::name_from(t, k, n)..report::name_to(t, k, n)]).collect::>().join(", ") +} + +/// The one greppable line: `tests: N, pass: P, fail: F (names)`, or `, BLOCKED: `. +pub(crate) fn report_summary(t: &'static str) -> String { + let (b, f, n) = (report_names(t, report::LINE_BLOCKED), report_names(t, report::LINE_FAIL_NAME), |k| report::total(t, k)); + let tail = if !b.is_empty() { format!(", BLOCKED: {b}") } else if !f.is_empty() { format!(" ({f})") } else { String::new() }; + format!("tests: {}, pass: {}, fail: {}{tail}", n(report::LINE_TESTS), n(report::LINE_PASS), n(report::LINE_FAIL)) +} + +#[test] // the glue's one line over report.t27's reading: the #7400 demo's red report, and a BLOCKED one +fn the_summary_line_names_what_the_report_names() { + assert_eq!(report_summary(" FAIL negz\n\n tests 4\n pass 3\n FAIL 1\n"), "tests: 4, pass: 3, fail: 1 (negz)"); + assert_eq!(report_summary(" BLOCKED zig: error: x \n\n A blocked spec\n"), "tests: 0, pass: 0, fail: 0, BLOCKED: zig: error: x"); } fn cmd_verdict(toxic: bool) -> Result<()> { diff --git a/cli/tri/src/mutate.rs b/cli/tri/src/mutate.rs index 8deccfc9a7..dc937a3fb8 100644 --- a/cli/tri/src/mutate.rs +++ b/cli/tri/src/mutate.rs @@ -34,6 +34,8 @@ use survivors::{gate_on, mutate_exit, not_killed, survivor_exit, EXIT_OK, EXIT_S #[path = "../../../gen/rust/tri/mutate/lab.rs"] #[allow(dead_code, unused_parens)] mod lab; // of specs/tri/mutate/lab.t27 (#7050): what `mutate spec --lab` decides about a lab run +#[path = "../../../gen/rust/tri/mutate/plant.rs"] #[allow(dead_code, unused_parens)] +mod plant; // of specs/tri/mutate/plant.t27 (#7369): every decision of `mutate plant` #[derive(Subcommand)] pub enum MutateCmd { @@ -106,31 +108,8 @@ pub enum MutateCmd { #[arg(long)] lab_wait: Option, }, - /// Plant one hand-written mutant in a .t27 spec (--from becomes --to on - /// --line) and name the spec's tests that go red on it. - Plant { - /// The .t27 spec. It is never edited: the mutant is a copy in the work - /// dir `tri mutate spec` uses, and the original's sha256 is compared - /// before and after the run. - #[arg(long)] - file: String, - /// The 1-based line to change. - #[arg(long)] - line: usize, - /// The text to replace. It must occur exactly once on --line. - #[arg(long, allow_hyphen_values = true)] - from: String, - /// The text that replaces it, on the same line. - #[arg(long, allow_hyphen_values = true)] - to: String, - /// A test the mutant should fail; repeat for each. With it, exit 0 only - /// when exactly these tests fail; without it, when any test fails. - #[arg(long)] - expect: Vec, - /// The t27c binary; default target/release/t27c, then t27c on PATH. - #[arg(long)] - t27c: Option, - }, + /// Plant one hand-written mutant in a .t27 spec (--from becomes --to on --line) and name the spec's tests that go red on it (rules: specs/tri/mutate/plant.t27). + Plant { #[arg(long)] file: String, #[arg(long)] line: usize, #[arg(long, allow_hyphen_values = true)] from: String, #[arg(long, allow_hyphen_values = true)] to: String, #[arg(long)] expect: Vec, #[arg(long)] t27c: Option }, /// `mutate spec` on each .t27 in --dir by name, then one exit and killed share (rules: specs/tri/mutate/census.t27); every arg after --dir goes to each run. Census { #[arg(long)] @@ -216,14 +195,7 @@ pub fn run(cmd: &MutateCmd) -> Result<()> { }, ) } - MutateCmd::Plant { - file, - line, - from, - to, - expect, - t27c, - } => plant(Path::new(file), *line, from, to, expect, t27c.as_deref()), + MutateCmd::Plant { file, line, from, to, expect, t27c } => plant(Path::new(file), *line, from, to, expect, t27c.as_deref()), MutateCmd::Census { dir, pass } => { let mut specs: Vec = std::fs::read_dir(dir)?.filter_map(|e| Some(e.ok()?.path())).collect(); specs.retain(|p| p.extension().is_some_and(|x| x == "t27")); @@ -2077,6 +2049,40 @@ fn spec_work_dir(file: &Path, pid: u32) -> PathBuf { } } +/// `tri mutate plant`: the copies, the two t27c runs and the hashes; what they mean is plant.t27's and report.t27's. +fn plant(file: &Path, n: usize, from: &str, to: &str, expect: &[String], t27c: Option<&str>) -> Result<()> { + use crate::report::{self, LINE_FAIL_NAME as FAILS}; + let (leak, sha) = (|s: String| -> &'static str { s.leak() }, |b: &[u8]| format!("{:x}", ::digest(b))); + let t = leak(std::fs::read_to_string(file).with_context(|| format!("cannot read {}", file.display()))?); + let (from, to, want, k) = (leak(from.into()), leak(to.into()), leak(expect.join("\n")), expect.len() as u64); + let code = plant::plant_check(t, n, from, to); + anyhow::ensure!(code == plant::PLANT_OK, "{}:{n}: {}", file.display(), plant::why(code)); + let (a, b, at, before) = (plant::line_start(t, n), plant::line_stop(t, n), plant::plant_at(t, n, from), sha(t.as_bytes())); + let mutant = format!("{}{to}{}", &t[..at], &t[at + from.len()..]); + println!("{}:{n}\n- {}\n+ {}", file.display(), &t[a..b], &mutant[a..b - from.len() + to.len()]); + let (dir, name, t27c) = (spec_work_dir(file, std::process::id()), file.file_name().context("--file has no file name")?, resolve_t27c(t27c)); + let run = |p: &Path, s: &str, verbose: bool, label: &str| -> Result<(&'static str, u8)> { + std::fs::create_dir_all(p.parent().unwrap())?; + std::fs::write(p, s)?; + let (r, v) = crate::test_report(&t27c, p, verbose)?; + println!("{label}{} -- {}", crate::report_summary(r), report::why(v)); + Ok((r, v)) + }; + let runs = run(&dir.join("base").join(name), t, true, "baseline: ").and_then(|(bt, bv)| match report::passes(bv) { + true => Ok((bt, bv, run(&dir.join("mutant").join(name), &mutant, false, "mutant: ")?)), + false => Ok((bt, bv, ("", report::V_NO_TOTALS))), + }); + let _ = std::fs::remove_dir_all(&dir); + let (bt, bv, (mt, mv)) = runs?; + let o = plant::plant_outcome(report::passes(bv), report::all_named(bt, report::LINE_PASS_NAME, want, k), report::is_verdict(mv), mv != report::V_BLOCKED, report::count(mt, FAILS), k, report::same_names(mt, FAILS, want, k)); + let after = std::fs::read(file).map(|b| sha(&b)).unwrap_or_else(|e| format!("unreadable ({e})")); + let o = plant::plant_result(after == before, o); + println!("original: {} sha256 {before} before the run, {after} after it", file.display()); + anyhow::ensure!(plant::plant_passes(o), "{}", plant::why(o)); + println!("{} {}", plant::why(o), crate::report_names(mt, FAILS)); + Ok(()) +} + fn mutate_spec( file: &Path, func: Option<&str>, @@ -2205,310 +2211,6 @@ fn mutate_spec( } } -/// What `t27c test-report ` said about a spec's tests, read from its -/// text. Never from its exit code alone: test-report exits 0 when tests FAIL -/// and when the spec is BLOCKED (#7370; `test_report` reads both exits). -/// `tri test` used to run `t27c test`, which only lists the tests, and then -/// printed "tests passed" (#7369). -#[derive(Debug, Default, PartialEq)] -pub(crate) struct TestReport { - /// test-report's `BLOCKED `: the spec never built, so no test ran. - pub blocked: Option, - pub total: usize, - pub passed: usize, - pub failed: usize, - /// The ` FAIL ` lines, in report order. - pub fails: Vec, - /// The ` pass ` lines; test-report prints them only with --verbose. - pub passes: Vec, - /// Invariants proved at compile time. They are not tests. - pub invariants: usize, -} - -impl TestReport { - /// The one greppable line: `tests: N, pass: P, fail: F (names...)`. - pub fn summary(&self) -> String { - let mut s = format!("tests: {}, pass: {}, fail: {}", self.total, self.passed, self.failed); - if let Some(why) = &self.blocked { - s.push_str(&format!(", BLOCKED: {}", why.lines().next().unwrap_or(""))); - } else if !self.fails.is_empty() { - s.push_str(&format!(" ({})", self.fails.join(", "))); - } - s - } - - /// Green means: it built, at least one test ran, and none failed. - pub fn green(&self, spec: &Path) -> Result<()> { - if let Some(why) = &self.blocked { - bail!("{} is BLOCKED, so no test ran: {why}", spec.display()); - } - if self.failed > 0 { - bail!( - "{} of {} test(s) FAIL in {}: {}", - self.failed, - self.total, - spec.display(), - self.fails.join(", ") - ); - } - if self.total == 0 { - let inv = match self.invariants { - 0 => String::new(), - n => format!(" ({n} invariant(s) proved at compile time; an invariant is not a test)"), - }; - bail!("no test ran in {}: it has no `test` block{inv}", spec.display()); - } - Ok(()) - } -} - -/// Read test-report's text. Fails closed: a report with no `tests N` line -/// that is not BLOCKED, or whose FAIL count disagrees with the FAIL names it -/// listed, is an error, not a pass. -pub(crate) fn parse_test_report(out: &str) -> Result { - // Totals pad the word to a column (` tests 4`); a name line has - // exactly two spaces (` FAIL negz`); the per-test assert lines after - // the totals start with a number or `-`, never with `pass` or `FAIL`. - let blocked = regex::Regex::new(r"^ BLOCKED (.*)$").unwrap(); - let totals = regex::Regex::new(r"^ (tests|pass|FAIL) {3,}(\d+)\s*$").unwrap(); - let named = regex::Regex::new(r"^ (FAIL|pass) (\S.*?)\s*$").unwrap(); - let invariants = regex::Regex::new(r"^ invariants\s+(\d+)").unwrap(); - let mut r = TestReport::default(); - let (mut total, mut passed, mut failed) = (None, None, None); - for l in out.lines() { - if let Some(c) = blocked.captures(l) { - r.blocked = Some(c[1].trim().to_string()); - return Ok(r); - } - if let Some(c) = totals.captures(l) { - let slot = match &c[1] { - "tests" => &mut total, - "pass" => &mut passed, - _ => &mut failed, - }; - if slot.is_none() { - *slot = c[2].parse::().ok(); - } - } else if let Some(c) = named.captures(l) { - let list = if &c[1] == "FAIL" { &mut r.fails } else { &mut r.passes }; - list.push(c[2].to_string()); - } else if let Some(c) = invariants.captures(l) { - r.invariants = c[1].parse().unwrap_or(0); - } - } - let (Some(t), Some(p), Some(f)) = (total, passed, failed) else { - bail!("t27c test-report printed no `tests` / `pass` / `FAIL` totals and no BLOCKED line:\n{out}"); - }; - if p + f != t { - bail!("t27c test-report's totals do not add up: tests {t}, pass {p}, FAIL {f}"); - } - if r.fails.len() != f { - bail!( - "t27c test-report counts {f} FAIL but names {}: {}", - r.fails.len(), - r.fails.join(", ") - ); - } - (r.total, r.passed, r.failed) = (t, p, f); - Ok(r) -} - -/// Run ` test-report ` and read it. Returns the text too, so a -/// caller can print exactly what t27c said. -/// -/// The verdict is the text's. The exit code only says what an unreadable -/// report means: today test-report exits 0 on FAIL and on BLOCKED, and #7370 -/// makes it exit non-zero on both, so a red report with a non-zero exit is -/// that red verdict, not a tool error -- otherwise every killed mutant would -/// read as an error and a red `tri test` would not print its report. A -/// non-zero exit is an error when no report can be read, and when the report -/// reads green: a run that says every test passed and exits non-zero -/// disagrees with itself, and that is not a pass. -pub(crate) fn test_report(t27c: &str, spec: &Path, verbose: bool) -> Result<(String, TestReport)> { - let mut cmd = Command::new(t27c); - cmd.arg("test-report").arg(spec); - if verbose { - cmd.arg("--verbose"); - } - let out = cmd - .output() - .with_context(|| format!("cannot start `{t27c} test-report`"))?; - let text = String::from_utf8_lossy(&out.stdout).into_owned(); - let read = parse_test_report(&text); - if out.status.success() { - return read.map(|r| (text, r)); - } - let exited = format!( - "`{t27c} test-report {}` exited {}", - spec.display(), - out.status.code().map_or_else(|| "on a signal".to_string(), |c| c.to_string()) - ); - match read { - Ok(r) if r.blocked.is_some() || r.failed > 0 => Ok((text, r)), - Ok(r) => bail!( - "{exited}, yet its report reads green ({}); a run that disagrees with itself is not a pass", - r.summary() - ), - Err(e) => bail!( - "{exited}: {}\nand no report could be read from its output: {e:#}", - first_error(&String::from_utf8_lossy(&out.stderr)) - ), - } -} - -/// `tri test `: run the spec's tests with `t27c test-report`, print the -/// report and the summary line, and fail unless every test passed and at -/// least one ran. -pub(crate) fn test_spec(spec: &Path, t27c: Option<&str>) -> Result<()> { - if !spec.is_file() { - bail!("not a file: {}", spec.display()); - } - let t27c = resolve_t27c(t27c); - let (text, r) = test_report(&t27c, spec, false)?; - print!("{text}"); - println!("{}", r.summary()); - r.green(spec) -} - -/// The mutant text: `from` replaced by `to` on 1-based line `line` only. -/// Returns (old line, new line, whole mutant). Refuses anything but exactly -/// one occurrence of `from` on that line, so the experiment is the one named. -pub(crate) fn plant_line(text: &str, line: usize, from: &str, to: &str) -> Result<(String, String, String)> { - if from.is_empty() { - bail!("--from is empty"); - } - if from == to { - bail!("--from and --to are the same text, so nothing would change"); - } - if to.contains('\n') || to.contains('\r') { - bail!("--to holds a line break: a planted mutant changes one line into one line"); - } - let mut lines: Vec<&str> = text.split('\n').collect(); - let count = if text.ends_with('\n') { lines.len() - 1 } else { lines.len() }; - if line == 0 || line > count { - bail!("--line {line}: the file has {count} line(s), numbered from 1"); - } - let old = lines[line - 1]; - let hits = old.char_indices().filter(|(i, _)| old[*i..].starts_with(from)).count(); - if hits != 1 { - bail!( - "--from {from:?} occurs {hits} time(s) on line {line}; it must occur exactly once\n {line}: {}", - old.trim_end_matches('\r') - ); - } - let new = old.replacen(from, to, 1); - lines[line - 1] = &new; - let mutant = lines.join("\n"); - Ok(( - old.trim_end_matches('\r').to_string(), - new.trim_end_matches('\r').to_string(), - mutant, - )) -} - -fn sha256_hex(bytes: &[u8]) -> String { - use sha2::{Digest, Sha256}; - format!("{:x}", Sha256::digest(bytes)) -} - -/// `tri mutate plant`: one named experiment. The original is hashed before -/// and after, and a changed original is an error whatever the tests said. -fn plant( - file: &Path, - line: usize, - from: &str, - to: &str, - expect: &[String], - t27c: Option<&str>, -) -> Result<()> { - let bytes = std::fs::read(file).with_context(|| format!("cannot read {}", file.display()))?; - let before = sha256_hex(&bytes); - let text = String::from_utf8(bytes).with_context(|| format!("{} is not UTF-8", file.display()))?; - let (old, new, mutant) = plant_line(&text, line, from, to)?; - println!("{}:{line}", file.display()); - println!("- {old}"); - println!("+ {new}"); - - let dir = spec_work_dir(file, std::process::id()); - let verdict = plant_run(&resolve_t27c(t27c), file, &dir, &text, &mutant, expect); - let _ = std::fs::remove_dir_all(&dir); - - let after = std::fs::read(file) - .map(|b| sha256_hex(&b)) - .unwrap_or_else(|e| format!("unreadable ({e})")); - if after != before { - let ran = match &verdict { - Ok(()) => "the verdict above would have been a pass".to_string(), - Err(e) => format!("the run also said: {e:#}"), - }; - bail!( - "ORIGINAL CHANGED: {} was sha256 {before} before the run and is {after} after it. \ - Restore it from git before anything else; {ran}", - file.display() - ); - } - println!("original unchanged: {} sha256 {before}", file.display()); - verdict -} - -/// Baseline first, then the mutant, then the verdict. -fn plant_run(t27c: &str, file: &Path, dir: &Path, original: &str, mutant: &str, expect: &[String]) -> Result<()> { - let name = file.file_name().context("--file has no file name")?; - let base = dir.join("base").join(name); - let mutated = dir.join("mutant").join(name); - for (path, text) in [(&base, original), (&mutated, mutant)] { - std::fs::create_dir_all(path.parent().unwrap())?; - std::fs::write(path, text)?; - } - - // Against a red baseline every mutant "is killed" and says nothing. The - // copy sits in the work dir like the mutant, so a `use` it cannot resolve - // shows up here, not as a kill. - let (_, b) = test_report(t27c, &base, true)?; - println!("baseline: {}", b.summary()); - b.green(file).context("the unmutated spec is not green, so the mutant would mean nothing")?; - for e in expect { - if !b.passes.iter().any(|p| p == e) { - bail!("--expect {e}: the spec has no test of that name (its tests: {})", b.passes.join(", ")); - } - } - - let (_, m) = test_report(t27c, &mutated, false)?; - println!("mutant: {}", m.summary()); - if let Some(why) = &m.blocked { - bail!("the mutant does not build, so no test judged it (unviable, not killed): {why}"); - } - for f in &m.fails { - println!(" FAIL {f}"); - } - let mut got: Vec<&str> = m.fails.iter().map(String::as_str).collect(); - got.sort_unstable(); - got.dedup(); - if expect.is_empty() { - if got.is_empty() { - bail!("SURVIVED: no test failed on the mutant"); - } - println!("KILLED by: {}", got.join(", ")); - return Ok(()); - } - let mut want: Vec<&str> = expect.iter().map(String::as_str).collect(); - want.sort_unstable(); - want.dedup(); - if got != want { - let missing: Vec<&str> = want.iter().filter(|w| !got.contains(w)).copied().collect(); - let extra: Vec<&str> = got.iter().filter(|g| !want.contains(g)).copied().collect(); - bail!( - "expected FAIL {{{}}}, got FAIL {{{}}} (expected but passed: {}; failed but not expected: {})", - want.join(", "), - got.join(", "), - if missing.is_empty() { "none".into() } else { missing.join(", ") }, - if extra.is_empty() { "none".into() } else { extra.join(", ") } - ); - } - println!("KILLED as expected by: {}", got.join(", ")); - Ok(()) -} - #[cfg(test)] mod tests { use super::*; @@ -3267,261 +2969,6 @@ mod tests { ); } - // ---- tri test / tri mutate plant (#7369) ---- - - /// test-report's real text for specs/ternary/gft_relu.t27 with line 12 - /// planted, as `run_test_report` prints it (bootstrap/src/main.rs). - const REPORT_NEGZ: &str = concat!( - "--- test report: specs/ternary/gft_relu.t27 ---\n", - " FAIL negz\n", - "\n", - " tests 4\n", - " pass 3\n", - " FAIL 1\n", - " invariants 2 proved -- comptime, so compiling IS the check\n", - " rate 75.0%\n", - "\n", - " runtime asserts executed, per test (#6509; a pass with 0 is vacuous, T730):\n", - " 1 pos\n", - " - negz (failed, not counted)\n", - " 1 zero\n", - " 1 pos2\n", - " vacuous passes 0 of 3 (passed with 0 runtime asserts executed)\n", - ); - - #[test] - fn the_report_is_read_from_its_text() { - let r = parse_test_report(REPORT_NEGZ).unwrap(); - assert_eq!((r.total, r.passed, r.failed, r.invariants), (4, 3, 1, 2)); - assert_eq!(r.fails, vec!["negz"]); - assert!(r.passes.is_empty(), "no `pass ` line without --verbose"); - assert_eq!(r.summary(), "tests: 4, pass: 3, fail: 1 (negz)"); - - let verbose = "--- test report: s.t27 ---\n pass pos\n pass zero\n\n \ - tests 2\n pass 2\n FAIL 0\n rate 100.0%\n"; - let r = parse_test_report(verbose).unwrap(); - assert_eq!(r.passes, vec!["pos", "zero"]); - assert_eq!(r.summary(), "tests: 2, pass: 2, fail: 0"); - - let blocked = "--- test report: s.t27 ---\n BLOCKED zig: error: expected type 'i32'\n\n \ - A blocked spec is not a failing one.\n"; - let r = parse_test_report(blocked).unwrap(); - assert_eq!(r.blocked.as_deref(), Some("zig: error: expected type 'i32'")); - assert!(r.summary().contains("BLOCKED"), "{}", r.summary()); - } - - /// A report the reader does not recognise is an error, never a pass. - #[test] - fn a_report_that_does_not_add_up_is_an_error() { - assert!(parse_test_report("").is_err(), "no totals at all"); - assert!(parse_test_report("--- test report: s.t27 ---\n listed 4 tests\n").is_err()); - let miscount = " FAIL a\n\n tests 2\n pass 0\n FAIL 2\n"; - assert!(parse_test_report(miscount).is_err(), "2 FAIL counted, 1 named"); - let sum = "\n tests 3\n pass 1\n FAIL 0\n"; - assert!(parse_test_report(sum).is_err(), "1 + 0 is not 3"); - } - - /// `tri test`'s verdict: green only when it built, a test ran and none - /// failed. The old command printed "tests passed" for all four. - #[test] - fn tri_test_is_green_only_when_every_test_passed_and_one_ran() { - let spec = Path::new("s.t27"); - let ok = TestReport { total: 4, passed: 4, ..Default::default() }; - assert!(ok.green(spec).is_ok()); - let red = parse_test_report(REPORT_NEGZ).unwrap(); - let e = format!("{:#}", red.green(spec).unwrap_err()); - assert!(e.contains("1 of 4 test(s) FAIL") && e.contains("negz"), "{e}"); - let none = TestReport { invariants: 2, ..Default::default() }; - let e = format!("{:#}", none.green(spec).unwrap_err()); - assert!(e.contains("no test ran") && e.contains("2 invariant"), "{e}"); - let blocked = TestReport { blocked: Some("zig: error: x".into()), ..Default::default() }; - assert!(format!("{:#}", blocked.green(spec).unwrap_err()).contains("BLOCKED")); - } - - #[test] - fn a_plant_changes_one_line_and_only_one_occurrence() { - let src = "module m;\nfn f(x: i32) -> i32 {\n if (x < 0) { return 0; } // negative -> 0\n return x;\n}\n"; - let (old, new, m) = plant_line(src, 3, "{ return 0; }", "{ return x; }").unwrap(); - assert_eq!(old, " if (x < 0) { return 0; } // negative -> 0"); - assert_eq!(new, " if (x < 0) { return x; } // negative -> 0"); - assert_eq!(m, src.replace("{ return 0; }", "{ return x; }")); - assert_eq!(m.lines().count(), src.lines().count()); - assert!(m.ends_with("}\n"), "the trailing newline is kept"); - - let err = |line, from: &str, to: &str| format!("{:#}", plant_line(src, line, from, to).unwrap_err()); - assert!(err(4, "{ return 0; }", "x").contains("0 time(s) on line 4"), "absent on that line"); - assert!(err(2, "i32", "u32").contains("2 time(s)"), "twice on the line"); - assert!(err(3, "00", "1").contains("0 time(s)")); - assert!(err(0, "m", "n").contains("numbered from 1")); - assert!(err(6, "m", "n").contains("5 line(s)"), "line past the end"); - assert!(err(3, "", "x").contains("empty")); - assert!(err(3, "x", "x").contains("same text")); - assert!(err(3, "return 0;", "a\nb").contains("line break")); - // Overlapping occurrences count: `aa` sits twice in `aaa`. - assert!(format!("{:#}", plant_line("aaa\n", 1, "aa", "b").unwrap_err()).contains("2 time(s)")); - } - - /// A tree with `specs/` so the work dir lands under its `target/`, and a - /// stub t27c: test `t` FAILs when the spec it is given holds `kills-t`. - /// `extra` runs first, for a stub that also does something it must not. - /// The stub exits 0 like today's test-report; `extra` may set FAIL_RC (the - /// exit after a FAIL, as #7370 proposes) or PASS_RC (the exit otherwise). - fn plant_tree(name: &str, extra: &str) -> (PathBuf, PathBuf, String) { - let root = std::env::temp_dir().join(format!("tri-plant-{}-{name}", std::process::id())); - let _ = std::fs::remove_dir_all(&root); - std::fs::create_dir_all(root.join("specs")).unwrap(); - let spec = root.join("specs").join("g.t27"); - std::fs::write(&spec, "module g;\nfn f(x: i32) -> i32 {\n if (x < 0) { return 0; }\n return x;\n}\n").unwrap(); - let stub = root.join("t27c"); - let script = format!( - "#!/bin/sh\n{extra}\nspec=\"$2\"\np=0\nf=0\necho \"--- test report: $spec ---\"\n\ - for t in pos negz zero pos2; do\n \ - if grep -q \"kills-$t\" \"$spec\"; then echo \" FAIL $t\"; f=$((f+1));\n \ - else p=$((p+1)); [ \"$3\" = \"--verbose\" ] && echo \" pass $t\"; fi\n\ - done\necho\necho \" tests 4\"\necho \" pass $p\"\necho \" FAIL $f\"\n\ - [ \"$f\" -gt 0 ] && exit ${{FAIL_RC:-0}}\nexit ${{PASS_RC:-0}}\n" - ); - // A child shell writes the stub; this process never opens it for - // writing. A test thread that forks while this process holds a write - // fd to the stub leaves that fd open in its child until the child - // execs, and exec'ing the stub inside that window fails with ETXTBSY - // ("Text file busy"): the parallel harness hit it in about one run - // in ten. `printf '%s'` writes the script byte for byte. - let wrote = Command::new("sh") - .arg("-c") - .arg("printf '%s' \"$1\" > \"$2\"") - .arg("sh") - .arg(&script) - .arg(&stub) - .status() - .unwrap(); - assert!(wrote.success(), "could not write the stub {}", stub.display()); - use std::os::unix::fs::PermissionsExt; - std::fs::set_permissions(&stub, std::fs::Permissions::from_mode(0o755)).unwrap(); - let s = stub.to_string_lossy().into_owned(); - (root, spec, s) - } - - fn plant_with(stub: &str, spec: &Path, to: &str, expect: &[&str]) -> Result<()> { - let expect: Vec = expect.iter().map(|s| s.to_string()).collect(); - plant(spec, 3, "{ return 0; }", to, &expect, Some(stub)) - } - - #[test] - fn plant_passes_on_exactly_the_expected_failures() { - let (root, spec, stub) = plant_tree("expect", ""); - let before = std::fs::read(&spec).unwrap(); - let negz = "{ return x; } // kills-negz"; - assert!(plant_with(&stub, &spec, negz, &["negz"]).is_ok()); - assert!(plant_with(&stub, &spec, negz, &[]).is_ok(), "no --expect: any failure is a kill"); - let e = format!("{:#}", plant_with(&stub, &spec, negz, &["pos"]).unwrap_err()); - assert!(e.contains("expected but passed: pos") && e.contains("failed but not expected: negz"), "{e}"); - let e = format!("{:#}", plant_with(&stub, &spec, negz, &["negz", "zero"]).unwrap_err()); - assert!(e.contains("expected but passed: zero"), "a subset is not the set: {e}"); - let e = format!("{:#}", plant_with(&stub, &spec, "{ return 0; } // same", &[]).unwrap_err()); - assert!(e.contains("SURVIVED"), "{e}"); - let e = format!("{:#}", plant_with(&stub, &spec, negz, &["nosuch"]).unwrap_err()); - assert!(e.contains("no test of that name"), "{e}"); - assert_eq!(std::fs::read(&spec).unwrap(), before, "the original is never written"); - assert!(!root.join("target").join(format!("tri-mutate-spec-{}", std::process::id())).exists()); - let _ = std::fs::remove_dir_all(&root); - } - - /// A mutant that fails MORE tests than --expect names is not the - /// experiment named: the verdict fails and names the extra test. A check - /// that only asked "is every expected test among the failures" let this - /// through (review of #7400). - #[test] - fn plant_fails_when_the_mutant_fails_more_than_expected() { - let (root, spec, stub) = plant_tree("extra", ""); - let both = "{ return x; } // kills-pos kills-pos2"; - assert!(plant_with(&stub, &spec, both, &["pos", "pos2"]).is_ok(), "the whole set is the set"); - let e = format!("{:#}", plant_with(&stub, &spec, both, &["pos"]).unwrap_err()); - assert!( - e.contains("got FAIL {pos, pos2}") - && e.contains("expected but passed: none") - && e.contains("failed but not expected: pos2"), - "{e}" - ); - let _ = std::fs::remove_dir_all(&root); - } - - /// A red baseline says nothing about the mutant, so it is refused. - #[test] - fn plant_refuses_a_spec_that_is_red_before_the_plant() { - let (root, spec, stub) = plant_tree("redbase", ""); - let src = std::fs::read_to_string(&spec).unwrap().replace("return x;", "return x; // kills-pos"); - std::fs::write(&spec, src).unwrap(); - let e = format!("{:#}", plant_with(&stub, &spec, "{ return x; } // kills-negz", &["negz"]).unwrap_err()); - assert!(e.contains("unmutated spec is not green"), "{e}"); - let _ = std::fs::remove_dir_all(&root); - } - - /// The original is hashed after the run whatever the verdict was: a run - /// that wrote to it fails even when its tests said exactly what was - /// expected. - #[test] - fn plant_fails_when_the_original_changed_during_the_run() { - let (root, spec, stub) = plant_tree("tamper", "echo '// touched' >> \"$(dirname \"$0\")/specs/g.t27\""); - let e = format!("{:#}", plant_with(&stub, &spec, "{ return x; } // kills-negz", &["negz"]).unwrap_err()); - assert!(e.contains("ORIGINAL CHANGED") && e.contains("would have been a pass"), "{e}"); - let _ = std::fs::remove_dir_all(&root); - } - - /// `tri test` on the stub: rc 0 on a clean spec, an error naming the - /// failing test once one fails. - #[test] - fn tri_test_fails_when_a_test_fails() { - let (root, spec, stub) = plant_tree("tritest", ""); - assert!(test_spec(&spec, Some(&stub)).is_ok()); - let src = std::fs::read_to_string(&spec).unwrap().replace("{ return 0; }", "{ return x; } // kills-negz"); - std::fs::write(&spec, src).unwrap(); - let e = format!("{:#}", test_spec(&spec, Some(&stub)).unwrap_err()); - assert!(e.contains("negz"), "{e}"); - let _ = std::fs::remove_dir_all(&root); - } - - /// #7370 makes test-report exit non-zero on FAIL. A red report with a - /// non-zero exit is that red verdict, read from the text, not a tool - /// error: a planted mutant is still KILLED as expected, and `tri test` - /// still prints the report and names the failing test. - #[test] - fn a_non_zero_exit_with_a_red_report_is_read_as_the_red_verdict() { - let (root, spec, stub) = plant_tree("failrc", "FAIL_RC=1"); - assert!(test_spec(&spec, Some(&stub)).is_ok(), "a green run exits 0 here too"); - assert!(plant_with(&stub, &spec, "{ return x; } // kills-negz", &["negz"]).is_ok()); - let src = std::fs::read_to_string(&spec).unwrap().replace("{ return 0; }", "{ return x; } // kills-negz"); - std::fs::write(&spec, src).unwrap(); - let (text, r) = test_report(&stub, &spec, false).unwrap(); - assert!(text.contains(" FAIL negz\n") && text.contains(" FAIL 1\n"), "{text}"); - assert_eq!((r.total, r.failed, r.fails.clone()), (4, 1, vec!["negz".to_string()])); - let e = format!("{:#}", test_spec(&spec, Some(&stub)).unwrap_err()); - assert!(e.contains("1 of 4 test(s) FAIL") && e.contains("negz") && !e.contains("exited"), "{e}"); - let _ = std::fs::remove_dir_all(&root); - - let blocked = "echo '--- test report'; echo ' BLOCKED zig: error: x'; exit 2"; - let (root, spec, stub) = plant_tree("blockedrc", blocked); - let (_, r) = test_report(&stub, &spec, false).unwrap(); - assert_eq!(r.blocked.as_deref(), Some("zig: error: x")); - let _ = std::fs::remove_dir_all(&root); - } - - /// A non-zero exit is an error when no report can be read from the - /// output, and when the report reads green: neither is a pass. - #[test] - fn a_non_zero_exit_without_a_red_report_is_an_error() { - let (root, spec, stub) = plant_tree("norep", "echo 'zig: error: out of memory' >&2; exit 3"); - let e = format!("{:#}", test_report(&stub, &spec, false).unwrap_err()); - assert!(e.contains("exited 3") && e.contains("error: out of memory") && e.contains("no report"), "{e}"); - let _ = std::fs::remove_dir_all(&root); - - let (root, spec, stub) = plant_tree("passrc", "PASS_RC=1"); - let e = format!("{:#}", test_report(&stub, &spec, false).unwrap_err()); - assert!(e.contains("exited 1") && e.contains("reads green") && e.contains("tests: 4, pass: 4"), "{e}"); - assert!(test_spec(&spec, Some(&stub)).is_err()); - let _ = std::fs::remove_dir_all(&root); - } - /// A value in a spec's asserts: a bool or a count. #[derive(Debug, Clone, Copy, PartialEq)] enum V { diff --git a/docs/reports/t27b_expectations.json b/docs/reports/t27b_expectations.json index 4119a8bf17..ce7499d729 100644 --- a/docs/reports/t27b_expectations.json +++ b/docs/reports/t27b_expectations.json @@ -957,6 +957,7 @@ {"path": "specs/tri/math/measurement.t27", "t27b": "pass_vacuous"}, {"path": "specs/tri/mutate/census.t27", "t27b": "pass"}, {"path": "specs/tri/mutate/lab.t27", "t27b": "pass"}, + {"path": "specs/tri/mutate/plant.t27", "t27b": "pass"}, {"path": "specs/tri/mutate/survivors.t27", "t27b": "pass"}, {"path": "specs/tri/net/cloud.t27", "t27b": "pass_vacuous"}, {"path": "specs/tri/pipeline/cloud_orchestrator.t27", "t27b": "pass_vacuous"}, @@ -1057,6 +1058,7 @@ {"path": "specs/tri/t27b/undefined_arg_plan.t27", "t27b": "pass"}, {"path": "specs/tri/t27b/void_bind_plan.t27", "t27b": "pass"}, {"path": "specs/tri/t27b/wide_plan.t27", "t27b": "pass"}, + {"path": "specs/tri/test/report.t27", "t27b": "pass"}, {"path": "specs/tri/utils/args.t27", "t27b": "pass"}, {"path": "specs/tri/utils/arrow_time.t27", "t27b": "pass_vacuous"}, {"path": "specs/tri/utils/colors.t27", "t27b": "pass_vacuous"}, diff --git a/gen/rust/tri/mutate/plant.rs b/gen/rust/tri/mutate/plant.rs new file mode 100644 index 0000000000..2be0651e62 --- /dev/null +++ b/gen/rust/tri/mutate/plant.rs @@ -0,0 +1,235 @@ +// Generated from .t27 spec +// DO NOT EDIT — generated by t27c +// NOT LOWERED BY THIS BACKEND: 8 test(s), 1 invariant(s), 0 function(s) with no body. +// This backend emits declarations only. The spec's checks live in +// the Zig and Verilog outputs; do not read this file as verified. + +pub const PLANT_OK: u8 = 0; + +pub const REFUSE_EMPTY_FROM: u8 = 1; + +pub const REFUSE_SAME_TEXT: u8 = 2; + +pub const REFUSE_LINE_BREAK: u8 = 3; + +pub const REFUSE_NO_SUCH_LINE: u8 = 4; + +pub const REFUSE_NOT_ONCE: u8 = 5; + +pub const KILLED: u8 = 10; + +pub const KILLED_AS_EXPECTED: u8 = 11; + +pub const SURVIVED: u8 = 12; + +pub const NOT_AS_EXPECTED: u8 = 13; + +pub const UNVIABLE: u8 = 14; + +pub const BASELINE_RED: u8 = 15; + +pub const NO_SUCH_TEST: u8 = 16; + +pub const MUTANT_UNREAD: u8 = 17; + +pub const ORIGINAL_CHANGED: u8 = 18; + +pub fn line_count(text: &'static str) -> usize { + let mut lines: usize = 0; + let mut k: usize = 0; + while (k < text.len()) { + if (text.as_bytes()[(k) as usize] == 10) { + lines = (lines + 1); + } + k = (k + 1); + } + if ((text.len() > 0) && (text.as_bytes()[((text.len() - 1)) as usize] != 10)) { + lines = (lines + 1); + } + return lines; +} + +pub fn line_start(text: &'static str, n: usize) -> usize { + if ((n == 0) || (n > line_count(text))) { + return (text.len() + 1); + } + let mut seen: usize = 1; + let mut k: usize = 0; + while (seen < n) { + if (text.as_bytes()[(k) as usize] == 10) { + seen = (seen + 1); + } + k = (k + 1); + } + return k; +} + +pub fn line_stop(text: &'static str, n: usize) -> usize { + let start: usize = line_start(text, n); + if (start > text.len()) { + return start; + } + let mut k: usize = start; + while ((k < text.len()) && (text.as_bytes()[(k) as usize] != 10)) { + k = (k + 1); + } + return k; +} + +pub fn sits_at(text: &'static str, at: usize, stop: usize, lit: &'static str) -> bool { + if ((at > stop) || (lit.len() > (stop - at))) { + return false; + } + let mut i: usize = 0; + while (i < lit.len()) { + if (text.as_bytes()[((at + i)) as usize] != lit.as_bytes()[(i) as usize]) { + return false; + } + i = (i + 1); + } + return true; +} + +pub fn occurrences(text: &'static str, from: usize, stop: usize, lit: &'static str) -> u64 { + let mut hits: u64 = 0; + let mut at: usize = from; + while (at < stop) { + if sits_at(text, at, stop, lit) { + hits = (hits + 1); + } + at = (at + 1); + } + return hits; +} + +pub fn plant_at(text: &'static str, n: usize, from: &'static str) -> usize { + let stop: usize = line_stop(text, n); + let mut at: usize = line_start(text, n); + while (at < stop) { + if sits_at(text, at, stop, from) { + return at; + } + at = (at + 1); + } + return (text.len() + 1); +} + +pub fn same_text(a: &'static str, b: &'static str) -> bool { + if (a.len() != b.len()) { + return false; + } + return sits_at(a, 0, a.len(), b); +} + +pub fn breaks_line(s: &'static str) -> bool { + let mut k: usize = 0; + while (k < s.len()) { + if (s.as_bytes()[(k) as usize] == 10) { + return true; + } + k = (k + 1); + } + return false; +} + +pub fn plant_check(text: &'static str, n: usize, from: &'static str, to: &'static str) -> u8 { + if (from.len() == 0) { + return REFUSE_EMPTY_FROM; + } + if same_text(from, to) { + return REFUSE_SAME_TEXT; + } + if breaks_line(to) { + return REFUSE_LINE_BREAK; + } + if (line_start(text, n) > text.len()) { + return REFUSE_NO_SUCH_LINE; + } + if (occurrences(text, line_start(text, n), line_stop(text, n), from) != 1) { + return REFUSE_NOT_ONCE; + } + return PLANT_OK; +} + +pub fn plant_outcome(base_green: bool, expects_known: bool, mutant_read: bool, mutant_built: bool, fails: u64, expects: u64, as_expected: bool) -> u8 { + if (base_green == false) { + return BASELINE_RED; + } + if (expects_known == false) { + return NO_SUCH_TEST; + } + if (mutant_read == false) { + return MUTANT_UNREAD; + } + if (mutant_built == false) { + return UNVIABLE; + } + if (expects == 0) { + if (fails > 0) { + return KILLED; + } + return SURVIVED; + } + if (as_expected && (fails > 0)) { + return KILLED_AS_EXPECTED; + } + return NOT_AS_EXPECTED; +} + +pub fn plant_result(original_same: bool, outcome: u8) -> u8 { + if (original_same == false) { + return ORIGINAL_CHANGED; + } + return outcome; +} + +pub fn plant_passes(code: u8) -> bool { + return ((code == KILLED) || (code == KILLED_AS_EXPECTED)); +} + +pub fn why(code: u8) -> &'static str { + if (code == PLANT_OK) { + return "the plant is one line and one occurrence"; + } + if (code == REFUSE_EMPTY_FROM) { + return "--from is empty"; + } + if (code == REFUSE_SAME_TEXT) { + return "--from and --to are the same text, so nothing would change"; + } + if (code == REFUSE_LINE_BREAK) { + return "--to holds a line break: a planted mutant changes one line into one line"; + } + if (code == REFUSE_NO_SUCH_LINE) { + return "--line is not a line of the file (lines are numbered from 1)"; + } + if (code == REFUSE_NOT_ONCE) { + return "--from must occur exactly once on --line"; + } + if (code == KILLED) { + return "KILLED by:"; + } + if (code == KILLED_AS_EXPECTED) { + return "KILLED as expected by:"; + } + if (code == SURVIVED) { + return "SURVIVED: no test failed on the mutant"; + } + if (code == NOT_AS_EXPECTED) { + return "NOT as expected: the tests that FAIL on the mutant (above) are not exactly the --expect set"; + } + if (code == UNVIABLE) { + return "the mutant does not build, so no test judged it (unviable, not killed)"; + } + if (code == BASELINE_RED) { + return "the unmutated spec is not green, so the mutant would mean nothing"; + } + if (code == NO_SUCH_TEST) { + return "an --expect names no test of the spec (its tests are the baseline's `pass` names)"; + } + if (code == MUTANT_UNREAD) { + return "the mutant's report could not be read, so it is no verdict"; + } + return "ORIGINAL CHANGED: the spec's sha256 after the run is not the one before it; restore it from git before anything else"; +} + diff --git a/gen/rust/tri/test/report.rs b/gen/rust/tri/test/report.rs new file mode 100644 index 0000000000..b81bd229c5 --- /dev/null +++ b/gen/rust/tri/test/report.rs @@ -0,0 +1,346 @@ +// Generated from .t27 spec +// DO NOT EDIT — generated by t27c +// NOT LOWERED BY THIS BACKEND: 13 test(s), 1 invariant(s), 0 function(s) with no body. +// This backend emits declarations only. The spec's checks live in +// the Zig and Verilog outputs; do not read this file as verified. + +pub const LINE_OTHER: u8 = 0; + +pub const LINE_BLOCKED: u8 = 1; + +pub const LINE_TESTS: u8 = 2; + +pub const LINE_PASS: u8 = 3; + +pub const LINE_FAIL: u8 = 4; + +pub const LINE_FAIL_NAME: u8 = 5; + +pub const LINE_PASS_NAME: u8 = 6; + +pub const MAX_TOTAL_DIGITS: usize = 18; + +pub const V_GREEN: u8 = 0; + +pub const V_FAILED: u8 = 1; + +pub const V_BLOCKED: u8 = 2; + +pub const V_NO_TEST: u8 = 3; + +pub const V_NO_TOTALS: u8 = 4; + +pub const V_TOTALS_DISAGREE: u8 = 5; + +pub const V_NAMES_DISAGREE: u8 = 6; + +pub const V_EXIT_GREEN: u8 = 7; + +pub const V_EXIT_UNREAD: u8 = 8; + +pub fn is_blank(c: u8) -> bool { + return ((c == 32) || ((c >= 9) && (c <= 13))); +} + +pub fn line_end(text: &'static str, at: usize) -> usize { + let mut k: usize = at; + while ((k < text.len()) && (text.as_bytes()[(k) as usize] != 10)) { + k = (k + 1); + } + return k; +} + +pub fn opens_with(text: &'static str, at: usize, end: usize, lit: &'static str) -> bool { + if ((at > end) || (lit.len() > (end - at))) { + return false; + } + let mut k: usize = 0; + while (k < lit.len()) { + if (text.as_bytes()[((at + k)) as usize] != lit.as_bytes()[(k) as usize]) { + return false; + } + k = (k + 1); + } + return true; +} + +pub fn trimmed_end(text: &'static str, from: usize, end: usize) -> usize { + let mut e: usize = end; + while ((e > from) && is_blank(text.as_bytes()[((e - 1)) as usize])) { + e = (e - 1); + } + return e; +} + +pub fn total_digits(text: &'static str, at: usize, end: usize, word: &'static str) -> usize { + if (opens_with(text, at, end, word) == false) { + return (end + 1); + } + let mut k: usize = (at + word.len()); + while ((k < end) && (text.as_bytes()[(k) as usize] == 32)) { + k = (k + 1); + } + if ((k - (at + word.len())) < 3) { + return (end + 1); + } + let digits: usize = k; + while (((k < end) && (text.as_bytes()[(k) as usize] >= 48)) && (text.as_bytes()[(k) as usize] <= 57)) { + k = (k + 1); + } + if ((k == digits) || ((k - digits) > MAX_TOTAL_DIGITS)) { + return (end + 1); + } + if (trimmed_end(text, k, end) != k) { + return (end + 1); + } + return digits; +} + +pub fn names_after(text: &'static str, at: usize, end: usize, word: &'static str) -> bool { + if (opens_with(text, at, end, word) == false) { + return false; + } + if ((at + word.len()) >= end) { + return false; + } + return (is_blank(text.as_bytes()[((at + word.len())) as usize]) == false); +} + +pub fn line_kind(text: &'static str, at: usize, end: usize) -> u8 { + if opens_with(text, at, end, " BLOCKED ") { + return LINE_BLOCKED; + } + if (total_digits(text, at, end, " tests") <= end) { + return LINE_TESTS; + } + if (total_digits(text, at, end, " pass") <= end) { + return LINE_PASS; + } + if (total_digits(text, at, end, " FAIL") <= end) { + return LINE_FAIL; + } + if names_after(text, at, end, " FAIL ") { + return LINE_FAIL_NAME; + } + if names_after(text, at, end, " pass ") { + return LINE_PASS_NAME; + } + return LINE_OTHER; +} + +pub fn line_of(text: &'static str, kind: u8, n: u64) -> usize { + let mut at: usize = 0; + let mut seen: u64 = 0; + while (at < text.len()) { + let end: usize = line_end(text, at); + if (line_kind(text, at, end) == kind) { + if (seen == n) { + return at; + } + seen = (seen + 1); + } + at = (end + 1); + } + return (text.len() + 1); +} + +pub fn count(text: &'static str, kind: u8) -> u64 { + let mut at: usize = 0; + let mut seen: u64 = 0; + while (at < text.len()) { + let end: usize = line_end(text, at); + if (line_kind(text, at, end) == kind) { + seen = (seen + 1); + } + at = (end + 1); + } + return seen; +} + +pub fn total(text: &'static str, kind: u8) -> u64 { + let at: usize = line_of(text, kind, 0); + if (at > text.len()) { + return 0; + } + let mut word: &'static str = " tests"; + if (kind == LINE_PASS) { + word = " pass"; + } + if (kind == LINE_FAIL) { + word = " FAIL"; + } + let end: usize = line_end(text, at); + let mut k: usize = total_digits(text, at, end, word); + let mut value: u64 = 0; + while (((k < end) && (text.as_bytes()[(k) as usize] >= 48)) && (text.as_bytes()[(k) as usize] <= 57)) { + value = ((value * 10) + ((text.as_bytes()[(k) as usize] as u64) - 48)); + k = (k + 1); + } + return value; +} + +pub fn name_from(text: &'static str, kind: u8, n: u64) -> usize { + let at: usize = line_of(text, kind, n); + if (at > text.len()) { + return text.len(); + } + if (kind == LINE_BLOCKED) { + return (at + 11); + } + return (at + 8); +} + +pub fn name_to(text: &'static str, kind: u8, n: u64) -> usize { + let from: usize = name_from(text, kind, n); + if (from >= text.len()) { + return text.len(); + } + return trimmed_end(text, from, line_end(text, from)); +} + +pub fn read(text: &'static str) -> u8 { + if (count(text, LINE_BLOCKED) > 0) { + return V_BLOCKED; + } + if (((count(text, LINE_TESTS) == 0) || (count(text, LINE_PASS) == 0)) || (count(text, LINE_FAIL) == 0)) { + return V_NO_TOTALS; + } + let tests: u64 = total(text, LINE_TESTS); + let failed: u64 = total(text, LINE_FAIL); + if ((total(text, LINE_PASS) + failed) != tests) { + return V_TOTALS_DISAGREE; + } + if (count(text, LINE_FAIL_NAME) != failed) { + return V_NAMES_DISAGREE; + } + if (failed > 0) { + return V_FAILED; + } + if (tests == 0) { + return V_NO_TEST; + } + return V_GREEN; +} + +pub fn verdict(text: &'static str, exited_ok: bool) -> u8 { + let r: u8 = read(text); + if exited_ok { + return r; + } + if ((r == V_FAILED) || (r == V_BLOCKED)) { + return r; + } + if ((r == V_GREEN) || (r == V_NO_TEST)) { + return V_EXIT_GREEN; + } + return V_EXIT_UNREAD; +} + +pub fn is_verdict(v: u8) -> bool { + return (v <= V_NO_TEST); +} + +pub fn passes(v: u8) -> bool { + return (v == V_GREEN); +} + +pub fn why(v: u8) -> &'static str { + if (v == V_GREEN) { + return "green: it built, at least one test ran, and none failed"; + } + if (v == V_FAILED) { + return "a test FAILs"; + } + if (v == V_BLOCKED) { + return "BLOCKED: the spec never built, so no test ran"; + } + if (v == V_NO_TEST) { + return "no test ran: the spec has no `test` block (an invariant is proved at compile time; it is not a test)"; + } + if (v == V_NO_TOTALS) { + return "t27c test-report printed no tests / pass / FAIL totals and no BLOCKED line"; + } + if (v == V_TOTALS_DISAGREE) { + return "t27c test-report's totals do not add up: pass + FAIL is not tests"; + } + if (v == V_NAMES_DISAGREE) { + return "t27c test-report counts a number of FAIL that is not the number of FAIL names it lists"; + } + if (v == V_EXIT_GREEN) { + return "t27c test-report exited non-zero, yet its report reads green or ran no test; a run that disagrees with itself is not a pass"; + } + return "t27c test-report exited non-zero and no report could be read from its output"; +} + +pub fn entry_from(list: &'static str, k: u64) -> usize { + let mut at: usize = 0; + let mut seen: u64 = 0; + while ((seen < k) && (at < list.len())) { + if (list.as_bytes()[(at) as usize] == 10) { + seen = (seen + 1); + } + at = (at + 1); + } + return at; +} + +pub fn is_entry(text: &'static str, from: usize, to: usize, list: &'static str, k: u64) -> bool { + let at: usize = entry_from(list, k); + let end: usize = line_end(list, at); + if ((end - at) != (to - from)) { + return false; + } + let mut i: usize = 0; + while (i < (to - from)) { + if (text.as_bytes()[((from + i)) as usize] != list.as_bytes()[((at + i)) as usize]) { + return false; + } + i = (i + 1); + } + return true; +} + +pub fn named(text: &'static str, kind: u8, list: &'static str, k: u64) -> bool { + let mut n: u64 = 0; + while (n < count(text, kind)) { + if is_entry(text, name_from(text, kind, n), name_to(text, kind, n), list, k) { + return true; + } + n = (n + 1); + } + return false; +} + +pub fn all_named(text: &'static str, kind: u8, list: &'static str, entries: u64) -> bool { + let mut k: u64 = 0; + while (k < entries) { + if (named(text, kind, list, k) == false) { + return false; + } + k = (k + 1); + } + return true; +} + +pub fn same_names(text: &'static str, kind: u8, list: &'static str, entries: u64) -> bool { + if (all_named(text, kind, list, entries) == false) { + return false; + } + let mut n: u64 = 0; + while (n < count(text, kind)) { + let mut found: bool = false; + let mut k: u64 = 0; + while (k < entries) { + if is_entry(text, name_from(text, kind, n), name_to(text, kind, n), list, k) { + found = true; + } + k = (k + 1); + } + if (found == false) { + return false; + } + n = (n + 1); + } + return true; +} + diff --git a/specs/tools/tri/mutate.t27 b/specs/tools/tri/mutate.t27 index 5698a388c0..c2ecadc5d0 100644 --- a/specs/tools/tri/mutate.t27 +++ b/specs/tools/tri/mutate.t27 @@ -26,7 +26,7 @@ pub const ABOUT : str = "Find the constants in a checker that nothing actually c pub const ABOUT_SOURCE : str = "clap /// doc on Commands::Mutate (cli/tri/src/main.rs; module cli/tri/src/mutate.rs)"; ; Nested subcommands (kebab-case as clap derives them) and their first /// paragraph ("" when none). pub const ACTIONS : [4]str = ["run", "spec", "plant", "census"]; -pub const ACTIONS_ABOUT : [4]str = ["Perturb each numeric literal in a file and report which ones the checker does not notice.", "Drop guards, flip operators and empty whole bodies inside the functions of a .t27 spec, and report the mutants its own tests do not notice.", "Plant one hand-written mutant in a .t27 spec (--from becomes --to on --line) and name the spec's tests that go red on it.", "`mutate spec` on each .t27 in --dir by name, then one exit and killed share (rules: specs/tri/mutate/census.t27); every arg after --dir goes to each run."]; +pub const ACTIONS_ABOUT : [4]str = ["Perturb each numeric literal in a file and report which ones the checker does not notice.", "Drop guards, flip operators and empty whole bodies inside the functions of a .t27 spec, and report the mutants its own tests do not notice.", "Plant one hand-written mutant in a .t27 spec (--from becomes --to on --line) and name the spec's tests that go red on it (rules: specs/tri/mutate/plant.t27).", "`mutate spec` on each .t27 in --dir by name, then one exit and killed share (rules: specs/tri/mutate/census.t27); every arg after --dir goes to each run."]; ; Top-level arguments/flags of the command itself, ": doc". pub const ARGS : [0]str = []; ; Agent letters that own this command, only where a source line names it (AGENTS_NOTE cites it). diff --git a/specs/tools/tri/test.t27 b/specs/tools/tri/test.t27 index 38ef194046..89fd01b0c0 100644 --- a/specs/tools/tri/test.t27 +++ b/specs/tools/tri/test.t27 @@ -5,7 +5,8 @@ ; cargo was not run, the enum and its /// docs were read at commit 3cb7a4352492; the Test ; variant's doc, its --t27c flag and the test-report run were read from the branch that adds ; them, gHashTag/t27#7369. Before it, cmd_test ran `t27c test`, which only lists the tests, -; and printed "tests passed" whatever they did). +; and printed "tests passed" whatever they did. The run prints the report, then one line +; `tests: N, pass: P, fail: F (names)`, and exits non-zero unless the report reads green). ; Layer 5 of the ladder Specs -> Skills -> Crons -> Agents -> Tools. ASCII only (L3). ; phi^2 + 1/phi^2 = 3 | TRINITY @@ -23,8 +24,8 @@ pub const VARIANT : str = "Commands::Test"; pub const SOURCE : str = "cli/tri/src/main.rs"; pub const ENTRY : str = "cli/tri/src/main.rs"; ; The clap about text (first /// paragraph), as `tri --help` would print it. -pub const ABOUT : str = "Run a .t27 spec's tests with `t27c test-report` and fail unless every test passed and at least one ran."; -pub const ABOUT_SOURCE : str = "clap /// doc on Commands::Test (cli/tri/src/main.rs); handler cmd_test calls mutate::test_spec (cli/tri/src/mutate.rs), which runs t27c test-report and reads the verdict from its text, since test-report exits 0 when a test FAILs (gHashTag/t27#7370)"; +pub const ABOUT : str = "Run a .t27 spec's tests with `t27c test-report` and fail unless every test passed and at least one ran (rules: specs/tri/test/report.t27)."; +pub const ABOUT_SOURCE : str = "clap /// doc on Commands::Test (cli/tri/src/main.rs); handler cmd_test runs t27c test-report and reads the verdict from its text through gen/rust/tri/test/report.rs, the t27c gen-rust of specs/tri/test/report.t27, since test-report exits 0 when a test FAILs (gHashTag/t27#7370)"; ; Nested subcommands (kebab-case as clap derives them) and their first /// paragraph ("" when none). pub const ACTIONS : [0]str = []; pub const ACTIONS_ABOUT : [0]str = []; @@ -34,7 +35,7 @@ pub const ARGS : [2]str = ["", "--t27c: The t27c binary; default targ pub const AGENTS : [2]str = ["T", "V"]; pub const AGENTS_NOTE : str = "docs/agents/AGENTS_ALPHABET.md \"Who holds a tool, and who holds a skill\" binds every tool to the Queen (T). Domain lead V: The spec's own AGENTS field is [\"T\",\"V\"] with AGENTS_NOTE citing .claude/agents/agent-v-verify.md:47 (\"tri test -- Run conformance tests\"); with T excluded by rule, V is the domain lead."; ; From the about text (long about when the doc has more than one paragraph); no marketing. -pub const WHEN_TO_USE : str = "Run a .t27 spec's tests with `t27c test-report` and fail unless every test passed and at least one ran. Prints the report, then one line `tests: N, pass: P, fail: F (names)`. Exits non-zero when a test FAILs, when the spec is BLOCKED, or when no test ran."; +pub const WHEN_TO_USE : str = "Run a .t27 spec's tests with `t27c test-report` and fail unless every test passed and at least one ran (rules: specs/tri/test/report.t27)."; pub const WITNESS : str = "source-parse"; pub const ENABLED : bool = true; diff --git a/specs/tri/mutate/plant.t27 b/specs/tri/mutate/plant.t27 new file mode 100644 index 0000000000..47542a0d4d --- /dev/null +++ b/specs/tri/mutate/plant.t27 @@ -0,0 +1,315 @@ +// SPDX-License-Identifier: Apache-2.0 +; specs/tri/mutate/plant.t27 -- one hand-written mutation experiment: `tri mutate plant` (#7369) +; `tri mutate spec` plants mutants by kind; `tri mutate plant --file F --line N --from OLD --to NEW +; [--expect TEST]...` plants ONE named mutant and says which tests of the spec go red on it. Before it, +; such an experiment was a sed on a scratch copy and a reading of the report by eye (the acceptance demo +; of gHashTag/t27#7400). This spec holds every decision of the command; the command +; (cli/tri/src/mutate.rs, through `t27c gen-rust` in gen/rust/tri/mutate/plant.rs) is glue: it reads +; the file, writes the copies into the work dir `tri mutate spec` uses, runs `t27c test-report` on a +; clean copy and on the mutant, reads both through specs/tri/test/report.t27, hashes the original +; before and after, and prints. +; Four rules: +; the plant is the one named -- --from occurs EXACTLY once on --line (overlapping occurrences +; count), --from is not empty, --to is not --from and holds no line break, and --line is a line of +; the file (from 1). Anything else is refused before t27c runs. A line ends at its '\n'; a '\r' +; is an ordinary byte here: gen (Zig) and gen-rust lower a '\r' escape differently, so no test could hold one. +; a red baseline means nothing -- against a spec that is not green before the plant every mutant +; "is killed"; the run is refused, and so is an --expect that names no test of the spec. +; the verdict -- a mutant that does not build is unviable, not killed. Without --expect the plant +; passes when a test FAILs; with it, only when the failing tests are exactly the --expect set. +; the original is never written -- its sha256 is taken before and after the run, and a changed +; original fails the command whatever the tests said. +; ASCII only (L3) +; phi^2 + 1/phi^2 = 3 | TRINITY + +module TriMutatePlant; + +// Refusals of the plant itself; 0 means the plant is the one named. +pub const PLANT_OK : u8 = 0; +pub const REFUSE_EMPTY_FROM : u8 = 1; +pub const REFUSE_SAME_TEXT : u8 = 2; +pub const REFUSE_LINE_BREAK : u8 = 3; +pub const REFUSE_NO_SUCH_LINE : u8 = 4; +pub const REFUSE_NOT_ONCE : u8 = 5; + +// Outcomes of a run. Only KILLED and KILLED_AS_EXPECTED pass. +pub const KILLED : u8 = 10; +pub const KILLED_AS_EXPECTED : u8 = 11; +pub const SURVIVED : u8 = 12; +pub const NOT_AS_EXPECTED : u8 = 13; +pub const UNVIABLE : u8 = 14; +pub const BASELINE_RED : u8 = 15; +pub const NO_SUCH_TEST : u8 = 16; +pub const MUTANT_UNREAD : u8 = 17; +pub const ORIGINAL_CHANGED : u8 = 18; + +/// Refusals sit below the outcomes, so one `why` names both. +invariant refusals_and_outcomes_do_not_overlap { assert PLANT_OK == 0 and REFUSE_NOT_ONCE == 5 and KILLED == 10 and ORIGINAL_CHANGED == 18; } + +/// How many lines the text has: one per '\n', and one more for a last line with no '\n'. +pub fn line_count(text: string) -> usize { + var lines : usize = 0; + var k : usize = 0; + while (k < text.len) { + if (text[k] == 10) { lines = lines + 1; } + k = k + 1; + } + if (text.len > 0 and text[text.len - 1] != 10) { lines = lines + 1; } + return lines; +} + +/// Where line n (from 1) starts, or text.len + 1 when the text has no line n. +pub fn line_start(text: string, n: usize) -> usize { + if (n == 0 or n > line_count(text)) { return text.len + 1; } + var seen : usize = 1; + var k : usize = 0; + while (seen < n) { + if (text[k] == 10) { seen = seen + 1; } + k = k + 1; + } + return k; +} + +/// Where line n ends: its '\n', or the end of the text. +pub fn line_stop(text: string, n: usize) -> usize { + const start : usize = line_start(text, n); + if (start > text.len) { return start; } + var k : usize = start; + while (k < text.len and text[k] != 10) { + k = k + 1; + } + return k; +} + +/// Does lit sit in text at offset at, wholly before stop? +pub fn sits_at(text: string, at: usize, stop: usize, lit: string) -> bool { + if (at > stop or lit.len > stop - at) { return false; } + var i : usize = 0; + while (i < lit.len) { + if (text[at + i] != lit[i]) { return false; } + i = i + 1; + } + return true; +} + +/// How many times lit occurs in text[from..stop], overlapping ones counted: `aa` sits twice in `aaa`. +pub fn occurrences(text: string, from: usize, stop: usize, lit: string) -> u64 { + var hits : u64 = 0; + var at : usize = from; + while (at < stop) { + if (sits_at(text, at, stop, lit)) { hits = hits + 1; } + at = at + 1; + } + return hits; +} + +/// Where --from sits on line n: the offset the glue splices --to into. text.len + 1 when it does not. +pub fn plant_at(text: string, n: usize, from: string) -> usize { + const stop : usize = line_stop(text, n); + var at : usize = line_start(text, n); + while (at < stop) { + if (sits_at(text, at, stop, from)) { return at; } + at = at + 1; + } + return text.len + 1; +} + +/// Are a and b the same bytes? +pub fn same_text(a: string, b: string) -> bool { + if (a.len != b.len) { return false; } + return sits_at(a, 0, a.len, b); +} + +/// Does s hold a '\n'? +pub fn breaks_line(s: string) -> bool { + var k : usize = 0; + while (k < s.len) { + if (s[k] == 10) { return true; } + k = k + 1; + } + return false; +} + +/// Is `--line n --from from --to to` on text the one named plant? PLANT_OK, or why not. +pub fn plant_check(text: string, n: usize, from: string, to: string) -> u8 { + if (from.len == 0) { return REFUSE_EMPTY_FROM; } + if (same_text(from, to)) { return REFUSE_SAME_TEXT; } + if (breaks_line(to)) { return REFUSE_LINE_BREAK; } + if (line_start(text, n) > text.len) { return REFUSE_NO_SUCH_LINE; } + if (occurrences(text, line_start(text, n), line_stop(text, n), from) != 1) { return REFUSE_NOT_ONCE; } + return PLANT_OK; +} + +/// The outcome of a run, from what the two reports said (specs/tri/test/report.t27): base_green, the +/// clean copy's verdict is GREEN; expects_known, every --expect is a test the baseline named; +/// mutant_read, the mutant's report is a verdict; mutant_built, that verdict is not BLOCKED; fails, the +/// tests that FAIL on the mutant; expects, how many --expect; as_expected, the FAIL names are exactly +/// the --expect set. The checks go in the order the run meets them. +pub fn plant_outcome(base_green: bool, expects_known: bool, mutant_read: bool, mutant_built: bool, fails: u64, expects: u64, as_expected: bool) -> u8 { + if (base_green == false) { return BASELINE_RED; } + if (expects_known == false) { return NO_SUCH_TEST; } + if (mutant_read == false) { return MUTANT_UNREAD; } + if (mutant_built == false) { return UNVIABLE; } + if (expects == 0) { + if (fails > 0) { return KILLED; } + return SURVIVED; + } + if (as_expected and fails > 0) { return KILLED_AS_EXPECTED; } + return NOT_AS_EXPECTED; +} + +/// The original's sha256 after the run is compared with the one before it: a changed original fails +/// the command whatever the outcome was. +pub fn plant_result(original_same: bool, outcome: u8) -> u8 { + if (original_same == false) { return ORIGINAL_CHANGED; } + return outcome; +} + +/// The command exits 0 only on a kill: any, without --expect; exactly the expected ones, with it. +pub fn plant_passes(code: u8) -> bool { + return code == KILLED or code == KILLED_AS_EXPECTED; +} + +/// What a refusal or an outcome means. The two kills are followed by the FAIL names. +pub fn why(code: u8) -> string { + if (code == PLANT_OK) { return "the plant is one line and one occurrence"; } + if (code == REFUSE_EMPTY_FROM) { return "--from is empty"; } + if (code == REFUSE_SAME_TEXT) { return "--from and --to are the same text, so nothing would change"; } + if (code == REFUSE_LINE_BREAK) { return "--to holds a line break: a planted mutant changes one line into one line"; } + if (code == REFUSE_NO_SUCH_LINE) { return "--line is not a line of the file (lines are numbered from 1)"; } + if (code == REFUSE_NOT_ONCE) { return "--from must occur exactly once on --line"; } + if (code == KILLED) { return "KILLED by:"; } + if (code == KILLED_AS_EXPECTED) { return "KILLED as expected by:"; } + if (code == SURVIVED) { return "SURVIVED: no test failed on the mutant"; } + if (code == NOT_AS_EXPECTED) { return "NOT as expected: the tests that FAIL on the mutant (above) are not exactly the --expect set"; } + if (code == UNVIABLE) { return "the mutant does not build, so no test judged it (unviable, not killed)"; } + if (code == BASELINE_RED) { return "the unmutated spec is not green, so the mutant would mean nothing"; } + if (code == NO_SUCH_TEST) { return "an --expect names no test of the spec (its tests are the baseline's `pass` names)"; } + if (code == MUTANT_UNREAD) { return "the mutant's report could not be read, so it is no verdict"; } + return "ORIGINAL CHANGED: the spec's sha256 after the run is not the one before it; restore it from git before anything else"; +} + +// Scenario: the acceptance demo of #7400 -- line 12 of specs/ternary/gft_relu.t27, `return 0;` for a +// negative input becomes `return x;`. Lines 10 to 13 of that spec, as they stand. +test the_gft_relu_plant_is_one_line_and_one_occurrence { + const t : string = "fn on_comb(x: u32) -> u32 {\n if (x == 0) { return 0; }\n if ((x >> 16) == 1) { return 0; } // negative -> 0\n return x; // positive -> identity\n}\n"; + assert line_count(t) == 5; + assert line_start(t, 3) == 58; + assert line_stop(t, 3) == 114; + assert plant_check(t, 3, "return 0; } // negative", "return x; } // negative") == PLANT_OK; + assert plant_at(t, 3, "return 0; } // negative") == 84; + assert plant_check(t, 3, "return 0;", "return x;") == PLANT_OK; + assert plant_check(t, 2, "return 0;", "return x;") == PLANT_OK; + assert plant_at(t, 2, "return 0;") == 46; +} + +// Scenario: --from absent from the line, twice on it, or overlapping itself is not the plant named. +test from_must_occur_exactly_once_on_the_line { + const t : string = "module m;\nfn f(x: i32) -> i32 {\n if (x < 0) { return 0; } // negative -> 0\n return x;\n}\n"; + assert plant_check(t, 4, "{ return 0; }", "x") == REFUSE_NOT_ONCE; + assert plant_check(t, 2, "i32", "u32") == REFUSE_NOT_ONCE; + assert occurrences(t, line_start(t, 2), line_stop(t, 2), "i32") == 2; + assert plant_check(t, 3, "00", "1") == REFUSE_NOT_ONCE; + assert plant_check("aaa\n", 1, "aa", "b") == REFUSE_NOT_ONCE; + assert occurrences("aaa\n", 0, 3, "aa") == 2; + assert plant_check("aab\n", 1, "aa", "b") == PLANT_OK; + assert plant_check("ab\nab\n", 2, "ab", "x") == PLANT_OK; + assert plant_at("ab\nab\n", 2, "ab") == 3; + assert plant_at("ab\nab\n", 2, "zz") == 7; +} + +// Scenario: the other refusals, each before t27c runs. +test the_plant_refuses_what_is_not_one_line_into_one_line { + const t : string = "module m;\nfn f() {}\n"; + assert plant_check(t, 1, "", "x") == REFUSE_EMPTY_FROM; + assert plant_check(t, 1, "m;", "m;") == REFUSE_SAME_TEXT; + assert plant_check(t, 1, "m;", "a\nb") == REFUSE_LINE_BREAK; + assert plant_check(t, 0, "m;", "n;") == REFUSE_NO_SUCH_LINE; + assert plant_check(t, 3, "m;", "n;") == REFUSE_NO_SUCH_LINE; + assert plant_check(t, 2, "f()", "g()") == PLANT_OK; + assert plant_check(t, 1, "m;", "M;") == PLANT_OK; + assert breaks_line("a\tb") == false; + assert breaks_line("") == false; + assert same_text("ab", "abc") == false; + assert same_text("", "") == true; +} + +// Scenario: lines are counted as an editor numbers them, with or without a last '\n'. +test lines_are_numbered_from_1 { + assert line_count("") == 0; + assert line_count("a") == 1; + assert line_count("a\n") == 1; + assert line_count("a\nb") == 2; + assert line_count("\n\n") == 2; + assert line_start("a\nb", 2) == 2; + assert line_stop("a\nb", 2) == 3; + assert line_start("a\nb", 3) == 4; + assert line_stop("a\nb", 3) == 4; + assert line_start("a\n\nc\n", 2) == 2; + assert line_stop("a\n\nc\n", 2) == 2; + assert sits_at("ab", 1, 2, "b") == true; + assert sits_at("ab", 1, 2, "bc") == false; + assert sits_at("ab", 3, 2, "") == false; +} + +// Scenario: the #7400 demo -- --expect negz, the mutant FAILs negz only: KILLED as expected; with +// --expect pos the same mutant is NOT as expected; without --expect it is KILLED. +test the_verdict_follows_expect { + assert plant_outcome(true, true, true, true, 1, 1, true) == KILLED_AS_EXPECTED; + assert plant_outcome(true, true, true, true, 1, 1, false) == NOT_AS_EXPECTED; + assert plant_outcome(true, true, true, true, 2, 1, false) == NOT_AS_EXPECTED; + assert plant_outcome(true, true, true, true, 1, 0, false) == KILLED; + assert plant_outcome(true, true, true, true, 0, 0, false) == SURVIVED; + assert plant_outcome(true, true, true, true, 0, 1, false) == NOT_AS_EXPECTED; + assert plant_outcome(true, true, true, true, 0, 1, true) == NOT_AS_EXPECTED; + assert plant_passes(KILLED_AS_EXPECTED) == true; + assert plant_passes(KILLED) == true; + assert plant_passes(SURVIVED) == false; + assert plant_passes(NOT_AS_EXPECTED) == false; +} + +// Scenario: a red baseline is refused before anything else, an unknown --expect next; a mutant that +// does not build is unviable, and one whose report cannot be read is no verdict -- never a kill. +test what_the_run_meets_first_decides { + assert plant_outcome(false, true, true, true, 1, 0, false) == BASELINE_RED; + assert plant_outcome(false, false, false, false, 1, 1, true) == BASELINE_RED; + assert plant_outcome(true, false, true, true, 1, 1, true) == NO_SUCH_TEST; + assert plant_outcome(true, true, false, false, 1, 0, false) == MUTANT_UNREAD; + assert plant_outcome(true, true, true, false, 0, 0, false) == UNVIABLE; + assert plant_outcome(true, true, true, false, 0, 1, true) == UNVIABLE; + assert plant_passes(BASELINE_RED) == false; + assert plant_passes(NO_SUCH_TEST) == false; + assert plant_passes(UNVIABLE) == false; + assert plant_passes(MUTANT_UNREAD) == false; +} + +// Scenario: a run that wrote to the original fails even when its tests said exactly what was +// expected; an untouched original keeps the outcome, whatever it is. +test a_changed_original_fails_whatever_the_tests_said { + assert plant_result(false, KILLED_AS_EXPECTED) == ORIGINAL_CHANGED; + assert plant_result(false, SURVIVED) == ORIGINAL_CHANGED; + assert plant_result(true, KILLED_AS_EXPECTED) == KILLED_AS_EXPECTED; + assert plant_result(true, BASELINE_RED) == BASELINE_RED; + assert plant_passes(plant_result(false, KILLED)) == false; + assert plant_passes(ORIGINAL_CHANGED) == false; + assert plant_passes(PLANT_OK) == false; +} + +// Scenario: every code has its own words; the two kills are the ones the FAIL names follow. +test each_code_says_what_it_means { + assert why(PLANT_OK)[4] == 112; + assert why(REFUSE_EMPTY_FROM)[7] == 105; + assert why(REFUSE_SAME_TEXT)[7] == 97; + assert why(REFUSE_LINE_BREAK)[2] == 116; + assert why(REFUSE_NO_SUCH_LINE)[2] == 108; + assert why(REFUSE_NOT_ONCE)[7] == 109; + assert why(KILLED).len == 10; + assert why(KILLED_AS_EXPECTED).len == 22; + assert why(SURVIVED)[0] == 83; + assert why(NOT_AS_EXPECTED)[0] == 78; + assert why(UNVIABLE)[10] == 32; + assert why(BASELINE_RED)[4] == 117; + assert why(NO_SUCH_TEST)[3] == 45; + assert why(MUTANT_UNREAD)[10] == 39; + assert why(ORIGINAL_CHANGED)[0] == 79; +} diff --git a/specs/tri/test/report.t27 b/specs/tri/test/report.t27 new file mode 100644 index 0000000000..ee28e72b1a --- /dev/null +++ b/specs/tri/test/report.t27 @@ -0,0 +1,505 @@ +// SPDX-License-Identifier: Apache-2.0 +; specs/tri/test/report.t27 -- what `t27c test-report ` said about a spec's tests (#7369) +; `tri test ` used to run `t27c test`, which only LISTS the tests, and then printed "tests passed" +; whatever they did (gHashTag/t27#7369). The tests run in `t27c test-report`, and its exit code is 0 +; even when a test FAILs and when the spec is BLOCKED (#7370), so the verdict is read from the report's +; TEXT. This spec holds that reading and the verdict; the command (`tri test`, cli/tri/src/main.rs, +; through `t27c gen-rust` in gen/rust/tri/test/report.rs) is glue: it runs t27c, hands the text and +; whether t27c exited 0 to `verdict`, and prints. `tri mutate plant` (specs/tri/mutate/plant.t27) +; reads its baseline and its mutant through the same functions. +; The report, as bootstrap/src/main.rs `run_test_report` prints it: +; --- test report: --- +; BLOCKED (then nothing about tests: the spec never built) +; FAIL (one per failing test) +; pass (one per passing test, only with --verbose) +; -- +; tests N +; pass P +; FAIL F +; invariants I proved -- ... (an invariant is checked at compile time; it is not a test) +; and after the totals, per-test assert counts whose lines start with a number or '-'. +; Two rules: +; fail closed -- a report with no totals and no BLOCKED line, totals that do not add up +; (pass + FAIL != tests), or a FAIL count that is not the number of FAIL names it lists, is not a +; verdict: it is an error, never a pass. +; the exit agrees -- test-report exits 0 today on FAIL and on BLOCKED; #7370 makes it exit non-zero +; on both. A red report with a non-zero exit is that red verdict. A non-zero exit with a report that +; reads green (or ran no test) disagrees with itself, and a non-zero exit with no readable report is +; an error: neither is a pass. +; Fixtures are test-report's real text for specs/ternary/gft_relu.t27 (2026-10-08, t27c from master). +; ASCII only (L3) +; phi^2 + 1/phi^2 = 3 | TRINITY + +module TriTestReport; + +// What one line of the report is. A line is the bytes up to its '\n' (or the end of the text). +pub const LINE_OTHER : u8 = 0; +pub const LINE_BLOCKED : u8 = 1; +pub const LINE_TESTS : u8 = 2; +pub const LINE_PASS : u8 = 3; +pub const LINE_FAIL : u8 = 4; +pub const LINE_FAIL_NAME : u8 = 5; +pub const LINE_PASS_NAME : u8 = 6; + +// A total of more digits than this is not one t27c printed; 18 digits always fit in u64. +pub const MAX_TOTAL_DIGITS : usize = 18; + +// The verdict on one run. GREEN, FAILED, BLOCKED and NO_TEST are verdicts about the tests; the codes +// from NO_TOTALS up say the report itself cannot be believed. +pub const V_GREEN : u8 = 0; +pub const V_FAILED : u8 = 1; +pub const V_BLOCKED : u8 = 2; +pub const V_NO_TEST : u8 = 3; +pub const V_NO_TOTALS : u8 = 4; +pub const V_TOTALS_DISAGREE : u8 = 5; +pub const V_NAMES_DISAGREE : u8 = 6; +pub const V_EXIT_GREEN : u8 = 7; +pub const V_EXIT_UNREAD : u8 = 8; + +/// The kinds and verdicts are numbered as the glue and the tests below name them. +invariant the_kinds_and_verdicts_are_numbered { assert LINE_OTHER == 0 and LINE_PASS_NAME == 6 and V_GREEN == 0 and V_NO_TEST == 3 and V_EXIT_UNREAD == 8 and MAX_TOTAL_DIGITS == 18; } + +/// Whitespace as a regex's \s reads ASCII: space, tab, newline, vertical tab, form feed, return. +pub fn is_blank(c: u8) -> bool { + return c == 32 or (c >= 9 and c <= 13); +} + +/// The end of the line that starts at `at`: the offset of its '\n', or text.len. +pub fn line_end(text: string, at: usize) -> usize { + var k : usize = at; + while (k < text.len and text[k] != 10) { + k = k + 1; + } + return k; +} + +/// Does the line text[at..end] start with lit? +pub fn opens_with(text: string, at: usize, end: usize, lit: string) -> bool { + if (at > end or lit.len > end - at) { return false; } + var k : usize = 0; + while (k < lit.len) { + if (text[at + k] != lit[k]) { return false; } + k = k + 1; + } + return true; +} + +/// text[from..end] without its trailing blanks: the offset just past the last non-blank, never before from. +pub fn trimmed_end(text: string, from: usize, end: usize) -> usize { + var e : usize = end; + while (e > from and is_blank(text[e - 1])) { + e = e - 1; + } + return e; +} + +/// A total line ``, then 3 or more spaces, 1 to MAX_TOTAL_DIGITS digits and only blanks to the +/// end of the line. Returns the offset of the digits, or end + 1 when the line is not that total. +pub fn total_digits(text: string, at: usize, end: usize, word: string) -> usize { + if (opens_with(text, at, end, word) == false) { return end + 1; } + var k : usize = at + word.len; + while (k < end and text[k] == 32) { + k = k + 1; + } + if (k - (at + word.len) < 3) { return end + 1; } + const digits : usize = k; + while (k < end and text[k] >= 48 and text[k] <= 57) { + k = k + 1; + } + if (k == digits or k - digits > MAX_TOTAL_DIGITS) { return end + 1; } + if (trimmed_end(text, k, end) != k) { return end + 1; } + return digits; +} + +/// A name line `` followed by a character that is not blank: the name starts there. +pub fn names_after(text: string, at: usize, end: usize, word: string) -> bool { + if (opens_with(text, at, end, word) == false) { return false; } + if (at + word.len >= end) { return false; } + return is_blank(text[at + word.len]) == false; +} + +/// The kind of the line text[at..end]. A total is tried before a name, so ` pass 3` is a total +/// and ` pass pos` a name; the per-test assert lines (` 1 pos`) are OTHER. +pub fn line_kind(text: string, at: usize, end: usize) -> u8 { + if (opens_with(text, at, end, " BLOCKED ")) { return LINE_BLOCKED; } + if (total_digits(text, at, end, " tests") <= end) { return LINE_TESTS; } + if (total_digits(text, at, end, " pass") <= end) { return LINE_PASS; } + if (total_digits(text, at, end, " FAIL") <= end) { return LINE_FAIL; } + if (names_after(text, at, end, " FAIL ")) { return LINE_FAIL_NAME; } + if (names_after(text, at, end, " pass ")) { return LINE_PASS_NAME; } + return LINE_OTHER; +} + +/// The offset of the n-th line (from 0) of this kind, or text.len + 1 when there are fewer. +pub fn line_of(text: string, kind: u8, n: u64) -> usize { + var at : usize = 0; + var seen : u64 = 0; + while (at < text.len) { + const end : usize = line_end(text, at); + if (line_kind(text, at, end) == kind) { + if (seen == n) { return at; } + seen = seen + 1; + } + at = end + 1; + } + return text.len + 1; +} + +/// How many lines of this kind the report has. +pub fn count(text: string, kind: u8) -> u64 { + var at : usize = 0; + var seen : u64 = 0; + while (at < text.len) { + const end : usize = line_end(text, at); + if (line_kind(text, at, end) == kind) { seen = seen + 1; } + at = end + 1; + } + return seen; +} + +/// The number on the FIRST line of a total kind (LINE_TESTS, LINE_PASS, LINE_FAIL); 0 when there is none. +pub fn total(text: string, kind: u8) -> u64 { + const at : usize = line_of(text, kind, 0); + if (at > text.len) { return 0; } + var word : string = " tests"; + if (kind == LINE_PASS) { word = " pass"; } + if (kind == LINE_FAIL) { word = " FAIL"; } + const end : usize = line_end(text, at); + var k : usize = total_digits(text, at, end, word); + var value : u64 = 0; + while (k < end and text[k] >= 48 and text[k] <= 57) { + value = value * 10 + ((text[k] as u64) - 48); + k = k + 1; + } + return value; +} + +/// Where the n-th name of a name kind (LINE_FAIL_NAME, LINE_PASS_NAME, or LINE_BLOCKED's why) starts; +/// text.len when there is no such line, so text[name_from..name_to] is then empty. +pub fn name_from(text: string, kind: u8, n: u64) -> usize { + const at : usize = line_of(text, kind, n); + if (at > text.len) { return text.len; } + if (kind == LINE_BLOCKED) { return at + 11; } + return at + 8; +} + +/// Where that name ends: its line's end without trailing blanks. +pub fn name_to(text: string, kind: u8, n: u64) -> usize { + const from : usize = name_from(text, kind, n); + if (from >= text.len) { return text.len; } + return trimmed_end(text, from, line_end(text, from)); +} + +/// The report's verdict, its exit code left aside. BLOCKED anywhere wins: the spec never built. +pub fn read(text: string) -> u8 { + if (count(text, LINE_BLOCKED) > 0) { return V_BLOCKED; } + if (count(text, LINE_TESTS) == 0 or count(text, LINE_PASS) == 0 or count(text, LINE_FAIL) == 0) { return V_NO_TOTALS; } + const tests : u64 = total(text, LINE_TESTS); + const failed : u64 = total(text, LINE_FAIL); + if (total(text, LINE_PASS) + failed != tests) { return V_TOTALS_DISAGREE; } + if (count(text, LINE_FAIL_NAME) != failed) { return V_NAMES_DISAGREE; } + if (failed > 0) { return V_FAILED; } + if (tests == 0) { return V_NO_TEST; } + return V_GREEN; +} + +/// The verdict on one run: the text's, and the exit only says what an unbelievable report means. A +/// red report (FAILED, BLOCKED) with a non-zero exit is that red verdict, so a killed mutant is not a +/// tool error once #7370 lands. +pub fn verdict(text: string, exited_ok: bool) -> u8 { + const r : u8 = read(text); + if (exited_ok) { return r; } + if (r == V_FAILED or r == V_BLOCKED) { return r; } + if (r == V_GREEN or r == V_NO_TEST) { return V_EXIT_GREEN; } + return V_EXIT_UNREAD; +} + +/// A verdict about the tests, not an error about the report. +pub fn is_verdict(v: u8) -> bool { + return v <= V_NO_TEST; +} + +/// `tri test` passes only on GREEN: it built, at least one test ran, and none failed. +pub fn passes(v: u8) -> bool { + return v == V_GREEN; +} + +/// What a verdict means, for the line the glue prints. +pub fn why(v: u8) -> string { + if (v == V_GREEN) { return "green: it built, at least one test ran, and none failed"; } + if (v == V_FAILED) { return "a test FAILs"; } + if (v == V_BLOCKED) { return "BLOCKED: the spec never built, so no test ran"; } + if (v == V_NO_TEST) { return "no test ran: the spec has no `test` block (an invariant is proved at compile time; it is not a test)"; } + if (v == V_NO_TOTALS) { return "t27c test-report printed no tests / pass / FAIL totals and no BLOCKED line"; } + if (v == V_TOTALS_DISAGREE) { return "t27c test-report's totals do not add up: pass + FAIL is not tests"; } + if (v == V_NAMES_DISAGREE) { return "t27c test-report counts a number of FAIL that is not the number of FAIL names it lists"; } + if (v == V_EXIT_GREEN) { return "t27c test-report exited non-zero, yet its report reads green or ran no test; a run that disagrees with itself is not a pass"; } + return "t27c test-report exited non-zero and no report could be read from its output"; +} + +// The names of a list the glue builds from repeated flags: `count` entries joined by '\n'. + +/// Where entry k of a '\n'-joined list starts. +pub fn entry_from(list: string, k: u64) -> usize { + var at : usize = 0; + var seen : u64 = 0; + while (seen < k and at < list.len) { + if (list[at] == 10) { seen = seen + 1; } + at = at + 1; + } + return at; +} + +/// Is text[from..to] byte for byte the entry k of list? +pub fn is_entry(text: string, from: usize, to: usize, list: string, k: u64) -> bool { + const at : usize = entry_from(list, k); + const end : usize = line_end(list, at); + if (end - at != to - from) { return false; } + var i : usize = 0; + while (i < to - from) { + if (text[from + i] != list[at + i]) { return false; } + i = i + 1; + } + return true; +} + +/// Is entry k of list one of the report's names of this kind? +pub fn named(text: string, kind: u8, list: string, k: u64) -> bool { + var n : u64 = 0; + while (n < count(text, kind)) { + if (is_entry(text, name_from(text, kind, n), name_to(text, kind, n), list, k)) { return true; } + n = n + 1; + } + return false; +} + +/// Is every one of the `entries` entries of list a name of this kind? (`--expect` must name a test.) +pub fn all_named(text: string, kind: u8, list: string, entries: u64) -> bool { + var k : u64 = 0; + while (k < entries) { + if (named(text, kind, list, k) == false) { return false; } + k = k + 1; + } + return true; +} + +/// Are the report's names of this kind exactly the entries of list, as sets: every entry is a name, +/// and every name is an entry. A name the list lacks, or an entry no line names, breaks it. +pub fn same_names(text: string, kind: u8, list: string, entries: u64) -> bool { + if (all_named(text, kind, list, entries) == false) { return false; } + var n : u64 = 0; + while (n < count(text, kind)) { + var found : bool = false; + var k : u64 = 0; + while (k < entries) { + if (is_entry(text, name_from(text, kind, n), name_to(text, kind, n), list, k)) { found = true; } + k = k + 1; + } + if (found == false) { return false; } + n = n + 1; + } + return true; +} + +// Scenario: specs/ternary/gft_relu.t27 as it stands -- 4 tests, all pass, no FAIL line. +test the_clean_gft_relu_report_is_green { + const r : string = "--- test report: specs/ternary/gft_relu.t27 ---\n\n tests 4\n pass 4\n FAIL 0\n rate 100.0%\n\n runtime asserts executed, per test (#6509; a pass with 0 is vacuous, T730):\n 1 pos\n 1 negz\n 1 zero\n 1 pos2\n vacuous passes 0 of 4 (passed with 0 runtime asserts executed)\n"; + assert total(r, LINE_TESTS) == 4; + assert total(r, LINE_PASS) == 4; + assert total(r, LINE_FAIL) == 0; + assert count(r, LINE_FAIL_NAME) == 0; + assert count(r, LINE_PASS_NAME) == 0; + assert read(r) == V_GREEN; + assert verdict(r, true) == V_GREEN; + assert passes(verdict(r, true)) == true; + assert verdict(r, false) == V_EXIT_GREEN; + assert passes(verdict(r, false)) == false; +} + +// Scenario: line 12 of gft_relu.t27 planted (`return 0;` -> `return x;` for a negative input): negz +// FAILs. `tri test` used to print "tests passed" here. +test the_planted_gft_relu_report_names_negz { + const r : string = "--- test report: specs/ternary/gft_relu.t27 ---\n FAIL negz\n\n tests 4\n pass 3\n FAIL 1\n invariants 2 proved -- comptime, so compiling IS the check\n rate 75.0%\n\n runtime asserts executed, per test (#6509; a pass with 0 is vacuous, T730):\n 1 pos\n - negz (failed, not counted)\n 1 zero\n 1 pos2\n vacuous passes 0 of 3 (passed with 0 runtime asserts executed)\n"; + assert total(r, LINE_TESTS) == 4; + assert total(r, LINE_PASS) == 3; + assert total(r, LINE_FAIL) == 1; + assert count(r, LINE_FAIL_NAME) == 1; + assert name_from(r, LINE_FAIL_NAME, 0) == 56; + assert name_to(r, LINE_FAIL_NAME, 0) == 60; + assert read(r) == V_FAILED; + assert verdict(r, true) == V_FAILED; + assert verdict(r, false) == V_FAILED; + assert passes(V_FAILED) == false; + assert is_verdict(V_FAILED) == true; +} + +// Scenario: with --verbose the passing tests are named too, and the names are read in report order. +test verbose_names_are_read_in_order { + const r : string = "--- test report: s.t27 ---\n pass pos\n FAIL negz\n pass zero \n\n tests 3\n pass 2\n FAIL 1\n"; + assert count(r, LINE_PASS_NAME) == 2; + assert name_from(r, LINE_PASS_NAME, 0) == 35; + assert name_to(r, LINE_PASS_NAME, 0) == 38; + assert name_from(r, LINE_PASS_NAME, 1) == 60; + assert name_to(r, LINE_PASS_NAME, 1) == 64; + assert name_from(r, LINE_PASS_NAME, 2) == r.len; + assert name_to(r, LINE_PASS_NAME, 2) == r.len; + assert line_of(r, LINE_FAIL_NAME, 0) == 39; + assert line_of(r, LINE_FAIL_NAME, 1) == r.len + 1; + assert read(r) == V_FAILED; +} + +// Scenario: a spec whose generated Zig does not build. The report says BLOCKED and nothing about +// tests; it is red, whatever the exit, and its why is the name of the BLOCKED line. +test a_blocked_report_is_red_with_either_exit { + const r : string = "--- test report: s.t27 ---\n BLOCKED zig: error: expected type 'i32' \n\n A blocked spec is not a failing one. It never produced a\n binary, so it has no per-test result to report.\n"; + assert count(r, LINE_BLOCKED) == 1; + assert name_from(r, LINE_BLOCKED, 0) == 38; + assert name_to(r, LINE_BLOCKED, 0) == 69; + assert read(r) == V_BLOCKED; + assert verdict(r, true) == V_BLOCKED; + assert verdict(r, false) == V_BLOCKED; + assert passes(V_BLOCKED) == false; + assert is_verdict(V_BLOCKED) == true; +} + +// Scenario: a spec with invariants and no `test` block ran no test. That is not a pass. +test no_test_is_not_a_pass { + const r : string = "--- test report: s.t27 ---\n\n tests 0\n pass 0\n FAIL 0\n invariants 2 proved -- comptime, so compiling IS the check\n"; + assert read(r) == V_NO_TEST; + assert verdict(r, true) == V_NO_TEST; + assert passes(V_NO_TEST) == false; + assert is_verdict(V_NO_TEST) == true; + assert verdict(r, false) == V_EXIT_GREEN; +} + +// Scenario: reports the reader cannot believe are errors, never passes, with either exit. +test a_report_that_does_not_add_up_is_an_error { + assert read("") == V_NO_TOTALS; + assert read("--- test report: s.t27 ---\n listed 4 tests\n") == V_NO_TOTALS; + assert read("\n tests 2\n pass 2\n") == V_NO_TOTALS; + assert read("\n tests 3\n pass 1\n FAIL 0\n") == V_TOTALS_DISAGREE; + assert read(" FAIL a\n\n tests 2\n pass 0\n FAIL 2\n") == V_NAMES_DISAGREE; + assert read(" FAIL a\n FAIL b\n\n tests 2\n pass 1\n FAIL 1\n") == V_NAMES_DISAGREE; + assert verdict("", true) == V_NO_TOTALS; + assert verdict("", false) == V_EXIT_UNREAD; + assert verdict("\n tests 3\n pass 1\n FAIL 0\n", false) == V_EXIT_UNREAD; + assert is_verdict(V_NO_TOTALS) == false; + assert is_verdict(V_TOTALS_DISAGREE) == false; + assert is_verdict(V_NAMES_DISAGREE) == false; + assert is_verdict(V_EXIT_GREEN) == false; + assert is_verdict(V_EXIT_UNREAD) == false; + assert passes(V_EXIT_UNREAD) == false; +} + +// Scenario: only the first line of each total counts, as the first reader did; a later line that looks +// like a total (the 2-of-3 vacuous summary does not) changes nothing. +test the_first_total_counts { + const r : string = "\n tests 2\n pass 2\n FAIL 0\n tests 9\n"; + assert count(r, LINE_TESTS) == 2; + assert total(r, LINE_TESTS) == 2; + assert read(r) == V_GREEN; +} + +// Scenario: which line is which -- a total needs 3 or more spaces and only blanks after its digits, a +// name exactly two spaces and a non-blank, and a total is tried first. +test each_line_is_read_as_its_kind { + assert line_kind(" tests 4", 0, 15) == LINE_TESTS; + assert line_kind(" pass 3", 0, 15) == LINE_PASS; + assert line_kind(" FAIL 1", 0, 15) == LINE_FAIL; + assert line_kind(" FAIL 1", 0, 10) == LINE_FAIL; + assert line_kind(" FAIL 1", 0, 9) == LINE_FAIL_NAME; + assert line_kind(" FAIL negz", 0, 12) == LINE_FAIL_NAME; + assert line_kind(" pass pos", 0, 11) == LINE_PASS_NAME; + assert line_kind(" pass ", 0, 9) == LINE_OTHER; + assert line_kind(" FAIL 1 x", 0, 17) == LINE_OTHER; + assert line_kind(" tests run 4", 0, 28) == LINE_OTHER; + assert line_kind(" tests 4 \t", 0, 17) == LINE_TESTS; + assert line_kind(" BLOCKED x", 0, 12) == LINE_BLOCKED; + assert line_kind(" BLOCKED x", 0, 11) == LINE_OTHER; + assert line_kind(" 1 pos", 0, 13) == LINE_OTHER; + assert line_kind(" vacuous passes 0 of 3", 0, 24) == LINE_OTHER; + assert line_kind(" FAIL negz", 0, 11) == LINE_OTHER; + assert line_kind("", 0, 0) == LINE_OTHER; +} + +// Scenario: a total of 18 digits is read, 19 is not a total, and a total with no digits is not one. +test a_total_has_one_to_18_digits { + assert total_digits(" tests 123456789012345678", 0, 28, " tests") == 10; + assert total_digits(" tests 1234567890123456789", 0, 29, " tests") == 30; + assert total_digits(" tests ", 0, 13, " tests") == 14; + assert total_digits(" tests 4", 0, 10, " tests") == 11; + assert total_digits(" tests 4x", 0, 12, " tests") == 13; + assert total("\n tests 123456789012345678\n", LINE_TESTS) == 123456789012345678; + assert total("no totals", LINE_TESTS) == 0; +} + +// Scenario: the small readers on their edges. +test the_line_readers_hold_on_their_edges { + assert line_end("ab\ncd", 0) == 2; + assert line_end("ab\ncd", 3) == 5; + assert line_end("", 0) == 0; + assert opens_with("abc", 0, 3, "ab") == true; + assert opens_with("abc", 1, 3, "ab") == false; + assert opens_with("abc", 0, 1, "ab") == false; + assert opens_with("abc", 4, 3, "") == false; + assert trimmed_end("ab \t", 0, 5) == 2; + assert trimmed_end(" ", 1, 3) == 1; + assert is_blank(32) == true; + assert is_blank(9) == true; + assert is_blank(13) == true; + assert is_blank(8) == false; + assert is_blank(14) == false; + assert is_blank(97) == false; + assert names_after(" FAIL ", 0, 8, " FAIL ") == false; + assert names_after(" FAIL \t", 0, 9, " FAIL ") == false; + assert names_after(" FAIL n", 0, 9, " FAIL ") == true; +} + +// Scenario: `tri mutate plant --expect negz --expect zero` reads its two flags as one '\n'-joined list. +test the_expect_list_is_read_entry_by_entry { + const list : string = "negz\nzero"; + assert entry_from(list, 0) == 0; + assert entry_from(list, 1) == 5; + assert entry_from(list, 2) == 9; + assert is_entry("xnegzx", 1, 5, list, 0) == true; + assert is_entry("xnegzx", 1, 4, list, 0) == false; + assert is_entry("xzerox", 1, 5, list, 1) == true; + assert is_entry("xzerox", 1, 5, list, 0) == false; + assert is_entry("x", 0, 0, list, 2) == true; +} + +// Scenario: every --expect must name a test of the spec (the baseline's `pass` names), and the mutant's +// FAIL names must be exactly the --expect set: a subset is not the set, and neither is a superset. +test the_failing_set_is_the_expected_set_or_nothing { + const base : string = " pass pos\n pass negz\n pass zero\n pass pos2\n\n tests 4\n pass 4\n FAIL 0\n"; + assert all_named(base, LINE_PASS_NAME, "negz", 1) == true; + assert all_named(base, LINE_PASS_NAME, "negz\npos2", 2) == true; + assert all_named(base, LINE_PASS_NAME, "nosuch", 1) == false; + assert all_named(base, LINE_PASS_NAME, "negz\nneg", 2) == false; + assert all_named(base, LINE_PASS_NAME, "", 0) == true; + const one : string = " FAIL negz\n\n tests 4\n pass 3\n FAIL 1\n"; + assert same_names(one, LINE_FAIL_NAME, "negz", 1) == true; + assert same_names(one, LINE_FAIL_NAME, "pos", 1) == false; + assert same_names(one, LINE_FAIL_NAME, "negz\nzero", 2) == false; + assert same_names(one, LINE_FAIL_NAME, "negz\nnegz", 2) == true; + assert same_names(one, LINE_FAIL_NAME, "", 0) == false; + const two : string = " FAIL pos\n FAIL pos2\n\n tests 4\n pass 2\n FAIL 2\n"; + assert same_names(two, LINE_FAIL_NAME, "pos\npos2", 2) == true; + assert same_names(two, LINE_FAIL_NAME, "pos2\npos", 2) == true; + assert same_names(two, LINE_FAIL_NAME, "pos", 1) == false; + assert named(two, LINE_FAIL_NAME, "pos\npos2", 1) == true; + assert named(two, LINE_FAIL_NAME, "pos\npos3", 1) == false; +} + +// Scenario: each verdict has its own words, and the green one is not an error's. +test each_verdict_says_what_it_means { + assert why(V_GREEN)[0] == 103; + assert why(V_FAILED)[0] == 97; + assert why(V_BLOCKED)[0] == 66; + assert why(V_NO_TEST)[0] == 110; + assert why(V_NO_TOTALS)[17] == 112; + assert why(V_TOTALS_DISAGREE)[17] == 115; + assert why(V_NAMES_DISAGREE)[17] == 99; + assert why(V_EXIT_GREEN)[32] == 44; + assert why(V_EXIT_UNREAD)[32] == 32; +} From 1ab3a30aa2a6e16d92f03db7980cda80702e2d86 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 09:56:28 +0000 Subject: [PATCH 7/8] spec(tri): close the mutation survivors of report.t27 and plant.t27 (Refs #7369) `tri mutate spec` on the two new specs: report.t27 185 of 219 killed, 18 survived; plant.t27 115 of 128, 6 survived. Each survivor was a test gap or a dead guard: - dead guards removed: total()'s and name_to()'s early returns (the general path gives the same 0 / empty name), line_stop()'s (its loop never runs past the end); `<= end` on a total's offset and `> text.len` on a missing line become exact tests (`!= end + 1`, `== text.len + 1`, `n == 0 or n > line_count`), so no flip is equivalent. - asserts added: opens_with on a line it exactly fills and past `end`, OTHER lines counted without a phantom one after the last '\n', a total at the very end of the text and one followed by a blank, the missing-pass and missing-tests reports, an --expect list longer than its count, an empty entry, an empty --from searched on an empty line, same_text on a longer first argument. After: report.t27 207 of 223 killed, plant.t27 121 of 128; 0 survived in both. Every hung mutant is a dropped loop step or an `at = end - 1` that makes its own loop endless. Specs regenerated with t27c gen-rust and re-sealed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01ScHVrGSr6zZUdwkdC8DR9k --- .trinity/seals/mutate_TriMutatePlant.json | 12 +++++------ .trinity/seals/test_TriTestReport.json | 12 +++++------ gen/rust/tri/mutate/plant.rs | 8 ++----- gen/rust/tri/test/report.rs | 14 ++++-------- specs/tri/mutate/plant.t27 | 10 +++++---- specs/tri/test/report.t27 | 26 +++++++++++++++++------ 6 files changed, 43 insertions(+), 39 deletions(-) diff --git a/.trinity/seals/mutate_TriMutatePlant.json b/.trinity/seals/mutate_TriMutatePlant.json index 34ddc5f095..3f3c01afc7 100644 --- a/.trinity/seals/mutate_TriMutatePlant.json +++ b/.trinity/seals/mutate_TriMutatePlant.json @@ -1,14 +1,14 @@ { "built_by": "t27c-bootstrap@0.5.1+c0b721c83", - "gen_hash_c": "sha256:00ec0fd9a5b0eee498cd770bd03fa5fc79f61c018d3b94ad663ab3661529cf55", - "gen_hash_rust": "sha256:bedfb86cc4d9c5b0fa972e11f4434d10f2c20cb68bf17afff3a3c572a081071a", - "gen_hash_verilog": "sha256:2270d197974d4609000b6ded806598a348ff1a56ff4b91a61f67de22c110772c", - "gen_hash_zig": "sha256:2ad2379438ea12d57aa48758e25ba3d2c6bd11ee144e1192e03bdff663c0b0b8", + "gen_hash_c": "sha256:479a9162fcf9c62f5e4783b828c3a130394e803df8fdd298729914cd0b4efa5d", + "gen_hash_rust": "sha256:089e787b9c9e09dba36a67064d2fadbc221bed4ebd917fdfb98a66db14fc42f5", + "gen_hash_verilog": "sha256:f75f945662fb2f4b9a8984a400e24a8a5fa6fb2fe8a76c9b73ab9311b973e22a", + "gen_hash_zig": "sha256:50201a55310aec35d00f79e704678bbefc62b8a03db23360636ba3c3f420780d", "module": "TriMutatePlant", "ring": 12, - "sealed_at": "2026-10-09T08:49:38Z", + "sealed_at": "2026-10-09T09:16:56Z", "sealed_by": "t27c-bootstrap@0.5.1", - "spec_hash": "sha256:6619f0b6cafdc9cc5abe33c35f11b030bb8036671e14baa1e79936c579909df0", + "spec_hash": "sha256:792b96b8997c508ae143e62a51d6383dff5fa57ea2dd3ece6c1f8a56ea2e6ddf", "spec_path": "specs/tri/mutate/plant.t27", "tests": { "failed": 0, diff --git a/.trinity/seals/test_TriTestReport.json b/.trinity/seals/test_TriTestReport.json index e8a415e435..cfd7090b6e 100644 --- a/.trinity/seals/test_TriTestReport.json +++ b/.trinity/seals/test_TriTestReport.json @@ -1,14 +1,14 @@ { "built_by": "t27c-bootstrap@0.5.1+c0b721c83", - "gen_hash_c": "sha256:5727a9e7b4cace405dff3ea3023789ee5d5bf8697b991c2a937a61d6449167e1", - "gen_hash_rust": "sha256:97f5785cb929ad2499cec12d236e721d5973b837dd522b8f1fc5aa7b5947cdfc", - "gen_hash_verilog": "sha256:67d0c7871b80a0f058665154f604b3d255dcd32393a1620199a65fb329bdafbf", - "gen_hash_zig": "sha256:43c7dccfe44c23285ba074bab8197afaff1556015ea80889483f24c5ddad2d42", + "gen_hash_c": "sha256:74674d3e38f3f62b92020f64892cf374e8def374e40e3c0089015b554f12fd8b", + "gen_hash_rust": "sha256:1de6c6f3a0ed67d11ff54d090d90429164ffa82c95756dc19c2286c8f6060697", + "gen_hash_verilog": "sha256:06d3c6e9febf8fd3f6c0cbe5a7a1fa71d9cac27dec668e4c85dc3d3c1beda9bc", + "gen_hash_zig": "sha256:a3ee9cd2ebbd6997c908463e3f3f2cda151186b679f4e78941f3c9770523d053", "module": "TriTestReport", "ring": 12, - "sealed_at": "2026-10-09T08:49:37Z", + "sealed_at": "2026-10-09T09:43:17Z", "sealed_by": "t27c-bootstrap@0.5.1", - "spec_hash": "sha256:c5914b13c06cfd8ff1b10448106d616ef8e256f3720cbb4d8f8d2519667d142c", + "spec_hash": "sha256:2e781b2dc9d841f0c6482e06a328ac2c27a1c74bad71d847c283cf9111c62bd0", "spec_path": "specs/tri/test/report.t27", "tests": { "failed": 0, diff --git a/gen/rust/tri/mutate/plant.rs b/gen/rust/tri/mutate/plant.rs index 2be0651e62..5506d872c6 100644 --- a/gen/rust/tri/mutate/plant.rs +++ b/gen/rust/tri/mutate/plant.rs @@ -65,11 +65,7 @@ pub fn line_start(text: &'static str, n: usize) -> usize { } pub fn line_stop(text: &'static str, n: usize) -> usize { - let start: usize = line_start(text, n); - if (start > text.len()) { - return start; - } - let mut k: usize = start; + let mut k: usize = line_start(text, n); while ((k < text.len()) && (text.as_bytes()[(k) as usize] != 10)) { k = (k + 1); } @@ -142,7 +138,7 @@ pub fn plant_check(text: &'static str, n: usize, from: &'static str, to: &'stati if breaks_line(to) { return REFUSE_LINE_BREAK; } - if (line_start(text, n) > text.len()) { + if ((n == 0) || (n > line_count(text))) { return REFUSE_NO_SUCH_LINE; } if (occurrences(text, line_start(text, n), line_stop(text, n), from) != 1) { diff --git a/gen/rust/tri/test/report.rs b/gen/rust/tri/test/report.rs index b81bd229c5..d4e5b2304c 100644 --- a/gen/rust/tri/test/report.rs +++ b/gen/rust/tri/test/report.rs @@ -110,13 +110,13 @@ pub fn line_kind(text: &'static str, at: usize, end: usize) -> u8 { if opens_with(text, at, end, " BLOCKED ") { return LINE_BLOCKED; } - if (total_digits(text, at, end, " tests") <= end) { + if (total_digits(text, at, end, " tests") != (end + 1)) { return LINE_TESTS; } - if (total_digits(text, at, end, " pass") <= end) { + if (total_digits(text, at, end, " pass") != (end + 1)) { return LINE_PASS; } - if (total_digits(text, at, end, " FAIL") <= end) { + if (total_digits(text, at, end, " FAIL") != (end + 1)) { return LINE_FAIL; } if names_after(text, at, end, " FAIL ") { @@ -159,9 +159,6 @@ pub fn count(text: &'static str, kind: u8) -> u64 { pub fn total(text: &'static str, kind: u8) -> u64 { let at: usize = line_of(text, kind, 0); - if (at > text.len()) { - return 0; - } let mut word: &'static str = " tests"; if (kind == LINE_PASS) { word = " pass"; @@ -181,7 +178,7 @@ pub fn total(text: &'static str, kind: u8) -> u64 { pub fn name_from(text: &'static str, kind: u8, n: u64) -> usize { let at: usize = line_of(text, kind, n); - if (at > text.len()) { + if (at == (text.len() + 1)) { return text.len(); } if (kind == LINE_BLOCKED) { @@ -192,9 +189,6 @@ pub fn name_from(text: &'static str, kind: u8, n: u64) -> usize { pub fn name_to(text: &'static str, kind: u8, n: u64) -> usize { let from: usize = name_from(text, kind, n); - if (from >= text.len()) { - return text.len(); - } return trimmed_end(text, from, line_end(text, from)); } diff --git a/specs/tri/mutate/plant.t27 b/specs/tri/mutate/plant.t27 index 47542a0d4d..160f30b775 100644 --- a/specs/tri/mutate/plant.t27 +++ b/specs/tri/mutate/plant.t27 @@ -72,9 +72,7 @@ pub fn line_start(text: string, n: usize) -> usize { /// Where line n ends: its '\n', or the end of the text. pub fn line_stop(text: string, n: usize) -> usize { - const start : usize = line_start(text, n); - if (start > text.len) { return start; } - var k : usize = start; + var k : usize = line_start(text, n); while (k < text.len and text[k] != 10) { k = k + 1; } @@ -135,7 +133,7 @@ pub fn plant_check(text: string, n: usize, from: string, to: string) -> u8 { if (from.len == 0) { return REFUSE_EMPTY_FROM; } if (same_text(from, to)) { return REFUSE_SAME_TEXT; } if (breaks_line(to)) { return REFUSE_LINE_BREAK; } - if (line_start(text, n) > text.len) { return REFUSE_NO_SUCH_LINE; } + if (n == 0 or n > line_count(text)) { return REFUSE_NO_SUCH_LINE; } if (occurrences(text, line_start(text, n), line_stop(text, n), from) != 1) { return REFUSE_NOT_ONCE; } return PLANT_OK; } @@ -216,6 +214,9 @@ test from_must_occur_exactly_once_on_the_line { assert plant_check("ab\nab\n", 2, "ab", "x") == PLANT_OK; assert plant_at("ab\nab\n", 2, "ab") == 3; assert plant_at("ab\nab\n", 2, "zz") == 7; + assert plant_check("a\n\nc\n", 2, "x", "y") == REFUSE_NOT_ONCE; + assert plant_at("a\n\nc\n", 2, "") == 6; + assert occurrences("ab", 0, 2, "") == 2; } // Scenario: the other refusals, each before t27c runs. @@ -231,6 +232,7 @@ test the_plant_refuses_what_is_not_one_line_into_one_line { assert breaks_line("a\tb") == false; assert breaks_line("") == false; assert same_text("ab", "abc") == false; + assert same_text("abc", "ab") == false; assert same_text("", "") == true; } diff --git a/specs/tri/test/report.t27 b/specs/tri/test/report.t27 index ee28e72b1a..a09aa44672 100644 --- a/specs/tri/test/report.t27 +++ b/specs/tri/test/report.t27 @@ -122,9 +122,9 @@ pub fn names_after(text: string, at: usize, end: usize, word: string) -> bool { /// and ` pass pos` a name; the per-test assert lines (` 1 pos`) are OTHER. pub fn line_kind(text: string, at: usize, end: usize) -> u8 { if (opens_with(text, at, end, " BLOCKED ")) { return LINE_BLOCKED; } - if (total_digits(text, at, end, " tests") <= end) { return LINE_TESTS; } - if (total_digits(text, at, end, " pass") <= end) { return LINE_PASS; } - if (total_digits(text, at, end, " FAIL") <= end) { return LINE_FAIL; } + if (total_digits(text, at, end, " tests") != end + 1) { return LINE_TESTS; } + if (total_digits(text, at, end, " pass") != end + 1) { return LINE_PASS; } + if (total_digits(text, at, end, " FAIL") != end + 1) { return LINE_FAIL; } if (names_after(text, at, end, " FAIL ")) { return LINE_FAIL_NAME; } if (names_after(text, at, end, " pass ")) { return LINE_PASS_NAME; } return LINE_OTHER; @@ -157,10 +157,10 @@ pub fn count(text: string, kind: u8) -> u64 { return seen; } -/// The number on the FIRST line of a total kind (LINE_TESTS, LINE_PASS, LINE_FAIL); 0 when there is none. +/// The number on the FIRST line of a total kind (LINE_TESTS, LINE_PASS, LINE_FAIL); 0 when there is none +/// (line_of is then text.len + 1, past every digit). pub fn total(text: string, kind: u8) -> u64 { const at : usize = line_of(text, kind, 0); - if (at > text.len) { return 0; } var word : string = " tests"; if (kind == LINE_PASS) { word = " pass"; } if (kind == LINE_FAIL) { word = " FAIL"; } @@ -178,7 +178,7 @@ pub fn total(text: string, kind: u8) -> u64 { /// text.len when there is no such line, so text[name_from..name_to] is then empty. pub fn name_from(text: string, kind: u8, n: u64) -> usize { const at : usize = line_of(text, kind, n); - if (at > text.len) { return text.len; } + if (at == text.len + 1) { return text.len; } if (kind == LINE_BLOCKED) { return at + 11; } return at + 8; } @@ -186,7 +186,6 @@ pub fn name_from(text: string, kind: u8, n: u64) -> usize { /// Where that name ends: its line's end without trailing blanks. pub fn name_to(text: string, kind: u8, n: u64) -> usize { const from : usize = name_from(text, kind, n); - if (from >= text.len) { return text.len; } return trimmed_end(text, from, line_end(text, from)); } @@ -377,6 +376,8 @@ test a_report_that_does_not_add_up_is_an_error { assert read("") == V_NO_TOTALS; assert read("--- test report: s.t27 ---\n listed 4 tests\n") == V_NO_TOTALS; assert read("\n tests 2\n pass 2\n") == V_NO_TOTALS; + assert read("\n tests 2\n FAIL 0\n") == V_NO_TOTALS; + assert read("\n pass 2\n FAIL 0\n") == V_NO_TOTALS; assert read("\n tests 3\n pass 1\n FAIL 0\n") == V_TOTALS_DISAGREE; assert read(" FAIL a\n\n tests 2\n pass 0\n FAIL 2\n") == V_NAMES_DISAGREE; assert read(" FAIL a\n FAIL b\n\n tests 2\n pass 1\n FAIL 1\n") == V_NAMES_DISAGREE; @@ -431,6 +432,9 @@ test a_total_has_one_to_18_digits { assert total_digits(" tests 4x", 0, 12, " tests") == 13; assert total("\n tests 123456789012345678\n", LINE_TESTS) == 123456789012345678; assert total("no totals", LINE_TESTS) == 0; + assert total(" tests 4", LINE_TESTS) == 4; + assert total(" tests 4 \n", LINE_TESTS) == 4; + assert total(" pass 7\n FAIL 2", LINE_FAIL) == 2; } // Scenario: the small readers on their edges. @@ -442,6 +446,12 @@ test the_line_readers_hold_on_their_edges { assert opens_with("abc", 1, 3, "ab") == false; assert opens_with("abc", 0, 1, "ab") == false; assert opens_with("abc", 4, 3, "") == false; + assert opens_with("abc", 3, 3, "") == true; + assert opens_with("ab", 0, 2, "ab") == true; + assert opens_with("abcd", 1, 2, "bc") == false; + assert count("a\nb\n", LINE_OTHER) == 2; + assert count("a\n\nb", LINE_OTHER) == 3; + assert line_of("a\n", LINE_OTHER, 1) == 3; assert trimmed_end("ab \t", 0, 5) == 2; assert trimmed_end(" ", 1, 3) == 1; assert is_blank(32) == true; @@ -487,6 +497,8 @@ test the_failing_set_is_the_expected_set_or_nothing { assert same_names(two, LINE_FAIL_NAME, "pos\npos2", 2) == true; assert same_names(two, LINE_FAIL_NAME, "pos2\npos", 2) == true; assert same_names(two, LINE_FAIL_NAME, "pos", 1) == false; + assert same_names(two, LINE_FAIL_NAME, "pos\npos2", 1) == false; + assert named(two, LINE_FAIL_NAME, "", 0) == false; assert named(two, LINE_FAIL_NAME, "pos\npos2", 1) == true; assert named(two, LINE_FAIL_NAME, "pos\npos3", 1) == false; } From ed195241ea3d3a03697de74934ab0d7f7179dda0 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 10:18:48 +0000 Subject: [PATCH 8/8] tri-mcp: tri_test runs the spec's tests and reads the verdict through report.t27 (Closes #8140) The MCP tool tri_test ran `t27c test`, which only lists a spec's tests, and reported success from its exit code whatever the tests did -- the defect #7369 fixes for `tri test`, left in tri-mcp. It now runs `t27c test-report` and takes its verdict from specs/tri/test/report.t27 through gen/rust/tri/test/report.rs, the same generated module `tri test` mounts with #[path]. tri-mcp holds only I/O glue: run t27c, hand the text and the exit to report::verdict, return isError unless report::passes, with report::why, the totals and the FAIL / BLOCKED names. An optional `t27c` input picks the binary, as `tri test --t27c`. - Cargo.toml: cli/tri-mcp joins the workspace members, so `cargo test -p tri-mcp` exists (it was in neither list, Refs #5395); docs/reports/orphan_modules.json gets its ceiling (0), which modreach's member/ledger test requires. - cli/tri-mcp/tests/tri_test.rs drives the server over stdio from a temp root (the tracked akashic log is not written): gft_relu green, a temp copy with #7400's plant red naming negz, the copy restored and its sha256 checked, the tree's spec sha256 unchanged; negative controls: a stand-in t27c that only lists tests (verdict NO_TOTALS) and one that prints a green report and exits 1 (EXIT_GREEN) -- both isError. - specs/tools/mcp/tri-mcp.t27: tri_test's description and inputs as the code now lists them; resealed with t27c built at this tree. Stacked on #7400. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01ScHVrGSr6zZUdwkdC8DR9k --- .trinity/seals/mcp_tool_mcp_tri_mcp.json | 16 ++--- Cargo.lock | 14 +++++ Cargo.toml | 2 +- cli/tri-mcp/src/main.rs | 78 ++++++++---------------- cli/tri-mcp/tests/tri_test.rs | 40 ++++++++++++ docs/reports/orphan_modules.json | 1 + specs/tools/mcp/tri-mcp.t27 | 8 ++- 7 files changed, 95 insertions(+), 64 deletions(-) create mode 100644 cli/tri-mcp/tests/tri_test.rs diff --git a/.trinity/seals/mcp_tool_mcp_tri_mcp.json b/.trinity/seals/mcp_tool_mcp_tri_mcp.json index 6bcddc7694..ec6e892f46 100644 --- a/.trinity/seals/mcp_tool_mcp_tri_mcp.json +++ b/.trinity/seals/mcp_tool_mcp_tri_mcp.json @@ -1,14 +1,14 @@ { - "built_by": "t27c-bootstrap@0.5.0+1120d30ae", - "gen_hash_c": "sha256:559f17c9e2942a4e732666d74fb1079d35d1ea8f2e1e8a967b453c076f3eb595", - "gen_hash_rust": "sha256:1c820f3d29f91cd8e570b0f90709bf2a4d56b47b505f94df0880949a74145c50", - "gen_hash_verilog": "sha256:2ec9763a4c749561783d3ebb34252dc1f4a884064987bbb8d7cbd0c555d64f4d", - "gen_hash_zig": "sha256:4d499c482afffae6f97ee309682df8a914f721b30fe5ca89b6af0b91d897adb5", + "built_by": "t27c-bootstrap@0.5.2+1ab3a30aa", + "gen_hash_c": "sha256:94f2b9d84e506db530f2b6cbc1fa5ae9ceeb953a9bdbd5fdcd5a8ec1698e26cf", + "gen_hash_rust": "sha256:0dfc81535eadd2f97bb23984253f6bc8e5c41d13b415d3d7f5dfe9c5d65ce290", + "gen_hash_verilog": "sha256:eaaa0d85d10e6a39a9fce9f2a1833f7d1b5d7f0fb86b84c5c834e546560a0b48", + "gen_hash_zig": "sha256:faaea991871b395879a7f56e114e93e1119820145500c2b47ee85e0d2507a02f", "module": "tool_mcp_tri_mcp", "ring": 12, - "sealed_at": "2026-10-07T23:11:20Z", - "sealed_by": "t27c-bootstrap@0.5.0", - "spec_hash": "sha256:2db3e41d25dd8eff540b15459ce20402bac3d4520731a356b20a6fa8a81b9f3d", + "sealed_at": "2026-10-09T10:15:58Z", + "sealed_by": "t27c-bootstrap@0.5.2", + "spec_hash": "sha256:a0ff5b0e9e6f1793032f14f7a5f4baf31ea8fb6df1179e7a8fffc75a6e0f9db2", "spec_path": "specs/tools/mcp/tri-mcp.t27", "tests": { "failed": 0, diff --git a/Cargo.lock b/Cargo.lock index ff02e9da60..04ceac5d3f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2590,6 +2590,20 @@ dependencies = [ "uuid", ] +[[package]] +name = "tri-mcp" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "clap", + "serde", + "serde_json", + "sha2 0.10.9", + "tokio", + "uuid", +] + [[package]] name = "trios-bridge" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index a3970a77ea..180b184bbb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] resolver = "2" -members = ["bootstrap", "cli/tri", "cli/trios-bridge", "cli/flash-spi", "cli/dlc10", "cli/t27b"] +members = ["bootstrap", "cli/tri", "cli/tri-mcp", "cli/trios-bridge", "cli/flash-spi", "cli/dlc10", "cli/t27b"] # `ffi` was in NEITHER members nor exclude, so `cargo publish` in it failed on # manifest load -- "current package believes it's in a workspace when it's not" # -- before it could reach a registry. That is why the crates.io leg of the diff --git a/cli/tri-mcp/src/main.rs b/cli/tri-mcp/src/main.rs index e0f85d11fa..9951b44ebf 100644 --- a/cli/tri-mcp/src/main.rs +++ b/cli/tri-mcp/src/main.rs @@ -10,6 +10,9 @@ use serde_json::Value; use sha2::{Digest, Sha256}; use uuid::Uuid; +#[path = "../../../gen/rust/tri/test/report.rs"] #[allow(dead_code, unused_parens)] +mod report; // t27c gen-rust of specs/tri/test/report.t27 (#7369): the same reading of `t27c test-report` that `tri test` uses + #[derive(Debug, Clone, Serialize, Deserialize)] struct ActiveSkill { skill_id: String, @@ -226,14 +229,15 @@ fn build_tools_list() -> Vec { }), serde_json::json!({ "name": "tri_test", - "description": "Runs tests for a .t27 spec file using t27c.", + "description": "Runs a .t27 spec's tests with `t27c test-report`; isError unless every test passed and at least one ran (rules: specs/tri/test/report.t27).", "inputSchema": { "type": "object", "properties": { "spec_path": { "type": "string", "description": "Path to the .t27 spec file" - } + }, + "t27c": {"type": "string", "description": "The t27c binary; default target/release/t27c, then target/debug/t27c, then t27c on PATH"} }, "required": ["spec_path"] } @@ -722,58 +726,28 @@ fn cmd_gen(root: &Path, args: &Value) -> Result { } } +/// `t27c test` only lists the tests (#8140); they run in `t27c test-report`, and the verdict is report.t27's, as in `tri test`. fn cmd_test(root: &Path, args: &Value) -> Result { - let spec_path = args - .get("spec_path") - .and_then(|v| v.as_str()) - .ok_or_else(|| anyhow::anyhow!("missing 'spec_path' parameter"))?; - - let spec = Path::new(spec_path); - if !spec.exists() { - return Ok(tool_error_text(&format!( - "Spec file not found: {}", - spec_path - ))); - } - - let output = Command::new("t27c") - .args(["test", spec_path]) - .current_dir(root) - .output(); - - match output { - Ok(out) => { - let stdout = String::from_utf8_lossy(&out.stdout).to_string(); - let stderr = String::from_utf8_lossy(&out.stderr).to_string(); - let success = out.status.success(); - - append_akashic( - root, - &make_akashic_event( - "tri.test", - if success { "success" } else { "failure" }, - serde_json::json!({ - "spec_path": spec_path, - "exit_code": out.status.code() - }), - ), - )?; - - let result = serde_json::json!({ - "success": success, - "exit_code": out.status.code(), - "stdout": stdout, - "stderr": stderr, - "spec_path": spec_path - }); - - Ok(tool_result_text(&serde_json::to_string_pretty(&result)?)) - } - Err(e) => { - let msg = format!("Failed to execute t27c: {}. Is t27c on PATH?", e); - Ok(tool_error_text(&msg)) - } + let spec = args.get("spec_path").and_then(|v| v.as_str()).ok_or_else(|| anyhow::anyhow!("missing 'spec_path' parameter"))?; + if !root.join(spec).exists() { + return Ok(tool_error_text(&format!("Spec file not found: {}", spec))); } + let built = ["target/release/t27c", "target/debug/t27c"].iter().map(|p| root.join(p)).find(|p| p.exists()); + let t27c = args.get("t27c").and_then(|v| v.as_str()).map(PathBuf::from).or(built).unwrap_or_else(|| "t27c".into()); + let out = Command::new(&t27c).args(["test-report", spec]).current_dir(root).output().map_err(|e| anyhow::anyhow!("cannot start `{} test-report`: {}", t27c.display(), e))?; + let text: &'static str = String::from_utf8_lossy(&out.stdout).into_owned().leak(); // gen-rust's `string` is `&'static str` (#7449) + let (v, total) = (report::verdict(text, out.status.success()), |k| report::total(text, k)); + let names = |k| (0..report::count(text, k)).map(|n| &text[report::name_from(text, k, n)..report::name_to(text, k, n)]).collect::>(); + let outcome = if report::passes(v) { "success" } else { "failure" }; + append_akashic(root, &make_akashic_event("tri.test", outcome, serde_json::json!({"spec_path": spec, "verdict": v, "exit_code": out.status.code()})))?; + let result = serde_json::json!({ + "spec_path": spec, "passes": report::passes(v), "verdict": v, "why": report::why(v), + "tests": total(report::LINE_TESTS), "pass": total(report::LINE_PASS), "fail": total(report::LINE_FAIL), + "failed": names(report::LINE_FAIL_NAME), "blocked": names(report::LINE_BLOCKED), + "exit_code": out.status.code(), "report": text, "stderr": String::from_utf8_lossy(&out.stderr) + }); + let shown = serde_json::to_string_pretty(&result)?; + Ok(if report::passes(v) { tool_result_text(&shown) } else { tool_error_text(&shown) }) } fn cmd_verdict(root: &Path) -> Result { diff --git a/cli/tri-mcp/tests/tri_test.rs b/cli/tri-mcp/tests/tri_test.rs new file mode 100644 index 0000000000..ff8dc4395f --- /dev/null +++ b/cli/tri-mcp/tests/tri_test.rs @@ -0,0 +1,40 @@ +//! tri_test runs a spec's tests and reads the verdict through report.t27 (#8140): a green spec, a red copy +//! with one planted failure, and a stand-in t27c that only lists the tests, as `t27c test` did. +use serde_json::{json, Value}; use sha2::{Digest, Sha256}; +use std::{io::Write, path::Path, process::{Command, Stdio}}; + +fn sha(p: &Path) -> Vec { Sha256::digest(std::fs::read(p).unwrap()).to_vec() } +/// One tools/call of tri_test on a server started in `root`: the result's text as JSON, and isError. +fn tri_test(root: &Path, t27c: &Path) -> (Value, bool) { + let mut s = Command::new(env!("CARGO_BIN_EXE_tri-mcp")).current_dir(root).stdin(Stdio::piped()).stdout(Stdio::piped()).spawn().unwrap(); + let args = json!({"spec_path": "gft_relu.t27", "t27c": t27c}); + writeln!(s.stdin.take().unwrap(), "{}", json!({"jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": {"name": "tri_test", "arguments": args}})).unwrap(); + let r: Value = serde_json::from_slice(&s.wait_with_output().unwrap().stdout).unwrap(); + (serde_json::from_str(r["result"]["content"][0]["text"].as_str().unwrap()).unwrap(), r["result"]["isError"].as_bool().unwrap()) +} + +#[test] +fn tri_test_runs_the_tests_and_a_listing_is_not_a_pass() { + let repo = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let t27c = std::env::var_os("T27C").map(Into::into).or_else(|| ["release", "debug"].iter().map(|d| repo.join("target").join(d).join("t27c")).find(|p| p.exists())); + let t27c: std::path::PathBuf = t27c.expect("no t27c: run `cargo build -p t27c` or set T27C"); + let (spec, root) = (repo.join("specs/ternary/gft_relu.t27"), std::env::temp_dir().join(format!("tri-mcp-test-{}", std::process::id()))); + let (before, copy) = (sha(&spec), root.join("gft_relu.t27")); // a temp root, so the tracked akashic log stays as it is + std::fs::create_dir_all(root.join(".trinity")).unwrap(); + std::fs::copy(&spec, ©).unwrap(); + let (green, err) = tri_test(&root, &t27c); + assert!(!err && green["passes"] == true && green["tests"] == 4 && green["pass"] == 4 && green["fail"] == 0, "{green}"); + let src = std::fs::read_to_string(©).unwrap(); + std::fs::write(©, src.replacen("{ return 0; } // negative", "{ return x; } // negative", 1)).unwrap(); // #7400's plant + let (red, err) = tri_test(&root, &t27c); + assert!(err && red["passes"] == false && red["tests"] == 4 && red["pass"] == 3 && red["fail"] == 1 && red["failed"] == json!(["negz"]), "{red}"); + std::fs::write(©, &src).unwrap(); + assert_eq!(sha(©), before, "the copy is restored byte for byte"); + let stub = |name: &str, body: &str| { let p = root.join(name); assert!(Command::new("sh").args(["-c", "printf '%s' \"$2\" > \"$1\" && chmod +x \"$1\"", "sh"]).arg(&p).arg(body).status().unwrap().success()); p }; + let (listed, err) = tri_test(&root, &stub("lists-only", "#!/bin/sh\necho listed 4 tests\n")); // negative control: what `t27c test` gives + assert!(err && listed["passes"] == false && listed["verdict"] == 4 && listed["tests"] == 0, "{listed}"); + let (torn, err) = tri_test(&root, &stub("green-exit-1", "#!/bin/sh\nprintf '\\n tests 4\\n pass 4\\n FAIL 0\\n'\nexit 1\n")); + assert!(err && torn["passes"] == false && torn["verdict"] == 7 && torn["tests"] == 4, "{torn}"); // a green report from a failed run + std::fs::remove_dir_all(&root).unwrap(); + assert_eq!(sha(&spec), before, "the spec in the tree is untouched"); +} diff --git a/docs/reports/orphan_modules.json b/docs/reports/orphan_modules.json index 149a5c3104..1af6615a00 100644 --- a/docs/reports/orphan_modules.json +++ b/docs/reports/orphan_modules.json @@ -6,6 +6,7 @@ "ceilings": { "bootstrap": 7, "cli/tri": 0, + "cli/tri-mcp": 0, "cli/trios-bridge": 0, "cli/flash-spi": 0, "cli/dlc10": 0, diff --git a/specs/tools/mcp/tri-mcp.t27 b/specs/tools/mcp/tri-mcp.t27 index e3a9352289..2fe32f4698 100644 --- a/specs/tools/mcp/tri-mcp.t27 +++ b/specs/tools/mcp/tri-mcp.t27 @@ -2,7 +2,9 @@ ; specs/tools/mcp/tri-mcp.t27 -- tool mcp/tri-mcp, MCP server "tri-mcp" (gHashTag/t27) ; Source of truth for the tool card on t27.ai (#/tools?tool=mcp/tri-mcp). ; Tool names, descriptions and input keys are read from the server code or manifest named in -; SOURCE (witness source-parse, gHashTag/t27 @ 3cb7a4352492); nothing is typed by hand. +; SOURCE (witness source-parse, gHashTag/t27 @ 3cb7a4352492; tri_test's description and its t27c input +; were read from the branch that makes it run the tests, gHashTag/t27#8140. Before it, tri_test ran +; `t27c test`, which only lists the tests, and reported success whatever they did); nothing is typed by hand. ; Layer 5 of the ladder Specs -> Skills -> Crons -> Agents -> Tools. ASCII only (L3). ; phi^2 + 1/phi^2 = 3 | TRINITY @@ -27,8 +29,8 @@ pub const ABOUT : str = "Rust MCP server exposing the PHI LOOP tri commands as t pub const ABOUT_SOURCE : str = "derived from the tool descriptions in cli/tri-mcp/src/main.rs build_tools_list"; ; Tools as the server lists them (tools/list), their descriptions and inputSchema property keys. pub const TOOLS : [10]str = ["tri_status", "tri_skill_begin", "tri_skill_end", "tri_cell_checkpoint", "tri_cell_seal", "tri_gen", "tri_test", "tri_verdict", "tri_experience_save", "tri_health"]; -pub const TOOLS_ABOUT : [10]str = ["Returns PHI LOOP status including active skill, cell, issue binding, and uncommitted changes.", "Begins a new PHI LOOP skill session bound to an issue.", "Ends the active PHI LOOP skill session.", "Records a checkpoint step in the active cell.", "Seals the active cell by computing a SHA-256 hash of its checkpoint history.", "Generates backends from a .t27 spec file using t27c.", "Runs tests for a .t27 spec file using t27c.", "Checks for toxic regressions by analyzing swarm health metrics.", "Saves an experience episode to the akashic log.", "Returns queen and swarm health status from .trinity state."]; -pub const TOOLS_INPUTS : [10]str = ["", "issue,description", "", "step", "", "spec_path", "spec_path", "", "episode_type,summary,ring", ""]; +pub const TOOLS_ABOUT : [10]str = ["Returns PHI LOOP status including active skill, cell, issue binding, and uncommitted changes.", "Begins a new PHI LOOP skill session bound to an issue.", "Ends the active PHI LOOP skill session.", "Records a checkpoint step in the active cell.", "Seals the active cell by computing a SHA-256 hash of its checkpoint history.", "Generates backends from a .t27 spec file using t27c.", "Runs a .t27 spec's tests with `t27c test-report`; isError unless every test passed and at least one ran (rules: specs/tri/test/report.t27).", "Checks for toxic regressions by analyzing swarm health metrics.", "Saves an experience episode to the akashic log.", "Returns queen and swarm health status from .trinity state."]; +pub const TOOLS_INPUTS : [10]str = ["", "issue,description", "", "step", "", "spec_path", "spec_path,t27c", "", "episode_type,summary,ring", ""]; pub const RESOURCES : [0]str = []; pub const RESOURCES_ABOUT : [0]str = []; ; Why TOOLS may be empty or differ from other declarations (external package, dynamic list).