From 997a4c5bb9dac5b9da1490ed454b91c2a6ac2100 Mon Sep 17 00:00:00 2001 From: Dmitrii Vasilev Date: Mon, 5 Oct 2026 06:06:23 +0700 Subject: [PATCH 1/4] fix(codegen): sample the corpus compile check from the baseline The compile sample was every 32nd entry of the run's live clean list, so one spec counted clean that the baseline did not hold shifted every later pick by one. specs/tri/phi_utils_multi.tri is at baseline index 447, one before the slot at 448, and its output does not compile. All nine codegen-corpus reds on main from 09-24 to 10-04 name it, and e6eac0900 was red on push and green on schedule. - Slot k is baseline entry k * stride, so the sample is a function of the commit's baseline only. - Clean now needs the generator to exit 0 and leave a non-empty regular file. vibee_gen prints "Behaviors: N" before generating and exits 0 when it cannot write or produced 0 bytes; zig ast-check on a missing file says "error: unable to open file", which has no ": error: ", so such a spec counted as clean. - ast-check must also exit 0. - The output file is deleted before each generation, and its path mirrors the spec path: flattening '/' to '_' sent specs/storm/main.tri and specs/storm_main.tri to one file. - The clean count, regressed, new-clean and no-output specs are printed on both paths, by name. - codegen.yml records the runner's CPU model; vibee_gen is built for the native CPU. Fixes #885. Refs #851 (W15). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/codegen.yml | 8 +- tools/codegen_corpus.zig | 173 ++++++++++++++++++++++++++++------ 2 files changed, 152 insertions(+), 29 deletions(-) diff --git a/.github/workflows/codegen.yml b/.github/workflows/codegen.yml index dec7d0b3b..dc00d0dfc 100644 --- a/.github/workflows/codegen.yml +++ b/.github/workflows/codegen.yml @@ -50,8 +50,14 @@ jobs: tar xf zig.tar.xz echo "$PWD/zig-x86_64-linux-${ZIG_VERSION}" >> $GITHUB_PATH + # vibee_gen is built for the runner's native CPU (standardTargetOptions), + # so the CPU is part of what a codegen-corpus run measured. Two runs of + # one commit disagreed (e6eac0900: red on push, green on schedule); the + # model name is the cheapest thing to have on record next time. - name: zig version - run: zig version + run: | + zig version + grep -m1 'model name' /proc/cpuinfo || true # One question per step. The previous version asked "does it build and # generate and test" with a single exit code, so a compile error and a diff --git a/tools/codegen_corpus.zig b/tools/codegen_corpus.zig index c5bf73373..ad3a98b66 100644 --- a/tools/codegen_corpus.zig +++ b/tools/codegen_corpus.zig @@ -35,6 +35,15 @@ const out_dir = ".zig-cache/codegen-corpus"; /// `ternary_mathematics` failed the four-spec gate through five iterations of /// a change while a contiguous 24-spec prefix said everything was clean -- /// a prefix samples one directory, not the corpus. +/// +/// The sample is drawn from the committed BASELINE, not from this run's clean +/// list. It used to be every 32nd entry of the live list, so one spec counted +/// clean that the baseline did not hold shifted every later pick by one. +/// `specs/tri/phi_utils_multi.tri` sits at baseline index 447, one before the +/// slot at 448, and its output does not compile; all nine codegen-corpus reds +/// on main between 09-24 and 10-04 name it, and commit e6eac0900 was red on +/// push and green on schedule. A sample that moves with the run is a gate +/// whose verdict is not a function of the commit. const default_compile_sample = 24; pub fn main(init: std.process.Init.Minimal) !void { @@ -97,6 +106,8 @@ pub fn main(init: std.process.Init.Minimal) !void { var clean: std.ArrayList([]const u8) = .empty; defer clean.deinit(gpa); + var no_output: std.ArrayList([]const u8) = .empty; + defer no_output.deinit(gpa); var with_behaviours: usize = 0; var dirty: usize = 0; @@ -104,17 +115,34 @@ pub fn main(init: std.process.Init.Minimal) !void { const dest = try outPathFor(gpa, spec); defer gpa.free(dest); - const behaviours = try generate(gpa, io, gen, spec, dest); - if (behaviours == 0) continue; + const g = try generate(gpa, io, gen, spec, dest); + const no_behaviours = g.behaviours == 0; + if (no_behaviours) continue; with_behaviours += 1; - if (try astCheckOk(gpa, io, dest)) { + // The generator prints `Behaviors: N` BEFORE it generates, and on a + // generate or write error it leaves no file. `zig ast-check` on a + // missing file says "error: unable to open file", which has no + // ": error: " in it, so such a spec used to count as clean. + const left_no_file = !g.wrote; + if (left_no_file) { + try no_output.append(gpa, spec); + dirty += 1; + continue; + } + + const ast_clean = try astCheckOk(gpa, io, dest); + if (ast_clean) { try clean.append(gpa, spec); } else { dirty += 1; } } + for (no_output.items) |s| { + std.debug.print(" NO OUTPUT: {s} reported behaviours, then exited non-zero or left no file\n", .{s}); + } + if (update) { var file = try std.Io.Dir.cwd().createFile(io, baseline_path, .{}); defer file.close(io); @@ -140,34 +168,71 @@ pub fn main(init: std.process.Init.Minimal) !void { defer now.deinit(); for (clean.items) |s| try now.put(s, {}); - // A spec whose output USED to be clean and now is not. Regressions only -- - // a spec that was already failing is not this gate's business. - var regressed: usize = 0; + // The baseline in file order: the compile sample is drawn from it. + var base: std.ArrayList([]const u8) = .empty; + defer base.deinit(gpa); + var in_base = std.StringHashMap(void).init(gpa); + defer in_base.deinit(); var lines = std.mem.splitScalar(u8, baseline.?, '\n'); while (lines.next()) |line| { const t = std.mem.trim(u8, line, " \t\r"); - if (t.len == 0) continue; - if (!now.contains(t)) { + const blank = t.len == 0; + if (blank) continue; + try base.append(gpa, t); + try in_base.put(t, {}); + } + + // A spec whose output USED to be clean and now is not. Regressions only -- + // a spec that was already failing is not this gate's business. + var regressed: usize = 0; + for (base.items) |t| { + const lost = !now.contains(t); + if (lost) { std.debug.print(" REGRESSED: {s} no longer generates ast-check-clean output\n", .{t}); regressed += 1; } } + // Clean now, absent from the baseline. Not a failure -- the ratchet + // defends only what it recorded -- but named, because an unexplained one + // is the first thing to look at when two runs of one commit disagree. + var new_clean: usize = 0; + for (clean.items) |s| { + const unrecorded = !in_base.contains(s); + if (unrecorded) { + std.debug.print(" NEW CLEAN: {s} is not in the baseline\n", .{s}); + new_clean += 1; + } + } + // The compile sample: what ast-check cannot see. Arity, types, members. var compiled: usize = 0; var compile_failed: usize = 0; - const sample = @min(compile_sample, clean.items.len); - // Even stride, not a prefix: the clean list is sorted by path, so the - // first N are all from one directory. - const stride = if (sample == 0) 1 else clean.items.len / sample; + const sample = @min(compile_sample, base.items.len); + // Even stride, not a prefix: the baseline is sorted by path, so the first + // N are all from one directory. Slot k is baseline entry k * stride, so + // the picks depend on the commit's baseline and nothing else. + const stride = if (sample == 0) 1 else base.items.len / sample; var picked: std.ArrayList([]const u8) = .empty; defer picked.deinit(gpa); { - var i: usize = 0; - while (i < clean.items.len and picked.items.len < sample) : (i += @max(1, stride)) { - try picked.append(gpa, clean.items[i]); + var k: usize = 0; + while (k < sample) : (k += 1) { + const entry = base.items[k * stride]; + // A regressed entry is already a failure above; compiling its + // output would only report it twice. + const still_clean = now.contains(entry); + if (still_clean) try picked.append(gpa, entry); } } + + std.debug.print( + \\codegen corpus: {d} of {d} specs with behaviours clean, baseline {d} + \\ {d} regressed, {d} new clean, {d} reported behaviours and left no file + \\ compile sample: {d} slots, stride {d} over the baseline + \\ + , .{ clean.items.len, with_behaviours, base.items.len, regressed, new_clean, no_output.items.len, sample, stride }); + for (picked.items) |spec| { const dest = try outPathFor(gpa, spec); defer gpa.free(dest); @@ -179,7 +244,8 @@ pub fn main(init: std.process.Init.Minimal) !void { } } - if (regressed > 0 or compile_failed > 0) { + const failed = regressed > 0 or compile_failed > 0; + if (failed) { std.debug.print( \\ \\{d} spec(s) regressed, {d} of {d} sampled outputs do not compile. @@ -187,7 +253,7 @@ pub fn main(init: std.process.Init.Minimal) !void { \\Re-record only if the change is intended: \\ zig build codegen-corpus-update \\ - , .{ regressed, compile_failed, sample }); + , .{ regressed, compile_failed, picked.items.len }); return error.CodegenCorpusRegressed; } @@ -219,27 +285,55 @@ fn findGenerator(gpa: std.mem.Allocator, io: std.Io) ![]u8 { return owned; } +/// The output mirrors the spec's path under `out_dir`. It used to flatten +/// '/' to '_', which is not one-to-one: `specs/storm/main.tri` and +/// `specs/storm_main.tri` both became `specs_storm_main.tri.zig`, so the +/// compile sample could test the later spec's output under the earlier +/// spec's name. The generator creates the parent directory itself. fn outPathFor(gpa: std.mem.Allocator, spec: []const u8) ![]u8 { - const flat = try gpa.dupe(u8, spec); - defer gpa.free(flat); - for (flat) |*c| { - if (c.* == '/') c.* = '_'; - } - return std.fmt.allocPrint(gpa, "{s}/{s}.zig", .{ out_dir, flat }); + return std.fmt.allocPrint(gpa, "{s}/{s}.zig", .{ out_dir, spec }); } -/// Returns the behaviour count the generator reports, or 0. -fn generate(gpa: std.mem.Allocator, io: std.Io, gen: []const u8, spec: []const u8, dest: []const u8) !usize { +const Generated = struct { + /// The count the generator reports, or 0. + behaviours: usize, + /// The generator exited 0 and left a non-empty file at `dest`. + wrote: bool, +}; + +fn generate(gpa: std.mem.Allocator, io: std.Io, gen: []const u8, spec: []const u8, dest: []const u8) !Generated { + // A file left by an earlier run must not stand in for this one. + std.Io.Dir.cwd().deleteFile(io, dest) catch {}; + const r = tri_proc.runIo(io, .{ .allocator = gpa, .argv = &.{ gen, "gen", spec, dest }, .max_output_bytes = 256 * 1024, - }) catch return 0; + }) catch return .{ .behaviours = 0, .wrote = false }; defer gpa.free(r.stdout); defer gpa.free(r.stderr); + const exited_zero = switch (r.term) { + .exited => |c| c == 0, + else => false, + }; + // The generator exits 0 when it cannot create or write the file, and when + // it produced 0 bytes (`writeGenerated(...) catch return`), so the exit + // status alone is not enough. + const nonempty_file = blk: { + const st = std.Io.Dir.cwd().statFile(io, dest, .{}) catch break :blk false; + break :blk st.kind == .file and st.size > 0; + }; + return .{ + .behaviours = parseBehaviours(r.stdout, r.stderr), + .wrote = exited_zero and nonempty_file, + }; +} + +fn parseBehaviours(stdout: []const u8, stderr: []const u8) usize { const needle = "Behaviors: "; - const hay = if (std.mem.indexOf(u8, r.stderr, needle) != null) r.stderr else r.stdout; + const in_stderr = std.mem.indexOf(u8, stderr, needle) != null; + const hay = if (in_stderr) stderr else stdout; const at = std.mem.indexOf(u8, hay, needle) orelse return 0; var i = at + needle.len; var n: usize = 0; @@ -257,7 +351,30 @@ fn astCheckOk(gpa: std.mem.Allocator, io: std.Io, path: []const u8) !bool { }) catch return false; defer gpa.free(r.stdout); defer gpa.free(r.stderr); - return std.mem.indexOf(u8, r.stderr, ": error: ") == null; + // Exit status AND the error text. The text alone passed a missing file: + // "error: unable to open file '...': FileNotFound" exits 1 and has no + // ": error: " in it. + const exited_zero = switch (r.term) { + .exited => |c| c == 0, + else => false, + }; + const no_error_line = std.mem.indexOf(u8, r.stderr, ": error: ") == null; + return exited_zero and no_error_line; +} + +test "parseBehaviours reads the count from stderr first, then stdout" { + try std.testing.expectEqual(@as(usize, 7), parseBehaviours("", " Types: 2\n Behaviors: 7\n")); + try std.testing.expectEqual(@as(usize, 12), parseBehaviours("Behaviors: 12\n", "nothing here")); + try std.testing.expectEqual(@as(usize, 0), parseBehaviours("", "")); +} + +test "outPathFor is one-to-one where flattening was not" { + const gpa = std.testing.allocator; + const a = try outPathFor(gpa, "specs/storm/main.tri"); + defer gpa.free(a); + const b = try outPathFor(gpa, "specs/storm_main.tri"); + defer gpa.free(b); + try std.testing.expect(!std.mem.eql(u8, a, b)); } fn zigTestOk(gpa: std.mem.Allocator, io: std.Io, path: []const u8) !bool { From ca3170de0f4db366490f6039650d5c71dc9a837c Mon Sep 17 00:00:00 2001 From: Dmitrii Vasilev Date: Mon, 5 Oct 2026 06:21:11 +0700 Subject: [PATCH 2/4] fix(codegen): drop 19 baseline entries that never wrote a .zig (#885) The first CI run of the strict tool (run 37242677263) reported 769 of 1033 specs clean against a baseline of 788: 19 regressed, 0 new clean, and all 19 "reported behaviours and left no file". They were in the baseline only because the old tool ran `zig ast-check` on the missing output, and "unable to open file" has no ": error: " in it. 18 of the 19 declare `language: varlog`. Reading vibee_gen.zig (not running it), those go to the Verilog backend; the run measured that it then exits non-zero or leaves no file at the .zig destination, but not which. The 19th, specs/tri/long_context_system.vibee, declares `language: zig` and also leaves nothing. That log cannot say why, so NO OUTPUT now prints the exit status and the generator's last error line instead of "exited non-zero or left no file". The baseline becomes the old one minus exactly those 19, in order: 769 entries, which is the clean set the strict run measured. 15 of the 19 sort before specs/tri/phi_utils_multi.tri, so it moves from index 447 to 432; with stride 32 the slots near it are 416 and 448, so it is still not sampled. describeFailure has a unit test; a mutant that prefers the last stderr line over the last error line fails it. Refs #885, #886. Co-Authored-By: Claude Opus 5.5 --- tools/codegen_corpus.zig | 79 ++++++++++++++++++++++++++++--- tools/codegen_corpus_baseline.txt | 19 -------- 2 files changed, 73 insertions(+), 25 deletions(-) diff --git a/tools/codegen_corpus.zig b/tools/codegen_corpus.zig index ad3a98b66..22bc0bf51 100644 --- a/tools/codegen_corpus.zig +++ b/tools/codegen_corpus.zig @@ -106,8 +106,12 @@ pub fn main(init: std.process.Init.Minimal) !void { var clean: std.ArrayList([]const u8) = .empty; defer clean.deinit(gpa); - var no_output: std.ArrayList([]const u8) = .empty; - defer no_output.deinit(gpa); + const NoOutput = struct { spec: []const u8, why: []u8 }; + var no_output: std.ArrayList(NoOutput) = .empty; + defer { + for (no_output.items) |n| gpa.free(n.why); + no_output.deinit(gpa); + } var with_behaviours: usize = 0; var dirty: usize = 0; @@ -116,6 +120,7 @@ pub fn main(init: std.process.Init.Minimal) !void { defer gpa.free(dest); const g = try generate(gpa, io, gen, spec, dest); + defer if (g.why) |w| gpa.free(w); const no_behaviours = g.behaviours == 0; if (no_behaviours) continue; with_behaviours += 1; @@ -126,7 +131,9 @@ pub fn main(init: std.process.Init.Minimal) !void { // ": error: " in it, so such a spec used to count as clean. const left_no_file = !g.wrote; if (left_no_file) { - try no_output.append(gpa, spec); + const why = try gpa.dupe(u8, g.why orelse "(no reason recorded)"); + errdefer gpa.free(why); + try no_output.append(gpa, .{ .spec = spec, .why = why }); dirty += 1; continue; } @@ -139,8 +146,8 @@ pub fn main(init: std.process.Init.Minimal) !void { } } - for (no_output.items) |s| { - std.debug.print(" NO OUTPUT: {s} reported behaviours, then exited non-zero or left no file\n", .{s}); + for (no_output.items) |n| { + std.debug.print(" NO OUTPUT: {s} reported behaviours, then {s}\n", .{ n.spec, n.why }); } if (update) { @@ -299,6 +306,10 @@ const Generated = struct { behaviours: usize, /// The generator exited 0 and left a non-empty file at `dest`. wrote: bool, + /// When `wrote` is false: the exit status and the generator's last error + /// line, owned by the caller. "Left no file" without a reason sent the + /// first reader of this gate's output to rebuild the generator locally. + why: ?[]u8 = null, }; fn generate(gpa: std.mem.Allocator, io: std.Io, gen: []const u8, spec: []const u8, dest: []const u8) !Generated { @@ -324,12 +335,68 @@ fn generate(gpa: std.mem.Allocator, io: std.Io, gen: []const u8, spec: []const u const st = std.Io.Dir.cwd().statFile(io, dest, .{}) catch break :blk false; break :blk st.kind == .file and st.size > 0; }; + const wrote = exited_zero and nonempty_file; + const why: ?[]u8 = if (wrote) null else try describeFailure(gpa, r.term, nonempty_file, r.stderr); return .{ .behaviours = parseBehaviours(r.stdout, r.stderr), - .wrote = exited_zero and nonempty_file, + .wrote = wrote, + .why = why, }; } +/// "exit 1, error: Foo" or "exit 0, no file, Error: the generator produced +/// 0 bytes for ...". The line is the last one that starts with `error`/`Error` +/// once trimmed -- Zig's own report of an error returned from main, and the +/// generator's write and empty-output messages, all have that shape; the +/// parser's " spec error: ..." lines do not. Failing that, the last +/// non-empty line. +fn describeFailure(gpa: std.mem.Allocator, term: tri_proc.Term, nonempty_file: bool, stderr: []const u8) ![]u8 { + var status_buf: [32]u8 = undefined; + const status = switch (term) { + .exited => |c| try std.fmt.bufPrint(&status_buf, "exit {d}", .{c}), + .signal => |s| try std.fmt.bufPrint(&status_buf, "signal {d}", .{@intFromEnum(s)}), + .stopped => |s| try std.fmt.bufPrint(&status_buf, "stopped {d}", .{@intFromEnum(s)}), + .unknown => |u| try std.fmt.bufPrint(&status_buf, "unknown {d}", .{u}), + }; + const file_note: []const u8 = if (nonempty_file) "" else ", no file"; + + var last_error: ?[]const u8 = null; + var last_line: ?[]const u8 = null; + var it = std.mem.splitScalar(u8, stderr, '\n'); + while (it.next()) |raw| { + const line = std.mem.trim(u8, raw, " \t\r"); + const blank = line.len == 0; + if (blank) continue; + last_line = line; + const is_error_line = std.mem.startsWith(u8, line, "error") or std.mem.startsWith(u8, line, "Error"); + if (is_error_line) last_error = line; + } + const picked = last_error orelse last_line orelse "(no stderr)"; + const shown = picked[0..@min(picked.len, 200)]; + return std.fmt.allocPrint(gpa, "{s}{s}, {s}", .{ status, file_note, shown }); +} + +test "describeFailure names the exit status and the last error line" { + const gpa = std.testing.allocator; + const stderr = + \\ spec error: missing field + \\Generating Verilog... + \\error: UnsupportedType + \\/src/vibeec/verilog_codegen.zig:10:5: 0x1 in generate + ; + const a = try describeFailure(gpa, .{ .exited = 1 }, false, stderr); + defer gpa.free(a); + try std.testing.expectEqualStrings("exit 1, no file, error: UnsupportedType", a); + + const b = try describeFailure(gpa, .{ .exited = 0 }, false, "Error: the generator produced 0 bytes for x.zig\n"); + defer gpa.free(b); + try std.testing.expectEqualStrings("exit 0, no file, Error: the generator produced 0 bytes for x.zig", b); + + const c = try describeFailure(gpa, .{ .exited = 0 }, false, ""); + defer gpa.free(c); + try std.testing.expectEqualStrings("exit 0, no file, (no stderr)", c); +} + fn parseBehaviours(stdout: []const u8, stderr: []const u8) usize { const needle = "Behaviors: "; const in_stderr = std.mem.indexOf(u8, stderr, needle) != null; diff --git a/tools/codegen_corpus_baseline.txt b/tools/codegen_corpus_baseline.txt index 45f274d98..e4fba12af 100644 --- a/tools/codegen_corpus_baseline.txt +++ b/tools/codegen_corpus_baseline.txt @@ -79,16 +79,6 @@ specs/crypto/gf256.tri specs/depin/multi-cluster-live-v2.tri specs/depin/real-network-v1.tri specs/format/tri27_loader.tri -specs/fpga/blink.tri -specs/fpga/counter.tri -specs/fpga/fsm_simple.tri -specs/fpga/test_blink.tri -specs/fpga/test_counter.tri -specs/fpga/test_debouncer.tri -specs/fpga/test_multi_led.tri -specs/fpga/test_pwm.tri -specs/fpga/test_shift_register.tri -specs/fpga/test_values.tri specs/infra/build-purge-v2.tri specs/infra/build-transcendence-v1.tri specs/koschei/vsa_fpga_week2.tri @@ -164,7 +154,6 @@ specs/tri/benchmark_runner.vibee specs/tri/benchmarks_603x_final.vibee specs/tri/benchmarks_v7.vibee specs/tri/bitnet_inference.vibee -specs/tri/bitnet_mac.vibee specs/tri/black_hole_information.tri specs/tri/blind_spots_discovery.vibee specs/tri/bootstrap/impl_block_demo.tri @@ -303,9 +292,6 @@ specs/tri/fluent_general_chat.vibee specs/tri/formatters.vibee specs/tri/formula_discovery_engine.tri specs/tri/forward_pass.vibee -specs/tri/fpga/bitnet_mac.vibee -specs/tri/fpga/trinity_ai_core.vibee -specs/tri/fpga/trit_alu.vibee specs/tri/full_autonomous.vibee specs/tri/full_autonomous_sacred_evolution__autogit_commits__ml_patch_optimization__production_dashboard_deployment__selfhosting_loop.vibee specs/tri/generation_pipeline.vibee @@ -367,7 +353,6 @@ specs/tri/local_llm_fallback.vibee specs/tri/logger.vibee specs/tri/logging_middleware.vibee specs/tri/long_context_e2e.vibee -specs/tri/long_context_system.vibee specs/tri/mac_simulation.vibee specs/tri/main_menu.vibee specs/tri/make_sacred_intelligence_the_default_brain_of_tri_cli__live_dashboard__selfevolving_agent.vibee @@ -713,10 +698,8 @@ specs/tri/tri_vsa.tri specs/tri/tri_xml.tri specs/tri/tri_zenodo.tri specs/tri/trinity.tri -specs/tri/trinity_ai_core.vibee specs/tri/trinity_menu_system.vibee specs/tri/trinity_omega_awakening__full_selfawareness__multiagent_sacred_swarm__eternal_evolution_loop__sacred_governance.vibee -specs/tri/trit_alu.vibee specs/tri/tvc_integrated_system.vibee specs/tri/tvc_science.vibee specs/tri/tvc_service/tvc_http_api.tri @@ -779,8 +762,6 @@ specs/tri_trie.tri specs/trinet/trinet_node.tri specs/trinity/vsa_fpga_week2.tri specs/trinity/vsa_fpga_week3.tri -specs/uart/uart_mode_controller.tri -specs/uart/vsa_uart_operations.tri specs/vibee/self-mod-v1.tri specs/vibeec/emitter.tri specs/vibeec/parser.tri From d774e9cf2767b5610d457bd32f28e477010df1c7 Mon Sep 17 00:00:00 2001 From: Dmitrii Vasilev Date: Mon, 5 Oct 2026 06:29:43 +0700 Subject: [PATCH 3/4] fix(codegen): name the panic, not the last stack frame (#885) #886's own run on 02b9958de (37243448031) printed the reason for all 19 NO OUTPUT specs as "signal 6, no file, ???:?:?: 0x1571044 in ??? (???)": the generator aborts, and the last stderr line of a Zig panic is a frame with no symbols. describeFailure now prefers the last line holding "panic: ", then an error line, then the last line. A mutant that ranks error lines above the panic fails the new test case. Correction to 02b9958de's message. It said the 19 were in the baseline because the old tool ran `zig ast-check` on the missing output, and that the tool could not tell a non-zero exit from a missing file. Both parts are wrong or out of date: * All 19 end in SIGABRT (signal 6): measured in probe run 37242067946 and again in 37243448031 above. * 18 of them -- specs/fpga/*, specs/tri/bitnet_mac, specs/tri/fpga/*, specs/tri/trinity_ai_core, specs/tri/trit_alu, specs/uart/* -- leave an EMPTY file, not a missing one. `zig ast-check` passes an empty file with exit 0, so the old tool counted them clean through the empty file. Only specs/tri/long_context_system.vibee leaves no file, and it does so in 59 of 60 runs, not always. The 18 empty-file specs are the Verilog-target ones. Why they abort is read from the generator's source, not measured, and is filed separately as a compiler change. Refs #885, #886. Co-Authored-By: Claude Opus 5.5 --- tools/codegen_corpus.zig | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/tools/codegen_corpus.zig b/tools/codegen_corpus.zig index 22bc0bf51..83d17bf32 100644 --- a/tools/codegen_corpus.zig +++ b/tools/codegen_corpus.zig @@ -344,12 +344,14 @@ fn generate(gpa: std.mem.Allocator, io: std.Io, gen: []const u8, spec: []const u }; } -/// "exit 1, error: Foo" or "exit 0, no file, Error: the generator produced -/// 0 bytes for ...". The line is the last one that starts with `error`/`Error` -/// once trimmed -- Zig's own report of an error returned from main, and the -/// generator's write and empty-output messages, all have that shape; the -/// parser's " spec error: ..." lines do not. Failing that, the last -/// non-empty line. +/// "signal 6, no file, thread 9 panic: ..." or "exit 1, no file, error: Foo". +/// The line is the last one holding "panic: " -- 19 specs abort the generator +/// with SIGABRT (run 37243448031), and a panic's last line is a stack frame +/// with no symbols, "???:?:?: 0x1571044 in ??? (???)", the same for all 19. +/// Failing that, the last line that starts with `error`/`Error` once trimmed: +/// Zig's report of an error returned from main and the generator's write and +/// empty-output messages have that shape, the parser's " spec error: ..." +/// lines do not. Failing that, the last non-empty line. fn describeFailure(gpa: std.mem.Allocator, term: tri_proc.Term, nonempty_file: bool, stderr: []const u8) ![]u8 { var status_buf: [32]u8 = undefined; const status = switch (term) { @@ -360,6 +362,7 @@ fn describeFailure(gpa: std.mem.Allocator, term: tri_proc.Term, nonempty_file: b }; const file_note: []const u8 = if (nonempty_file) "" else ", no file"; + var last_panic: ?[]const u8 = null; var last_error: ?[]const u8 = null; var last_line: ?[]const u8 = null; var it = std.mem.splitScalar(u8, stderr, '\n'); @@ -368,10 +371,12 @@ fn describeFailure(gpa: std.mem.Allocator, term: tri_proc.Term, nonempty_file: b const blank = line.len == 0; if (blank) continue; last_line = line; + const is_panic_line = std.mem.startsWith(u8, line, "panic: ") or std.mem.indexOf(u8, line, " panic: ") != null; + if (is_panic_line) last_panic = line; const is_error_line = std.mem.startsWith(u8, line, "error") or std.mem.startsWith(u8, line, "Error"); if (is_error_line) last_error = line; } - const picked = last_error orelse last_line orelse "(no stderr)"; + const picked = last_panic orelse last_error orelse last_line orelse "(no stderr)"; const shown = picked[0..@min(picked.len, 200)]; return std.fmt.allocPrint(gpa, "{s}{s}, {s}", .{ status, file_note, shown }); } @@ -395,6 +400,22 @@ test "describeFailure names the exit status and the last error line" { const c = try describeFailure(gpa, .{ .exited = 0 }, false, ""); defer gpa.free(c); try std.testing.expectEqualStrings("exit 0, no file, (no stderr)", c); + + // The last frame is the one run 37243448031 printed for all 19 aborts; + // the two lines above it are illustrative, since that run printed only + // the frame. A panic wins over an earlier error line. + const aborted = + \\Generating Verilog... + \\error(gpa): Allocation size 512 bytes does not match free size 300. + \\thread 4242 panic: Invalid free + \\???:?:?: 0x1570f10 in ??? (???) + \\???:?:?: 0x1571044 in ??? (???) + ; + const d = try describeFailure(gpa, .{ .signal = .ABRT }, false, aborted); + defer gpa.free(d); + const want = try std.fmt.allocPrint(gpa, "signal {d}, no file, thread 4242 panic: Invalid free", .{@intFromEnum(std.posix.SIG.ABRT)}); + defer gpa.free(want); + try std.testing.expectEqualStrings(want, d); } fn parseBehaviours(stdout: []const u8, stderr: []const u8) usize { From 728130ae6aea7f777239bbf1181e3cec094eb4e3 Mon Sep 17 00:00:00 2001 From: Dmitrii Vasilev Date: Mon, 5 Oct 2026 06:40:07 +0700 Subject: [PATCH 4/4] fix(codegen): say "empty file" when it is one, and name crash-handler lines (#885) Run 37243963781 on d959eb722 printed the panic line for 18 of the 19 aborting specs: signal 6, no file, thread 2978 panic: programmer bug caused syscall error: FAULT That confirms #887's inference: the Verilog path writes freed memory, the write gets EFAULT, and the standard library panics. It also showed two defects in what this gate prints: - "no file" was printed for any file that was not non-empty. The probe (37242067946) found that 18 of the 19 leave an EMPTY file, which is a different failure: the file was created and the write then died. The gate's own wording repeated the mistake 02b9958de's message made. The generator's result is now one of none / empty / nonempty, and the reason says "no file" or "empty file". The summary line says "wrote no usable file" instead of "left no file". - specs/tri/long_context_system.vibee still printed only a stack frame, so its stderr has no "panic: " line. Zig 0.16's segfault handler prints "Segmentation fault at address 0x..." (std/debug.zig), then aborts. Lines starting with any of the handler names now count as crash lines. Which line long_context_system prints is not known yet; the next run says. The describeFailure test now uses the measured panic line, checks that an empty file is named as such, and adds a crash-handler case. Two mutants, each caught by exactly one assertion: - ignoring crash-handler lines: picks the earlier error line; - naming an empty file "no file". Co-Authored-By: Claude Opus 5.5 --- tools/codegen_corpus.zig | 112 ++++++++++++++++++++++++++++----------- 1 file changed, 80 insertions(+), 32 deletions(-) diff --git a/tools/codegen_corpus.zig b/tools/codegen_corpus.zig index 83d17bf32..eec088093 100644 --- a/tools/codegen_corpus.zig +++ b/tools/codegen_corpus.zig @@ -125,12 +125,13 @@ pub fn main(init: std.process.Init.Minimal) !void { if (no_behaviours) continue; with_behaviours += 1; - // The generator prints `Behaviors: N` BEFORE it generates, and on a - // generate or write error it leaves no file. `zig ast-check` on a + // The generator prints `Behaviors: N` BEFORE it generates, and when + // it fails it leaves no file or an empty one. `zig ast-check` on a // missing file says "error: unable to open file", which has no - // ": error: " in it, so such a spec used to count as clean. - const left_no_file = !g.wrote; - if (left_no_file) { + // ": error: " in it, and it passes an empty file outright, so both + // used to count as clean. + const no_usable_file = !g.wrote; + if (no_usable_file) { const why = try gpa.dupe(u8, g.why orelse "(no reason recorded)"); errdefer gpa.free(why); try no_output.append(gpa, .{ .spec = spec, .why = why }); @@ -235,7 +236,7 @@ pub fn main(init: std.process.Init.Minimal) !void { std.debug.print( \\codegen corpus: {d} of {d} specs with behaviours clean, baseline {d} - \\ {d} regressed, {d} new clean, {d} reported behaviours and left no file + \\ {d} regressed, {d} new clean, {d} reported behaviours and wrote no usable file \\ compile sample: {d} slots, stride {d} over the baseline \\ , .{ clean.items.len, with_behaviours, base.items.len, regressed, new_clean, no_output.items.len, sample, stride }); @@ -331,12 +332,15 @@ fn generate(gpa: std.mem.Allocator, io: std.Io, gen: []const u8, spec: []const u // The generator exits 0 when it cannot create or write the file, and when // it produced 0 bytes (`writeGenerated(...) catch return`), so the exit // status alone is not enough. - const nonempty_file = blk: { - const st = std.Io.Dir.cwd().statFile(io, dest, .{}) catch break :blk false; - break :blk st.kind == .file and st.size > 0; + const file: FileLeft = blk: { + const st = std.Io.Dir.cwd().statFile(io, dest, .{}) catch break :blk .none; + const not_a_file = st.kind != .file; + if (not_a_file) break :blk .none; + const is_empty = st.size == 0; + break :blk if (is_empty) .empty else .nonempty; }; - const wrote = exited_zero and nonempty_file; - const why: ?[]u8 = if (wrote) null else try describeFailure(gpa, r.term, nonempty_file, r.stderr); + const wrote = exited_zero and file == .nonempty; + const why: ?[]u8 = if (wrote) null else try describeFailure(gpa, r.term, file, r.stderr); return .{ .behaviours = parseBehaviours(r.stdout, r.stderr), .wrote = wrote, @@ -344,15 +348,36 @@ fn generate(gpa: std.mem.Allocator, io: std.Io, gen: []const u8, spec: []const u }; } -/// "signal 6, no file, thread 9 panic: ..." or "exit 1, no file, error: Foo". -/// The line is the last one holding "panic: " -- 19 specs abort the generator -/// with SIGABRT (run 37243448031), and a panic's last line is a stack frame -/// with no symbols, "???:?:?: 0x1571044 in ??? (???)", the same for all 19. +/// What the generator left at the destination. 18 of the 19 aborting specs +/// leave an EMPTY file (probe 37242067946), which is not the same failure as +/// leaving none: the file was created and the write then died. +const FileLeft = enum { none, empty, nonempty }; + +/// The first words of the line Zig 0.16's crash handlers print before a stack +/// trace (std/debug.zig, the POSIX and Windows segfault handlers, printed by +/// `defaultHandleSegfault`): +/// "Segmentation fault at address 0x...", and so on. Like a panic, these end +/// in SIGABRT, and like a panic they are followed by symbol-less frames. +const crash_handler_lines = [_][]const u8{ + "Segmentation fault", + "Illegal instruction", + "Bus error", + "Arithmetic exception", + "General protection exception", + "Stack overflow", + "Unaligned memory access", +}; + +/// "signal 6, empty file, thread 9 panic: ..." or "exit 1, no file, error: Foo". +/// The line is the last crash line: one holding "panic: ", or one a crash +/// handler printed. 19 specs abort the generator with SIGABRT (run +/// 37243448031), and an abort's last line is a stack frame with no symbols, +/// "???:?:?: 0x1571044 in ??? (???)", the same for all 19. /// Failing that, the last line that starts with `error`/`Error` once trimmed: /// Zig's report of an error returned from main and the generator's write and /// empty-output messages have that shape, the parser's " spec error: ..." /// lines do not. Failing that, the last non-empty line. -fn describeFailure(gpa: std.mem.Allocator, term: tri_proc.Term, nonempty_file: bool, stderr: []const u8) ![]u8 { +fn describeFailure(gpa: std.mem.Allocator, term: tri_proc.Term, file: FileLeft, stderr: []const u8) ![]u8 { var status_buf: [32]u8 = undefined; const status = switch (term) { .exited => |c| try std.fmt.bufPrint(&status_buf, "exit {d}", .{c}), @@ -360,9 +385,13 @@ fn describeFailure(gpa: std.mem.Allocator, term: tri_proc.Term, nonempty_file: b .stopped => |s| try std.fmt.bufPrint(&status_buf, "stopped {d}", .{@intFromEnum(s)}), .unknown => |u| try std.fmt.bufPrint(&status_buf, "unknown {d}", .{u}), }; - const file_note: []const u8 = if (nonempty_file) "" else ", no file"; + const file_note: []const u8 = switch (file) { + .none => ", no file", + .empty => ", empty file", + .nonempty => "", + }; - var last_panic: ?[]const u8 = null; + var last_crash: ?[]const u8 = null; var last_error: ?[]const u8 = null; var last_line: ?[]const u8 = null; var it = std.mem.splitScalar(u8, stderr, '\n'); @@ -372,11 +401,16 @@ fn describeFailure(gpa: std.mem.Allocator, term: tri_proc.Term, nonempty_file: b if (blank) continue; last_line = line; const is_panic_line = std.mem.startsWith(u8, line, "panic: ") or std.mem.indexOf(u8, line, " panic: ") != null; - if (is_panic_line) last_panic = line; + const is_handler_line = for (crash_handler_lines) |p| { + const starts = std.mem.startsWith(u8, line, p); + if (starts) break true; + } else false; + const is_crash_line = is_panic_line or is_handler_line; + if (is_crash_line) last_crash = line; const is_error_line = std.mem.startsWith(u8, line, "error") or std.mem.startsWith(u8, line, "Error"); if (is_error_line) last_error = line; } - const picked = last_panic orelse last_error orelse last_line orelse "(no stderr)"; + const picked = last_crash orelse last_error orelse last_line orelse "(no stderr)"; const shown = picked[0..@min(picked.len, 200)]; return std.fmt.allocPrint(gpa, "{s}{s}, {s}", .{ status, file_note, shown }); } @@ -389,33 +423,47 @@ test "describeFailure names the exit status and the last error line" { \\error: UnsupportedType \\/src/vibeec/verilog_codegen.zig:10:5: 0x1 in generate ; - const a = try describeFailure(gpa, .{ .exited = 1 }, false, stderr); + const a = try describeFailure(gpa, .{ .exited = 1 }, .none, stderr); defer gpa.free(a); try std.testing.expectEqualStrings("exit 1, no file, error: UnsupportedType", a); - const b = try describeFailure(gpa, .{ .exited = 0 }, false, "Error: the generator produced 0 bytes for x.zig\n"); + const b = try describeFailure(gpa, .{ .exited = 0 }, .none, "Error: the generator produced 0 bytes for x.zig\n"); defer gpa.free(b); try std.testing.expectEqualStrings("exit 0, no file, Error: the generator produced 0 bytes for x.zig", b); - const c = try describeFailure(gpa, .{ .exited = 0 }, false, ""); + const c = try describeFailure(gpa, .{ .exited = 0 }, .none, ""); defer gpa.free(c); try std.testing.expectEqualStrings("exit 0, no file, (no stderr)", c); - // The last frame is the one run 37243448031 printed for all 19 aborts; - // the two lines above it are illustrative, since that run printed only - // the frame. A panic wins over an earlier error line. + // The panic line and the last frame are the ones runs 37243963781 and + // 37243448031 printed for the 18 Verilog specs; the error line above them + // is illustrative. A panic wins over an earlier error line, and an empty + // file is named as such. const aborted = \\Generating Verilog... - \\error(gpa): Allocation size 512 bytes does not match free size 300. - \\thread 4242 panic: Invalid free + \\error: illustrative earlier error + \\thread 2978 panic: programmer bug caused syscall error: FAULT \\???:?:?: 0x1570f10 in ??? (???) \\???:?:?: 0x1571044 in ??? (???) ; - const d = try describeFailure(gpa, .{ .signal = .ABRT }, false, aborted); + const d = try describeFailure(gpa, .{ .signal = .ABRT }, .empty, aborted); defer gpa.free(d); - const want = try std.fmt.allocPrint(gpa, "signal {d}, no file, thread 4242 panic: Invalid free", .{@intFromEnum(std.posix.SIG.ABRT)}); - defer gpa.free(want); - try std.testing.expectEqualStrings(want, d); + const want_d = try std.fmt.allocPrint(gpa, "signal {d}, empty file, thread 2978 panic: programmer bug caused syscall error: FAULT", .{@intFromEnum(std.posix.SIG.ABRT)}); + defer gpa.free(want_d); + try std.testing.expectEqualStrings(want_d, d); + + // A crash handler's line counts as a crash line too. Its wording is + // std/debug.zig's; the address is illustrative. + const segv = + \\error: illustrative earlier error + \\Segmentation fault at address 0x7f0000001000 + \\???:?:?: 0x1571064 in ??? (???) + ; + const e = try describeFailure(gpa, .{ .signal = .ABRT }, .none, segv); + defer gpa.free(e); + const want_e = try std.fmt.allocPrint(gpa, "signal {d}, no file, Segmentation fault at address 0x7f0000001000", .{@intFromEnum(std.posix.SIG.ABRT)}); + defer gpa.free(want_e); + try std.testing.expectEqualStrings(want_e, e); } fn parseBehaviours(stdout: []const u8, stderr: []const u8) usize {