From e9c2486cc8353071c943fc454b47bc16856cdce1 Mon Sep 17 00:00:00 2001 From: Dmitrii Fedorov Date: Fri, 9 Oct 2026 21:23:44 -0300 Subject: [PATCH 1/3] fix(trinet): a coordinator restart no longer pays the same (node, nonce) again ledger.zig kept the paid (node, nonce) set in memory and mesh.zig started next_nonce at 1 on every start, so a restarted coordinator credited the same (node, nonce) twice (two runs, nonce 1: 2 mTRI) and a receipt saved from an earlier run could match a reissued nonce. journal.zig is the coordinator's durable record, per gHashTag/t27 specs/trinet/node-work-credit.t27: - a nonce mark is written before any nonce past it is issued, and a restart resumes at the last mark, so no nonce is issued twice; - a credit line is fsynced before the credit counts, and the paid set is rebuilt from the journal on start; - an unreadable journal or a malformed line refuses to open; only a torn last line is skipped; - each credit says its level: self_reported when the coordinator's operator is the node's owner or unknown, verified otherwise. A nonce this run did not issue, or a credit the journal could not take, is not_recorded: not credited and not slashed. TRINET_JOURNAL, TRINET_OWNER and TRINET_OPERATOR wire it into fleet, demo and agent; without TRINET_JOURNAL nothing changes. The trinet-tests job now also runs when src/trinet changes. zig test src/trinet/agent.zig -lc: 67/67 (11 new); 9/9 code mutants caught by a failing test. Closes #906 --- .../workflows/conformance-frame-alignment.yml | 3 + src/trinet/journal.zig | 271 ++++++++++++++++++ src/trinet/ledger.zig | 45 ++- src/trinet/main.zig | 45 ++- src/trinet/mesh.zig | 177 +++++++++++- 5 files changed, 536 insertions(+), 5 deletions(-) create mode 100644 src/trinet/journal.zig diff --git a/.github/workflows/conformance-frame-alignment.yml b/.github/workflows/conformance-frame-alignment.yml index dfedc82ea6..f4c5d91523 100644 --- a/.github/workflows/conformance-frame-alignment.yml +++ b/.github/workflows/conformance-frame-alignment.yml @@ -11,11 +11,14 @@ on: - 'conformance/frame_alignment_check.py' - 'conformance/gfternary_compute_conformance_ax7203.py' - 'conformance/trinet_mac32_conformance_ax7203.py' + # the TRI-NET stack the trinet-tests job runs: a change to it must run them + - 'src/trinet/**' pull_request: paths: - 'conformance/**' - 'fpga/vivado/**' - 'fpga/openxc7-synth/**' + - 'src/trinet/**' schedule: # Daily, so a wrapper edit that silently orphans a host surfaces within a # day rather than at the next flash session. diff --git a/src/trinet/journal.zig b/src/trinet/journal.zig new file mode 100644 index 0000000000..ea018ea5a7 --- /dev/null +++ b/src/trinet/journal.zig @@ -0,0 +1,271 @@ +//! The coordinator's durable journal: what it has paid, and how far it has +//! handed out nonces, so that neither is forgotten when the process stops. +//! +//! Spec: gHashTag/t27 specs/trinet/node-work-credit.t27 (gHashTag/t27#8426). +//! Issue: gHashTag/trinity-fpga#906. +//! +//! Before this file the paid (node, nonce) set lived in Ledger memory and the +//! coordinator's next nonce started at 1 on every start, so a restart paid the +//! same (node, nonce) again. The journal fixes both: +//! +//! - a mark line `{"mark":N}` is written BEFORE any nonce at or above the +//! previous mark is issued, so after a restart every nonce that may have +//! gone out lies below the last mark and the coordinator resumes at it; +//! - a credit line is written and flushed to disk BEFORE the credit counts, +//! and the paid set is rebuilt from those lines on start. +//! +//! Reading fails closed. A missing file is a fresh network; a file that exists +//! but cannot be read, or holds a complete line we cannot parse, is an error: +//! "we do not know what was paid" must never read as "nothing was paid". Only a +//! torn LAST line (no newline: the process died mid-write) is skipped, because +//! a credit counts only after its whole line is on disk and a nonce is issued +//! only after its mark is. +//! +//! The file is JSON lines so a person can read it and `jq` can sum it. It +//! needs libc for fsync; without libc a journal cannot be opened. + +const std = @import("std"); +const builtin = @import("builtin"); + +/// NONCE_RESERVE_BLOCK in node-work-credit.t27. +pub const reserve_block: u32 = 1024; + +pub const Error = error{ + JournalUnavailable, + JournalUnreadable, + JournalMalformed, + JournalWrite, + NonceSpaceExhausted, + OutOfMemory, +}; + +/// Who vouches for a credit (record levels in node-work-credit.t27). A credit +/// written by a coordinator whose operator is the node's owner, or whose +/// operator is unknown, is the owner's own word about their node. +pub const Level = enum { + self_reported, + verified, + + pub fn of(operator: []const u8, owner: []const u8) Level { + if (operator.len == 0 or std.mem.eql(u8, operator, owner)) return .self_reported; + return .verified; + } +}; + +/// The spec's next_mark: 0 when the 32-bit nonce space is spent, never a wrap. +pub fn nextMark(mark: u32) u32 { + if (mark > std.math.maxInt(u32) - reserve_block) return 0; + return mark + reserve_block; +} + +pub fn spentKey(node_id: u32, nonce: u32) u64 { + return (@as(u64, node_id) << 32) | nonce; +} + +pub const Restored = struct { + /// The highest mark on disk; 0 for a fresh journal. + mark: u32 = 0, + credits: u64 = 0, + torn_tail: bool = false, +}; + +const C = struct { + extern "c" fn fsync(fd: c_int) c_int; + extern "c" fn fileno(f: *std.c.FILE) c_int; + extern "c" fn access(path: [*:0]const u8, mode: c_int) c_int; + extern "c" fn fflush(f: *std.c.FILE) c_int; + extern "c" fn ferror(f: *std.c.FILE) c_int; +}; + +pub const Journal = struct { + path: [:0]const u8, + + /// Append one line and put it on disk before returning. + fn append(self: Journal, line: []const u8) Error!void { + if (comptime !builtin.link_libc) return Error.JournalUnavailable; + const f = std.c.fopen(self.path.ptr, "a") orelse return Error.JournalWrite; + var ok = std.c.fwrite(line.ptr, 1, line.len, f) == line.len; + ok = ok and C.fflush(f) == 0; + ok = ok and C.fsync(C.fileno(f)) == 0; + ok = std.c.fclose(f) == 0 and ok; + if (!ok) return Error.JournalWrite; + } + + pub fn writeMark(self: Journal, mark: u32) Error!void { + var buf: [32]u8 = undefined; + const line = std.fmt.bufPrint(&buf, "{{\"mark\":{d}}}\n", .{mark}) catch unreachable; + return self.append(line); + } + + pub const Credit = struct { + operator: []const u8, + owner: []const u8, + node_id: u32, + physical: bool, + nonce: u32, + y: i32, + tag: u64, + tag_kind: []const u8, + level: Level, + mtri: u64, + }; + + pub fn writeCredit(self: Journal, c: Credit) Error!void { + // Handles go into the line unescaped, so only handle characters pass. + if (!isHandle(c.owner) or !(c.operator.len == 0 or isHandle(c.operator))) return Error.JournalWrite; + var ts: std.c.timespec = undefined; + _ = std.c.clock_gettime(.REALTIME, &ts); + var buf: [512]u8 = undefined; + const line = std.fmt.bufPrint(&buf, "{{\"t\":{d},\"operator\":\"{s}\",\"owner\":\"{s}\",\"node\":\"{x:0>8}\",\"physical\":{},\"nonce\":{d},\"y\":{d},\"tag\":\"{x:0>16}\",\"tag_kind\":\"{s}\",\"level\":\"{s}\",\"mtri\":{d}}}\n", .{ + ts.sec, c.operator, c.owner, c.node_id, c.physical, c.nonce, c.y, c.tag, c.tag_kind, @tagName(c.level), c.mtri, + }) catch return Error.JournalWrite; + return self.append(line); + } + + /// Rebuild the paid set and the nonce mark from disk. + pub fn restore(self: Journal, gpa: std.mem.Allocator, spent: *std.AutoHashMapUnmanaged(u64, void)) Error!Restored { + if (comptime !builtin.link_libc) return Error.JournalUnavailable; + if (C.access(self.path.ptr, 0) != 0) return .{}; + const f = std.c.fopen(self.path.ptr, "r") orelse return Error.JournalUnreadable; + defer _ = std.c.fclose(f); + + var text: std.ArrayList(u8) = .empty; + defer text.deinit(gpa); + var chunk: [4096]u8 = undefined; + while (true) { + const n = std.c.fread(&chunk, 1, chunk.len, f); + if (n == 0) break; + try text.appendSlice(gpa, chunk[0..n]); + } + if (C.ferror(f) != 0) return Error.JournalUnreadable; + return parse(gpa, text.items, spent); + } +}; + +fn isHandle(s: []const u8) bool { + if (s.len == 0) return false; + for (s) |ch| if (!std.ascii.isAlphanumeric(ch) and ch != '-' and ch != '_') return false; + return true; +} + +/// The text of a journal into the paid set and the mark. Public for tests. +pub fn parse(gpa: std.mem.Allocator, text: []const u8, spent: *std.AutoHashMapUnmanaged(u64, void)) Error!Restored { + var out: Restored = .{}; + var rest = text; + while (rest.len > 0) { + const nl = std.mem.indexOfScalar(u8, rest, '\n') orelse { + out.torn_tail = true; + break; + }; + const line = rest[0..nl]; + rest = rest[nl + 1 ..]; + if (line.len == 0) continue; + + if (std.mem.startsWith(u8, line, "{\"mark\":")) { + const v = numberAfter(line, "{\"mark\":") orelse return Error.JournalMalformed; + const m = std.fmt.parseInt(u32, v, 10) catch return Error.JournalMalformed; + if (m > out.mark) out.mark = m; + continue; + } + const node_hex = stringAfter(line, "\"node\":\"") orelse return Error.JournalMalformed; + const node_id = std.fmt.parseInt(u32, node_hex, 16) catch return Error.JournalMalformed; + const nonce_s = numberAfter(line, "\"nonce\":") orelse return Error.JournalMalformed; + const nonce = std.fmt.parseInt(u32, nonce_s, 10) catch return Error.JournalMalformed; + try spent.put(gpa, spentKey(node_id, nonce), {}); + out.credits += 1; + } + return out; +} + +fn stringAfter(line: []const u8, key: []const u8) ?[]const u8 { + const at = std.mem.indexOf(u8, line, key) orelse return null; + const from = at + key.len; + const end = std.mem.indexOfScalarPos(u8, line, from, '"') orelse return null; + return line[from..end]; +} + +fn numberAfter(line: []const u8, key: []const u8) ?[]const u8 { + const at = std.mem.indexOf(u8, line, key) orelse return null; + const from = at + key.len; + var end = from; + while (end < line.len and std.ascii.isDigit(line[end])) end += 1; + if (end == from) return null; + return line[from..end]; +} + +// --------------------------------------------------------------------------- + +const testing = std.testing; + +test "the nonce mark moves in blocks and never wraps" { + try testing.expectEqual(@as(u32, 1024), reserve_block); + try testing.expectEqual(@as(u32, 1025), nextMark(1)); + try testing.expectEqual(@as(u32, 3072), nextMark(2048)); + try testing.expectEqual(@as(u32, std.math.maxInt(u32)), nextMark(std.math.maxInt(u32) - 1024)); + try testing.expectEqual(@as(u32, 0), nextMark(std.math.maxInt(u32) - 1023)); +} + +test "a record the node's owner vouches for is self-reported" { + try testing.expectEqual(Level.self_reported, Level.of("", "dmitrii-f-t27")); + try testing.expectEqual(Level.self_reported, Level.of("dmitrii-f-t27", "dmitrii-f-t27")); + try testing.expectEqual(Level.verified, Level.of("gHashTag", "dmitrii-f-t27")); +} + +test "a journal rebuilds the paid set and the highest mark" { + var spent: std.AutoHashMapUnmanaged(u64, void) = .empty; + defer spent.deinit(testing.allocator); + const text = + "{\"mark\":1024}\n" ++ + "{\"t\":1,\"operator\":\"\",\"owner\":\"a\",\"node\":\"4e4f4431\",\"physical\":false,\"nonce\":1,\"y\":3,\"tag\":\"00\",\"tag_kind\":\"keyed\",\"level\":\"self_reported\",\"mtri\":1}\n" ++ + "{\"mark\":2048}\n"; + const r = try parse(testing.allocator, text, &spent); + try testing.expectEqual(@as(u32, 2048), r.mark); + try testing.expectEqual(@as(u64, 1), r.credits); + try testing.expect(spent.contains(spentKey(0x4E4F4431, 1))); + try testing.expect(!r.torn_tail); +} + +test "a torn last line is skipped, a malformed whole line is refused" { + var spent: std.AutoHashMapUnmanaged(u64, void) = .empty; + defer spent.deinit(testing.allocator); + const torn = "{\"mark\":1024}\n{\"t\":1,\"owner\":\"a\",\"node\":\"4e4f"; + const r = try parse(testing.allocator, torn, &spent); + try testing.expect(r.torn_tail); + try testing.expectEqual(@as(u32, 1024), r.mark); + try testing.expectEqual(@as(u64, 0), r.credits); + + try testing.expectError(Error.JournalMalformed, parse(testing.allocator, "{\"owner\":\"a\"}\n", &spent)); + try testing.expectError(Error.JournalMalformed, parse(testing.allocator, "{\"mark\":x}\n", &spent)); + try testing.expectError(Error.JournalMalformed, parse(testing.allocator, "{\"node\":\"zz\",\"nonce\":1}\n", &spent)); +} + +test "a missing journal is fresh, an unreadable one is an error" { + if (comptime !builtin.link_libc) return error.SkipZigTest; + var spent: std.AutoHashMapUnmanaged(u64, void) = .empty; + defer spent.deinit(testing.allocator); + const missing: Journal = .{ .path = "/tmp/trinet-journal-test-missing-906.jsonl" }; + _ = std.c.unlink(missing.path.ptr); + const r = try missing.restore(testing.allocator, &spent); + try testing.expectEqual(@as(u32, 0), r.mark); + + // A directory exists but cannot be read as a journal. + const dir: Journal = .{ .path = "/tmp" }; + try testing.expectError(Error.JournalUnreadable, dir.restore(testing.allocator, &spent)); +} + +test "a mark and a credit written are read back" { + if (comptime !builtin.link_libc) return error.SkipZigTest; + const j: Journal = .{ .path = "/tmp/trinet-journal-test-roundtrip-906.jsonl" }; + _ = std.c.unlink(j.path.ptr); + defer _ = std.c.unlink(j.path.ptr); + try j.writeMark(1024); + try j.writeCredit(.{ .operator = "", .owner = "dmitrii-f-t27", .node_id = 0x5452494E, .physical = true, .nonce = 7, .y = -2, .tag = 0xABCD, .tag_kind = "keyed", .level = .self_reported, .mtri = 1 }); + try testing.expectError(Error.JournalWrite, j.writeCredit(.{ .operator = "", .owner = "a\"b", .node_id = 1, .physical = false, .nonce = 1, .y = 0, .tag = 0, .tag_kind = "keyed", .level = .self_reported, .mtri = 1 })); + + var spent: std.AutoHashMapUnmanaged(u64, void) = .empty; + defer spent.deinit(testing.allocator); + const r = try j.restore(testing.allocator, &spent); + try testing.expectEqual(@as(u32, 1024), r.mark); + try testing.expectEqual(@as(u64, 1), r.credits); + try testing.expect(spent.contains(spentKey(0x5452494E, 7))); +} diff --git a/src/trinet/ledger.zig b/src/trinet/ledger.zig index 54aea75a9c..d52a78ad12 100644 --- a/src/trinet/ledger.zig +++ b/src/trinet/ledger.zig @@ -28,6 +28,7 @@ const std = @import("std"); const protocol = @import("protocol.zig"); +const journal_mod = @import("journal.zig"); /// Credits are tracked in milli-TRI so that per-job rewards stay integral. pub const milli: u64 = 1000; @@ -131,6 +132,11 @@ pub const Outcome = enum { unverifiable_not_charged, /// Node is suspended and should not have been dispatched to. not_eligible, + /// The coordinator could not put the credit on disk, or the nonce was not + /// issued by this coordinator run. Not credited and NOT slashed: both are + /// statements about us. A credit that is not on disk does not count + /// (gHashTag/t27 specs/trinet/node-work-credit.t27 may_credit). + not_recorded, }; pub const Settlement = struct { @@ -160,6 +166,12 @@ pub const Ledger = struct { total_corrupted: u64 = 0, total_unverifiable: u64 = 0, jobs_on_silicon: u64 = 0, + /// The durable record of what was paid (journal.zig). Without one, the + /// paid set lives in memory and a restart forgets it. + journal: ?journal_mod.Journal = null, + /// Who runs this coordinator. Empty, or equal to a node's owner, makes + /// that node's credits self-reported. + operator: []const u8 = "", pub fn init(gpa: std.mem.Allocator, policy: Policy) Error!Ledger { if (!policy.isSound()) return Error.UnsoundPolicy; @@ -171,6 +183,16 @@ pub const Ledger = struct { self.spent_nonces.deinit(self.gpa); } + /// Open the coordinator's journal: rebuild the paid set from it and return + /// the nonce mark the coordinator must resume at. An unreadable journal is + /// an error, never an empty one. + pub fn openJournal(self: *Ledger, j: journal_mod.Journal, operator: []const u8) journal_mod.Error!journal_mod.Restored { + const r = try j.restore(self.gpa, &self.spent_nonces); + self.journal = j; + self.operator = operator; + return r; + } + /// A developer attaches a node and bonds a stake. This is the whole /// onboarding step: an identity, an owner, and something to lose. pub fn register(self: *Ledger, node_id: u32, owner: []const u8, physical: bool, stake_mtri: u64) Error!void { @@ -300,9 +322,30 @@ pub const Ledger = struct { .detail = "receipt for an already-settled nonce", }; } + const reward = self.policy.reward_per_job_mtri; + + // Write before credit: with a journal, a credit counts only once its + // line is on disk, so a crash can lose a credit but never repeat one. + if (self.journal) |j| { + j.writeCredit(.{ + .operator = self.operator, + .owner = acct.owner, + .node_id = dispatched_to, + .physical = acct.physical, + .nonce = job.nonceValue(), + .y = receipt.y, + .tag = receipt.tag, + .tag_kind = @tagName(receipt.kind), + .level = journal_mod.Level.of(self.operator, acct.owner), + .mtri = reward, + }) catch return .{ + .node_id = dispatched_to, + .outcome = .not_recorded, + .detail = "journal write failed: not credited", + }; + } try self.spent_nonces.put(self.gpa, key, {}); - const reward = self.policy.reward_per_job_mtri; acct.credit_mtri += reward; acct.accepted += 1; acct.consecutive_rejections = 0; diff --git a/src/trinet/main.zig b/src/trinet/main.zig index 29e80723c9..a703c74276 100644 --- a/src/trinet/main.zig +++ b/src/trinet/main.zig @@ -285,6 +285,45 @@ var fleet_nodes = [_]FleetNode{ const key_file_env = "TRINET_KEYS"; const key_file_default = "trinet-keys.txt"; +/// The coordinator's journal (journal.zig, gHashTag/trinity-fpga#906). Unset: +/// credits live in memory, as before. Set: paid (node, nonce) pairs and the +/// nonce mark survive a restart, and a credit counts only once it is on disk. +const journal_env = "TRINET_JOURNAL"; +/// The handle a board attached by this process is credited to. +const owner_env = "TRINET_OWNER"; +/// Who runs this coordinator. Unset, or equal to the owner, writes every +/// credit as self-reported (gHashTag/t27 specs/trinet/node-work-credit.t27). +const operator_env = "TRINET_OPERATOR"; + +fn envOr(name: [*:0]const u8, default: []const u8) []const u8 { + const v = std.c.getenv(name) orelse return default; + const s = std.mem.span(v); + return if (s.len == 0) default else s; +} + +fn ownerHandle() []const u8 { + return envOr(owner_env, "operator"); +} + +/// Open TRINET_JOURNAL if it is set. A journal that cannot be read stops the +/// run: crediting without knowing what was already paid is the bug it fixes. +fn openJournalFromEnv(m: *mesh_mod.Mesh) !void { + const p = std.c.getenv(journal_env) orelse return; + const path = std.mem.span(p); + const operator = envOr(operator_env, ""); + const r = m.openJournal(.{ .path = path }, operator) catch |e| { + std.debug.print("journal {s}: {s} -- not crediting without it\n", .{ path, @errorName(e) }); + return e; + }; + std.debug.print("journal {s}: {d} paid jobs restored, nonces resume at {d}{s}, credits {s}\n", .{ + path, + r.credits, + m.next_nonce, + if (r.torn_tail) " (a torn last line was skipped)" else "", + if (operator.len == 0) "self-reported (TRINET_OPERATOR unset)" else "by operator", + }); +} + /// The UART divisor the fleet bitstream is built with. The board's line rate is /// CFGMCLK / this, so it is also the only honest way to read CFGMCLK back out of /// a negotiated rate. @@ -352,6 +391,7 @@ fn fleet(gpa: std.mem.Allocator, ports: []const [:0]const u8) !void { var m = try mesh_mod.Mesh.init(gpa, .{}); defer m.deinit(); + try openJournalFromEnv(&m); var attached: usize = 0; var stale_boards: usize = 0; @@ -406,7 +446,7 @@ fn fleet(gpa: std.mem.Allocator, ports: []const [:0]const u8) !void { stale_boards += 1; std.debug.print("{s}: PUBLISHED KEY — receipts carry no evidence. Re-flash with `trinet keygen`.\n", .{p}); } - try m.join(n, "operator", 100000); + try m.join(n, ownerHandle(), 100000); attached += 1; std.debug.print("{s}: identified as {s}, id {x:0>8} at {d} baud{s}\n", .{ p, spec.?.name, spec.?.id, found.baud, @@ -716,11 +756,12 @@ fn bench(gpa: std.mem.Allocator, path: []const u8, n: usize, baud: u32) !void { fn buildMesh(gpa: std.mem.Allocator, serial_path: ?[]const u8, buf: []u8) !mesh_mod.Mesh { var m = try mesh_mod.Mesh.init(gpa, .{}); errdefer m.deinit(); + try openJournalFromEnv(&m); if (serial_path) |p| { const zpath = try std.fmt.bufPrintZ(buf, "{s}", .{p}); if (node_mod.Node.initFpga(protocol.default_node_id, "ax7203-node0", zpath, default_baud)) |fpga| { - try m.join(fpga, "operator", 100000); + try m.join(fpga, ownerHandle(), 100000); std.debug.print("node 0: physical AX7203 on {s}\n", .{p}); } else |e| { std.debug.print("node 0: no board ({s}) — running without a physical node\n", .{@errorName(e)}); diff --git a/src/trinet/mesh.zig b/src/trinet/mesh.zig index af96125e1c..2cfd12dbfb 100644 --- a/src/trinet/mesh.zig +++ b/src/trinet/mesh.zig @@ -18,6 +18,7 @@ const std = @import("std"); const protocol = @import("protocol.zig"); const node_mod = @import("node.zig"); const ledger_mod = @import("ledger.zig"); +const journal_mod = @import("journal.zig"); pub const Node = node_mod.Node; pub const Ledger = ledger_mod.Ledger; @@ -34,6 +35,11 @@ pub const Error = error{ DuplicateNode, InsufficientStake, UnsoundPolicy, + /// The journal could not take the next nonce mark, so no nonce is issued. + JournalWrite, + JournalUnavailable, + /// The 32-bit nonce space is spent; nodes need a new key epoch. + NonceSpaceExhausted, }; pub const JobOutcome = struct { @@ -68,6 +74,8 @@ pub const Stats = struct { corrupt_jobs: u64 = 0, unverifiable_jobs: u64 = 0, not_eligible_jobs: u64 = 0, + /// Verified but not put on disk, or a nonce this run did not issue. + not_recorded_jobs: u64 = 0, on_silicon: u64 = 0, in_software: u64 = 0, @@ -92,6 +100,10 @@ pub const Mesh = struct { stats: Stats = .{}, cursor: usize = 0, next_nonce: u32 = 1, + /// With a journal: the highest nonce mark on disk, and where this run + /// resumed. Nonces are issued only in [resumed_from, nonce_mark). + nonce_mark: u32 = 0, + resumed_from: u32 = 1, pub fn init(gpa: std.mem.Allocator, policy: ledger_mod.Policy) Error!Mesh { return .{ .gpa = gpa, .ledger = try Ledger.init(gpa, policy) }; @@ -122,12 +134,38 @@ pub const Mesh = struct { return k; } - pub fn freshNonce(self: *Mesh) u32 { + /// Open the coordinator's journal (journal.zig): rebuild what was paid and + /// resume nonces at the last mark, so a restart never reissues a nonce. + pub fn openJournal(self: *Mesh, j: journal_mod.Journal, operator: []const u8) journal_mod.Error!journal_mod.Restored { + const r = try self.ledger.openJournal(j, operator); + self.nonce_mark = r.mark; + self.next_nonce = if (r.mark == 0) 1 else r.mark; + self.resumed_from = self.next_nonce; + return r; + } + + pub fn freshNonce(self: *Mesh) Error!u32 { + if (self.ledger.journal) |j| { + // Reserve before issuing: the mark is on disk before any nonce at + // or above the old mark goes out. + if (self.next_nonce >= self.nonce_mark) { + const mark = journal_mod.nextMark(self.nonce_mark); + if (mark == 0 or mark <= self.next_nonce) return Error.NonceSpaceExhausted; + j.writeMark(mark) catch return Error.JournalWrite; + self.nonce_mark = mark; + } + } const n = self.next_nonce; self.next_nonce +%= 1; return n; } + /// With a journal, only a nonce this run issued may be credited. + fn issuedByThisRun(self: *Mesh, nonce: u32) bool { + if (self.ledger.journal == null) return true; + return nonce >= self.resumed_from and nonce < self.next_nonce; + } + fn pickEligible(self: *Mesh) ?*Node { const count = self.nodes.items.len; if (count == 0) return null; @@ -167,6 +205,18 @@ pub const Mesh = struct { if (job.nonceValue() > n.highest_nonce_issued) n.highest_nonce_issued = job.nonceValue(); + if (!self.issuedByThisRun(job.nonceValue())) { + self.stats.not_recorded_jobs += 1; + return .{ + .node_id = n.id, + .node_name = n.name, + .physical = n.isPhysical(), + .y = receipt.y, + .verdict = protocol.verifyWithKey(job, receipt, n.key), + .settlement = .{ .node_id = n.id, .outcome = .not_recorded, .detail = "nonce not issued by this coordinator run" }, + }; + } + var verdict = protocol.verifyWithKey(job, receipt, n.key); // A nonce mismatch means either a replay attack or a stream that lost a @@ -198,6 +248,7 @@ pub const Mesh = struct { // We declined to use the node. That is our scheduling decision, not // the node's conduct. .not_eligible => self.stats.not_eligible_jobs += 1, + .not_recorded => self.stats.not_recorded_jobs += 1, .rejected_and_slashed, .identity_mismatch => { self.stats.rejected += 1; n.stats.rejected += 1; @@ -320,7 +371,7 @@ pub const Mesh = struct { for (0..take) |k| { const row = next_row + k; - const nonce = self.freshNonce(); + const nonce = try self.freshNonce(); if (nonce > n.highest_nonce_issued) n.highest_nonce_issued = nonce; jobs[k] = protocol.Job.withNonce(nonce, rows[row], x); row_of[k] = row; @@ -350,6 +401,7 @@ pub const Mesh = struct { .corrupt_not_charged => self.stats.corrupt_jobs += 1, .unverifiable_not_charged => self.stats.unverifiable_jobs += 1, .not_eligible => self.stats.not_eligible_jobs += 1, + .not_recorded => self.stats.not_recorded_jobs += 1, .rejected_and_slashed, .identity_mismatch => { self.stats.rejected += 1; n.stats.rejected += 1; @@ -663,3 +715,124 @@ test "a mesh with no eligible node fails loudly instead of silently faking work" defer m.deinit(); try std.testing.expectError(Error.NoEligibleNode, m.dispatch(testJob(0))); } + +// --------------------------------------------------------------------------- +// The coordinator's journal across restarts (gHashTag/trinity-fpga#906, +// gHashTag/t27 specs/trinet/node-work-credit.t27). + +fn journalText(path: [:0]const u8, buf: []u8) []const u8 { + const f = std.c.fopen(path.ptr, "r") orelse return ""; + defer _ = std.c.fclose(f); + return buf[0..std.c.fread(buf.ptr, 1, buf.len, f)]; +} + +fn journaledJob(m: *Mesh, seed: u8) !protocol.Job { + return protocol.Job.withNonce(try m.freshNonce(), @splat(seed), @splat(0x55)); +} + +test "a restarted coordinator never pays the same (node, nonce) twice" { + if (comptime !@import("builtin").link_libc) return error.SkipZigTest; + const j: journal_mod.Journal = .{ .path = "/tmp/trinet-mesh-restart-906.jsonl" }; + _ = std.c.unlink(j.path.ptr); + defer _ = std.c.unlink(j.path.ptr); + + var saved: protocol.Job = undefined; + { + var m = try Mesh.init(std.testing.allocator, .{}); + defer m.deinit(); + try m.join(Node.initEmulated(0x4E4F4431, "peer-1", .honest), "dmitrii-f-t27", 100000); + const r = try m.openJournal(j, "dmitrii-f-t27"); + try std.testing.expectEqual(@as(u32, 0), r.mark); + for (0..3) |i| { + const job = try journaledJob(&m, @intCast(i)); + if (i == 0) saved = job; + const o = try m.dispatch(job); + try std.testing.expectEqual(ledger_mod.Outcome.credited, o.settlement.outcome); + } + try std.testing.expectEqual(@as(u32, 1), saved.nonceValue()); + } + { + // The restart: a new process, the same journal. + var m = try Mesh.init(std.testing.allocator, .{}); + defer m.deinit(); + try m.join(Node.initEmulated(0x4E4F4431, "peer-1", .honest), "dmitrii-f-t27", 100000); + const r = try m.openJournal(j, "dmitrii-f-t27"); + try std.testing.expectEqual(@as(u64, 3), r.credits); + try std.testing.expectEqual(@as(u32, 1024), r.mark); + + // The job saved from the first run earns nothing now: its nonce lies + // below where this run resumed. + const replayed = try m.dispatch(saved); + try std.testing.expectEqual(ledger_mod.Outcome.not_recorded, replayed.settlement.outcome); + try std.testing.expectEqual(@as(u64, 0), replayed.settlement.credit_delta_mtri); + + // New work resumes at the mark, never at 1. + const fresh = try journaledJob(&m, 9); + try std.testing.expectEqual(@as(u32, 1024), fresh.nonceValue()); + const o = try m.dispatch(fresh); + try std.testing.expectEqual(ledger_mod.Outcome.credited, o.settlement.outcome); + } + var buf: [4096]u8 = undefined; + const text = journalText(j.path, &buf); + try std.testing.expectEqual(@as(usize, 4), std.mem.count(u8, text, "\"mtri\":1}")); + try std.testing.expectEqual(@as(usize, 1), std.mem.count(u8, text, "\"nonce\":1,")); + try std.testing.expect(std.mem.indexOf(u8, text, "{\"mark\":2048}") != null); +} + +test "a coordinator run by the node's owner writes self-reported credits" { + if (comptime !@import("builtin").link_libc) return error.SkipZigTest; + const j: journal_mod.Journal = .{ .path = "/tmp/trinet-mesh-level-906.jsonl" }; + for ([_][]const u8{ "dmitrii-f-t27", "", "gHashTag" }) |operator| { + _ = std.c.unlink(j.path.ptr); + var m = try Mesh.init(std.testing.allocator, .{}); + defer m.deinit(); + try m.join(Node.initEmulated(0x4E4F4431, "peer-1", .honest), "dmitrii-f-t27", 100000); + _ = try m.openJournal(j, operator); + _ = try m.dispatch(try journaledJob(&m, 1)); + var buf: [1024]u8 = undefined; + const text = journalText(j.path, &buf); + const want: []const u8 = if (std.mem.eql(u8, operator, "gHashTag")) "\"level\":\"verified\"" else "\"level\":\"self_reported\""; + try std.testing.expect(std.mem.indexOf(u8, text, want) != null); + } + _ = std.c.unlink(j.path.ptr); +} + +test "a forged answer is refused and never reaches the journal" { + if (comptime !@import("builtin").link_libc) return error.SkipZigTest; + const j: journal_mod.Journal = .{ .path = "/tmp/trinet-mesh-forged-906.jsonl" }; + _ = std.c.unlink(j.path.ptr); + defer _ = std.c.unlink(j.path.ptr); + var m = try Mesh.init(std.testing.allocator, .{}); + defer m.deinit(); + try m.join(Node.initEmulated(0x4E4F4432, "lazy", .lazy), "dmitrii-f-t27", 100000); + _ = try m.openJournal(j, "dmitrii-f-t27"); + const o = try m.dispatch(protocol.Job.withNonce(try m.freshNonce(), @splat(0x55), @splat(0x55))); + try std.testing.expect(o.settlement.outcome != .credited); + var buf: [1024]u8 = undefined; + try std.testing.expectEqual(@as(usize, 0), std.mem.count(u8, journalText(j.path, &buf), "\"mtri\"")); +} + +test "an unwritable journal issues no nonce and credits nothing" { + if (comptime !@import("builtin").link_libc) return error.SkipZigTest; + const j: journal_mod.Journal = .{ .path = "/nonexistent-dir-906/journal.jsonl" }; + var m = try Mesh.init(std.testing.allocator, .{}); + defer m.deinit(); + try m.join(Node.initEmulated(0x4E4F4431, "peer-1", .honest), "dmitrii-f-t27", 100000); + _ = try m.openJournal(j, "dmitrii-f-t27"); + try std.testing.expectError(Error.JournalWrite, m.freshNonce()); + + // Even a job whose nonce slipped through earns nothing it cannot record. + m.nonce_mark = 10; + m.next_nonce = 2; + m.resumed_from = 1; + const o = try m.dispatch(protocol.Job.withNonce(1, @splat(0x55), @splat(0x55))); + try std.testing.expectEqual(ledger_mod.Outcome.not_recorded, o.settlement.outcome); + try std.testing.expectEqual(@as(u64, 0), m.ledger.total_credited_mtri); +} + +test "an unreadable journal refuses to open" { + if (comptime !@import("builtin").link_libc) return error.SkipZigTest; + var m = try Mesh.init(std.testing.allocator, .{}); + defer m.deinit(); + try std.testing.expectError(journal_mod.Error.JournalUnreadable, m.openJournal(.{ .path = "/tmp" }, "")); +} From 060ff68ae2147991328581b314431e947052324e Mon Sep 17 00:00:00 2001 From: Dmitrii Fedorov Date: Fri, 9 Oct 2026 21:45:06 -0300 Subject: [PATCH 2/3] fix(trinet): refuse an owner name the journal cannot hold at registration Review on #907: writeCredit refuses an owner or operator that is not a handle (letters, digits, - and _), but register accepted any owner, so a node registered as "developer 1" earned work and every credit became not_recorded. With a journal open, register now returns InvalidOwner, and openJournal refuses an operator or an already-registered owner that is not a handle (InvalidHandle). zig test src/trinet/agent.zig -lc: 68/68; removing any of the three checks fails the new test. Refs #906 --- src/trinet/journal.zig | 7 ++++++- src/trinet/ledger.zig | 7 +++++++ src/trinet/mesh.zig | 27 +++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/src/trinet/journal.zig b/src/trinet/journal.zig index ea018ea5a7..08d9f06bf2 100644 --- a/src/trinet/journal.zig +++ b/src/trinet/journal.zig @@ -36,6 +36,8 @@ pub const Error = error{ JournalMalformed, JournalWrite, NonceSpaceExhausted, + /// An owner or operator name the journal cannot hold as written. + InvalidHandle, OutOfMemory, }; @@ -142,7 +144,10 @@ pub const Journal = struct { } }; -fn isHandle(s: []const u8) bool { +/// A GitHub-style handle: letters, digits, `-` and `_`. Names go into the +/// journal unescaped, so only these pass, and the ledger refuses anything else +/// at registration rather than at payout. +pub fn isHandle(s: []const u8) bool { if (s.len == 0) return false; for (s) |ch| if (!std.ascii.isAlphanumeric(ch) and ch != '-' and ch != '_') return false; return true; diff --git a/src/trinet/ledger.zig b/src/trinet/ledger.zig index d52a78ad12..d5febbc853 100644 --- a/src/trinet/ledger.zig +++ b/src/trinet/ledger.zig @@ -152,6 +152,9 @@ pub const Error = error{ UnknownNode, DuplicateNode, InsufficientStake, + /// With a journal open, an owner name the journal cannot hold. Refused at + /// registration: accepted here it would earn work and never be credited. + InvalidOwner, OutOfMemory, }; @@ -187,6 +190,9 @@ pub const Ledger = struct { /// the nonce mark the coordinator must resume at. An unreadable journal is /// an error, never an empty one. pub fn openJournal(self: *Ledger, j: journal_mod.Journal, operator: []const u8) journal_mod.Error!journal_mod.Restored { + if (operator.len != 0 and !journal_mod.isHandle(operator)) return journal_mod.Error.InvalidHandle; + var it = self.accounts.valueIterator(); + while (it.next()) |a| if (!journal_mod.isHandle(a.owner)) return journal_mod.Error.InvalidHandle; const r = try j.restore(self.gpa, &self.spent_nonces); self.journal = j; self.operator = operator; @@ -197,6 +203,7 @@ pub const Ledger = struct { /// onboarding step: an identity, an owner, and something to lose. pub fn register(self: *Ledger, node_id: u32, owner: []const u8, physical: bool, stake_mtri: u64) Error!void { if (self.accounts.contains(node_id)) return Error.DuplicateNode; + if (self.journal != null and !journal_mod.isHandle(owner)) return Error.InvalidOwner; if (stake_mtri < self.policy.min_stake_mtri) return Error.InsufficientStake; try self.accounts.put(self.gpa, node_id, .{ .node_id = node_id, diff --git a/src/trinet/mesh.zig b/src/trinet/mesh.zig index 2cfd12dbfb..cf7d037fa2 100644 --- a/src/trinet/mesh.zig +++ b/src/trinet/mesh.zig @@ -34,6 +34,7 @@ pub const Error = error{ UnknownNode, DuplicateNode, InsufficientStake, + InvalidOwner, UnsoundPolicy, /// The journal could not take the next nonce mark, so no nonce is issued. JournalWrite, @@ -836,3 +837,29 @@ test "an unreadable journal refuses to open" { defer m.deinit(); try std.testing.expectError(journal_mod.Error.JournalUnreadable, m.openJournal(.{ .path = "/tmp" }, "")); } + +test "an owner name the journal cannot hold is refused at the door, not at payout" { + if (comptime !@import("builtin").link_libc) return error.SkipZigTest; + const j: journal_mod.Journal = .{ .path = "/tmp/trinet-mesh-handle-906.jsonl" }; + _ = std.c.unlink(j.path.ptr); + defer _ = std.c.unlink(j.path.ptr); + { + var m = try Mesh.init(std.testing.allocator, .{}); + defer m.deinit(); + _ = try m.openJournal(j, ""); + try std.testing.expectError(Error.InvalidOwner, m.join(Node.initEmulated(0x4E4F4431, "peer-1", .honest), "developer 1", 100000)); + try m.join(Node.initEmulated(0x4E4F4432, "peer-2", .honest), "developer-2", 100000); + } + { + // Joined before the journal opened: the journal refuses to open. + var m = try Mesh.init(std.testing.allocator, .{}); + defer m.deinit(); + try m.join(Node.initEmulated(0x4E4F4431, "peer-1", .honest), "a\"b", 100000); + try std.testing.expectError(journal_mod.Error.InvalidHandle, m.openJournal(j, "")); + } + { + var m = try Mesh.init(std.testing.allocator, .{}); + defer m.deinit(); + try std.testing.expectError(journal_mod.Error.InvalidHandle, m.openJournal(j, "op erator")); + } +} From e884a989e230fe0d675929fb15205c00cbb308bb Mon Sep 17 00:00:00 2001 From: dmitrii-f-t27 Date: Sat, 10 Oct 2026 21:28:45 -0300 Subject: [PATCH 3/3] fix(trinet): refuse a torn journal for writing, and make room in the paid set before the credit is written Two accounting defects in the journal from #907, both of which break the spec's "one credit per (node, nonce) across restarts" (gHashTag/t27 specs/trinet/node-work-credit.t27): - Torn tail. A run that died mid-line left an unfinished last line, and the next run opened the journal and appended with fopen("a"), gluing its first mark onto the tear. Depending on where the tear was, the glued line either stops parsing (the journal becomes unreadable, crediting stops for good) or parses as a credit and swallows the mark (the next start resumes at an older mark and reissues nonces). `Ledger.openJournal` now returns JournalTornTail before any nonce or credit and leaves the file untouched; read-only `restore` still reports `torn_tail` for inspection. No automatic truncation. - Write before credit. The paid set grew after the credit line was fsynced, so an allocation failure left a durable credit this run never marked paid, and a second settle of the same (node, nonce) could write a second line. The room is now reserved before the write; the insert after it cannot fail. The spent key comes from journal.spentKey, the same function restore uses. Tests: zig test src/trinet/agent.zig -lc, 75/75 (Zig 0.16.0, macOS arm64). New: torn tail refused twice and left byte-identical, glued-line corruption reproduced, malformed line stops the coordinator, unverifiable and tampered receipts never reach the journal, a refused credit is neither paid nor marked paid, no credit reaches disk without room in the paid set, every credit line names its own node's owner. Refs #906 --- src/trinet/journal.zig | 73 ++++++++++++++++++++++ src/trinet/ledger.zig | 135 ++++++++++++++++++++++++++++++++++++++++- src/trinet/main.zig | 14 ++++- src/trinet/mesh.zig | 106 ++++++++++++++++++++++++++++++++ 4 files changed, 324 insertions(+), 4 deletions(-) diff --git a/src/trinet/journal.zig b/src/trinet/journal.zig index 08d9f06bf2..af63e1975f 100644 --- a/src/trinet/journal.zig +++ b/src/trinet/journal.zig @@ -21,6 +21,20 @@ //! a credit counts only after its whole line is on disk and a nonce is issued //! only after its mark is. //! +//! Skipped for READING is not the same as fit to WRITE. `append` opens with +//! fopen("a"), so the next line a run writes is glued onto the torn one, and +//! the result is worse than the tear: depending on where the process died the +//! glued line either stops parsing (so the journal becomes unreadable and +//! crediting stops for good) or parses as a credit and SWALLOWS the mark that +//! was glued into it (so the next start resumes at an older mark and reissues +//! nonces the previous run may already have handed out). A writing run +//! therefore refuses a torn journal at open -- `Ledger.openJournal` returns +//! `JournalTornTail` before any nonce or credit -- and leaves the file exactly +//! as it found it. Repairing it by truncation is NOT done here: another writer +//! may hold the same path, and an unproven repair of a payment record is worse +//! than a stop. `parse`/`restore` keep reporting `torn_tail` so a person or a +//! read-only tool can still inspect the file. +//! //! The file is JSON lines so a person can read it and `jq` can sum it. It //! needs libc for fsync; without libc a journal cannot be opened. @@ -34,6 +48,10 @@ pub const Error = error{ JournalUnavailable, JournalUnreadable, JournalMalformed, + /// The last line is incomplete: a run died mid-write. Readable, but not + /// writable -- see the note at the top of this file. The run stops with + /// the file untouched instead of appending onto the tear. + JournalTornTail, JournalWrite, NonceSpaceExhausted, /// An owner or operator name the journal cannot hold as written. @@ -68,6 +86,9 @@ pub const Restored = struct { /// The highest mark on disk; 0 for a fresh journal. mark: u32 = 0, credits: u64 = 0, + /// The file's last line has no newline, so it was never finished. Reading + /// skips it; a run that intends to WRITE must stop (`JournalTornTail`), + /// because appending glues the next line onto this one. torn_tail: bool = false, }; @@ -202,6 +223,14 @@ fn numberAfter(line: []const u8, key: []const u8) ?[]const u8 { const testing = std.testing; +/// Lay down a journal by hand, to read back a state a running coordinator +/// cannot produce on purpose. +fn writeText(path: [:0]const u8, text: []const u8) !void { + const f = std.c.fopen(path.ptr, "w") orelse return error.JournalWrite; + const n = std.c.fwrite(text.ptr, 1, text.len, f); + if (std.c.fclose(f) != 0 or n != text.len) return error.JournalWrite; +} + test "the nonce mark moves in blocks and never wraps" { try testing.expectEqual(@as(u32, 1024), reserve_block); try testing.expectEqual(@as(u32, 1025), nextMark(1)); @@ -258,6 +287,50 @@ test "a missing journal is fresh, an unreadable one is an error" { try testing.expectError(Error.JournalUnreadable, dir.restore(testing.allocator, &spent)); } +test "writing on top of a torn tail is what corrupts a journal" { + if (comptime !builtin.link_libc) return error.SkipZigTest; + // This is why a torn tail refuses to open for writing instead of being + // appended to: `append` uses fopen("a"), so the next line the coordinator + // writes is glued onto the unfinished one. Both halves below write exactly + // the mark a resuming run would have written next, and neither result is + // survivable. + const head = "{\"mark\":1024}\n"; + + // Torn before the nonce: the glued line stops parsing, so from here on the + // journal is unreadable and crediting stops for good. + { + const j: Journal = .{ .path = "/tmp/trinet-journal-test-glue-early-906.jsonl" }; + _ = std.c.unlink(j.path.ptr); + defer _ = std.c.unlink(j.path.ptr); + try writeText(j.path, head ++ "{\"t\":2,\"operator\":\"\",\"owner\":\"a\",\"node\":\"4e4f4431\",\"physi"); + try j.writeMark(2048); + + var spent: std.AutoHashMapUnmanaged(u64, void) = .empty; + defer spent.deinit(testing.allocator); + try testing.expectError(Error.JournalMalformed, j.restore(testing.allocator, &spent)); + } + + // Torn a few fields later: worse, because it is silent. The glued line + // parses as a paid credit and swallows the mark written into it, so the + // next start resumes at the OLD mark and reissues nonces the previous run + // may already have handed out -- the exact double payment this file exists + // to prevent. + { + const j: Journal = .{ .path = "/tmp/trinet-journal-test-glue-late-906.jsonl" }; + _ = std.c.unlink(j.path.ptr); + defer _ = std.c.unlink(j.path.ptr); + try writeText(j.path, head ++ "{\"t\":2,\"operator\":\"\",\"owner\":\"a\",\"node\":\"4e4f4432\",\"physical\":false,\"nonce\":5,\"y\":3,\"tag\":\"00"); + try j.writeMark(2048); + + var spent: std.AutoHashMapUnmanaged(u64, void) = .empty; + defer spent.deinit(testing.allocator); + const r = try j.restore(testing.allocator, &spent); + try testing.expectEqual(@as(u32, 1024), r.mark); // the 2048 just written is gone + try testing.expect(spent.contains(spentKey(0x4E4F4432, 5))); + try testing.expect(!r.torn_tail); // and it no longer looks damaged + } +} + test "a mark and a credit written are read back" { if (comptime !builtin.link_libc) return error.SkipZigTest; const j: Journal = .{ .path = "/tmp/trinet-journal-test-roundtrip-906.jsonl" }; diff --git a/src/trinet/ledger.zig b/src/trinet/ledger.zig index d5febbc853..94acacc76a 100644 --- a/src/trinet/ledger.zig +++ b/src/trinet/ledger.zig @@ -189,11 +189,19 @@ pub const Ledger = struct { /// Open the coordinator's journal: rebuild the paid set from it and return /// the nonce mark the coordinator must resume at. An unreadable journal is /// an error, never an empty one. + /// + /// This is the spec's `may_start_crediting` (gHashTag/t27 + /// specs/trinet/node-work-credit.t27): nothing is credited and no nonce is + /// issued unless the journal and its mark were read. A torn last line is + /// readable but not writable -- appending would glue the next line onto it + /// -- so it is refused here, before any nonce or credit, and the file is + /// left exactly as it was for an operator to look at. pub fn openJournal(self: *Ledger, j: journal_mod.Journal, operator: []const u8) journal_mod.Error!journal_mod.Restored { if (operator.len != 0 and !journal_mod.isHandle(operator)) return journal_mod.Error.InvalidHandle; var it = self.accounts.valueIterator(); while (it.next()) |a| if (!journal_mod.isHandle(a.owner)) return journal_mod.Error.InvalidHandle; const r = try j.restore(self.gpa, &self.spent_nonces); + if (r.torn_tail) return journal_mod.Error.JournalTornTail; self.journal = j; self.operator = operator; return r; @@ -321,7 +329,11 @@ pub const Ledger = struct { // Pay once per nonce, per node. Without this a node can resubmit the // same verified receipt forever. - const key = (@as(u64, dispatched_to) << 32) | job.nonceValue(); + // + // The key comes from journal.zig, which is also what `restore` rebuilds + // the set with. Spelling the same shift out twice is how a restart + // quietly stops recognising what it already paid. + const key = journal_mod.spentKey(dispatched_to, job.nonceValue()); if (self.spent_nonces.contains(key)) { return .{ .node_id = dispatched_to, @@ -331,6 +343,15 @@ pub const Ledger = struct { } const reward = self.policy.reward_per_job_mtri; + // Room for the paid key BEFORE the line goes to disk. The other order -- + // fsync the credit, then allocate -- can fail after the credit is + // durable, leaving a line on disk that this run never marked as paid; a + // later settle of the same (node, nonce) would then write a second line + // for it, which is the one thing the journal exists to prevent. Asking + // for the room first makes the failure happen before anything is + // written, and `putAssumeCapacity` below cannot fail. + try self.spent_nonces.ensureUnusedCapacity(self.gpa, 1); + // Write before credit: with a journal, a credit counts only once its // line is on disk, so a crash can lose a credit but never repeat one. if (self.journal) |j| { @@ -351,7 +372,7 @@ pub const Ledger = struct { .detail = "journal write failed: not credited", }; } - try self.spent_nonces.put(self.gpa, key, {}); + self.spent_nonces.putAssumeCapacity(key, {}); acct.credit_mtri += reward; acct.accepted += 1; @@ -615,6 +636,116 @@ test "a verifier holding no key charges nobody" { try std.testing.expectEqual(Status.active, l.get(0xB0A2D).?.status); } +test "a receipt we cannot check is recorded nowhere, not even as verified" { + // `verified` is a statement about who vouched, not about the arithmetic: + // a coordinator run by somebody other than the node's owner writes that + // level on every credit it records. So the question this test asks is + // whether a receipt the coordinator could not judge can reach the journal + // at all under exactly those conditions -- because if it could, the record + // would read "verified" over a receipt nobody verified. + if (comptime !@import("builtin").link_libc) return error.SkipZigTest; + const j: journal_mod.Journal = .{ .path = "/tmp/trinet-ledger-unverifiable-906.jsonl" }; + _ = std.c.unlink(j.path.ptr); + defer _ = std.c.unlink(j.path.ptr); + + var l = try Ledger.init(std.testing.allocator, .{}); + defer l.deinit(); + try l.register(0xB0A2D, "dmitrii-f-t27", true, 100_000); + _ = try l.openJournal(j, "gHashTag"); + try std.testing.expectEqual(journal_mod.Level.verified, journal_mod.Level.of("gHashTag", "dmitrii-f-t27")); + + // A keyed receipt and no key to check it with. + const job = makeJob(1); + var key: [16]u8 = undefined; + for (&key, 0..) |*b, i| b.* = @intCast(0x33 +% i); + const y = protocol.dot(job.w, job.x); + const keyed: protocol.Receipt = .{ + .kind = .siphash24, + .y = y, + .status = protocol.status_ok, + .nonce = job.nonce, + .node_id = 0xB0A2D, + .tag = protocol.receiptTagKeyed(job, y, 0xB0A2D, key), + }; + const verdict = protocol.verifyWithKey(job, keyed, null); + try std.testing.expectEqual(protocol.Verdict.unverifiable, verdict); + const s1 = try l.settle(0xB0A2D, job, keyed, verdict); + try std.testing.expectEqual(Outcome.unverifiable_not_charged, s1.outcome); + + // And a receipt whose tag was altered on the way back. + const job2 = makeJob(2); + var tampered = protocol.execute(job2, 0xB0A2D); + tampered.tag ^= 0x40; + const s2 = try l.settle(0xB0A2D, job2, tampered, protocol.verify(job2, tampered)); + try std.testing.expectEqual(Outcome.corrupt_not_charged, s2.outcome); + + // Neither of them is on disk, at any level. + var spent: std.AutoHashMapUnmanaged(u64, void) = .empty; + defer spent.deinit(std.testing.allocator); + const restored = try j.restore(std.testing.allocator, &spent); + try std.testing.expectEqual(@as(u64, 0), restored.credits); + try std.testing.expectEqual(@as(u64, 0), l.total_credited_mtri); +} + +test "a credit the journal refused is neither paid nor marked as paid" { + if (comptime !@import("builtin").link_libc) return error.SkipZigTest; + var l = try Ledger.init(std.testing.allocator, .{}); + defer l.deinit(); + try l.register(0x5005, "dmitrii-f-t27", true, 5000); + // Opening this is fine -- a journal that is not there yet is a fresh + // network. Writing to it can never work. + _ = try l.openJournal(.{ .path = "/nonexistent-dir-906/journal.jsonl" }, ""); + + const job = makeJob(5); + const r = protocol.execute(job, 0x5005); + const s = try l.settle(0x5005, job, r, protocol.verify(job, r)); + + try std.testing.expectEqual(Outcome.not_recorded, s.outcome); + try std.testing.expectEqual(@as(u64, 0), s.credit_delta_mtri); + // Not a punishment: a journal we cannot write is our failure, not the + // node's. + try std.testing.expectEqual(@as(u64, 0), s.slash_delta_mtri); + try std.testing.expectEqual(@as(u64, 0), l.get(0x5005).?.credit_mtri); + try std.testing.expectEqual(@as(u64, 5000), l.get(0x5005).?.stake_mtri); + try std.testing.expectEqual(@as(u64, 0), l.get(0x5005).?.rejected); + try std.testing.expectEqual(@as(u64, 0), l.total_credited_mtri); + // And not marked paid, so the same work is still payable once the journal + // works again. The line on disk, not this set, is what stops a second + // payment. + try std.testing.expect(!l.spent_nonces.contains(journal_mod.spentKey(0x5005, job.nonceValue()))); +} + +test "no credit reaches the disk that the paid set has no room to record" { + // The write-then-record order has one more failure between its two halves: + // the paid set has to grow. If it grew after the fsync, an allocation + // failure would leave a durable credit this run never marked paid, and the + // next settle of the same (node, nonce) would append a second line for it. + if (comptime !@import("builtin").link_libc) return error.SkipZigTest; + const j: journal_mod.Journal = .{ .path = "/tmp/trinet-ledger-noroom-906.jsonl" }; + _ = std.c.unlink(j.path.ptr); + defer _ = std.c.unlink(j.path.ptr); + + var l = try Ledger.init(std.testing.allocator, .{}); + defer l.deinit(); + try l.register(0x7007, "dmitrii-f-t27", true, 5000); + _ = try l.openJournal(j, ""); + + const job = makeJob(9); + const r = protocol.execute(job, 0x7007); + l.gpa = std.testing.failing_allocator; + try std.testing.expectError(Error.OutOfMemory, l.settle(0x7007, job, r, protocol.verify(job, r))); + // The accounts map was allocated by the real allocator and must be freed by + // it; the paid set never grew, so it owns nothing. + l.gpa = std.testing.allocator; + + var spent: std.AutoHashMapUnmanaged(u64, void) = .empty; + defer spent.deinit(std.testing.allocator); + const restored = try j.restore(std.testing.allocator, &spent); + try std.testing.expectEqual(@as(u64, 0), restored.credits); + try std.testing.expectEqual(@as(u64, 0), l.get(0x7007).?.credit_mtri); + try std.testing.expectEqual(@as(u64, 0), l.total_credited_mtri); +} + test "work we can never pay for eventually stops being dispatched" { // Not punishment — no stake moves — but there is no point sending jobs to a // node whose answers can never be credited. diff --git a/src/trinet/main.zig b/src/trinet/main.zig index a703c74276..cb35c58d30 100644 --- a/src/trinet/main.zig +++ b/src/trinet/main.zig @@ -313,13 +313,23 @@ fn openJournalFromEnv(m: *mesh_mod.Mesh) !void { const operator = envOr(operator_env, ""); const r = m.openJournal(.{ .path = path }, operator) catch |e| { std.debug.print("journal {s}: {s} -- not crediting without it\n", .{ path, @errorName(e) }); + switch (e) { + // Say what it is, because the file is fine to READ and the fix is + // an operator's decision, not a retry. + error.JournalTornTail => std.debug.print("The last line was never finished: a previous run died mid-write.\n" ++ + "Appending to it would glue the next line onto the tear, so this run\n" ++ + "stops and the file is left exactly as it is. Stop here and keep the\n" ++ + "file unchanged: it holds the paid set and the nonce reserve mark.\n" ++ + "Recover it only under control, after checking its complete credit\n" ++ + "lines and its last complete mark line.\n", .{}), + else => {}, + } return e; }; - std.debug.print("journal {s}: {d} paid jobs restored, nonces resume at {d}{s}, credits {s}\n", .{ + std.debug.print("journal {s}: {d} paid jobs restored, nonces resume at {d}, credits {s}\n", .{ path, r.credits, m.next_nonce, - if (r.torn_tail) " (a torn last line was skipped)" else "", if (operator.len == 0) "self-reported (TRINET_OPERATOR unset)" else "by operator", }); } diff --git a/src/trinet/mesh.zig b/src/trinet/mesh.zig index cf7d037fa2..1ae1129bd7 100644 --- a/src/trinet/mesh.zig +++ b/src/trinet/mesh.zig @@ -731,6 +731,14 @@ fn journaledJob(m: *Mesh, seed: u8) !protocol.Job { return protocol.Job.withNonce(try m.freshNonce(), @splat(seed), @splat(0x55)); } +/// Lay down a journal by hand, to restart on top of one a running coordinator +/// cannot produce on purpose: a torn tail, or a line that does not parse. +fn writeJournalText(path: [:0]const u8, text: []const u8) !void { + const f = std.c.fopen(path.ptr, "w") orelse return error.JournalWrite; + const written = std.c.fwrite(text.ptr, 1, text.len, f); + if (std.c.fclose(f) != 0 or written != text.len) return error.JournalWrite; +} + test "a restarted coordinator never pays the same (node, nonce) twice" { if (comptime !@import("builtin").link_libc) return error.SkipZigTest; const j: journal_mod.Journal = .{ .path = "/tmp/trinet-mesh-restart-906.jsonl" }; @@ -813,6 +821,104 @@ test "a forged answer is refused and never reaches the journal" { try std.testing.expectEqual(@as(usize, 0), std.mem.count(u8, journalText(j.path, &buf), "\"mtri\"")); } +test "every credit line names the owner its own node was registered to" { + // The handle in the line is what a payout is read off, so it has to be the + // node's owner -- not the coordinator's operator, and not whichever owner + // happened to be settled first. + if (comptime !@import("builtin").link_libc) return error.SkipZigTest; + const j: journal_mod.Journal = .{ .path = "/tmp/trinet-mesh-owner-906.jsonl" }; + _ = std.c.unlink(j.path.ptr); + defer _ = std.c.unlink(j.path.ptr); + + var m = try Mesh.init(std.testing.allocator, .{}); + defer m.deinit(); + try m.join(Node.initEmulated(0x4E4F4431, "peer-1", .honest), "developer-1", 100000); + try m.join(Node.initEmulated(0x4E4F4432, "peer-2", .honest), "developer-2", 100000); + _ = try m.openJournal(j, "gHashTag"); + + // Round-robin: one job each. + for (0..2) |_| { + const o = try m.dispatch(try journaledJob(&m, 1)); + try std.testing.expectEqual(ledger_mod.Outcome.credited, o.settlement.outcome); + } + + var buf: [4096]u8 = undefined; + const text = journalText(j.path, &buf); + try std.testing.expectEqual(@as(usize, 1), std.mem.count(u8, text, "\"owner\":\"developer-1\"")); + try std.testing.expectEqual(@as(usize, 1), std.mem.count(u8, text, "\"owner\":\"developer-2\"")); + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, text, "\"operator\":\"gHashTag\"")); + // The operator is not an owner of anything here. + try std.testing.expectEqual(@as(usize, 0), std.mem.count(u8, text, "\"owner\":\"gHashTag\"")); +} + +test "a journal torn mid-line refuses to open, twice over, and is left untouched" { + // The regression this exists for: the coordinator used to open a torn + // journal successfully and carry on writing. `append` uses fopen("a"), so + // the first mark it wrote was glued onto the unfinished line -- see + // journal.zig "writing on top of a torn tail is what corrupts a journal" + // for what that does. One start cannot see it; the SECOND start is where + // the damage surfaces, so this test performs both. + if (comptime !@import("builtin").link_libc) return error.SkipZigTest; + const j: journal_mod.Journal = .{ .path = "/tmp/trinet-mesh-torn-906.jsonl" }; + defer _ = std.c.unlink(j.path.ptr); + // A mark, one whole credit, and a line the process died inside. + const torn = "{\"mark\":1024}\n" ++ + "{\"t\":1,\"operator\":\"\",\"owner\":\"dmitrii-f-t27\",\"node\":\"4e4f4431\",\"physical\":false,\"nonce\":1,\"y\":3,\"tag\":\"0000000000000000\",\"tag_kind\":\"crc32\",\"level\":\"self_reported\",\"mtri\":1}\n" ++ + "{\"t\":2,\"operator\":\"\",\"owner\":\"dmitrii-f-t27\",\"node\":\"4e4f4431\",\"physi"; + try writeJournalText(j.path, torn); + + // First start after the crash: refused before any nonce or credit. + { + var m = try Mesh.init(std.testing.allocator, .{}); + defer m.deinit(); + try m.join(Node.initEmulated(0x4E4F4431, "peer-1", .honest), "dmitrii-f-t27", 100000); + try std.testing.expectError(journal_mod.Error.JournalTornTail, m.openJournal(j, "")); + try std.testing.expect(m.ledger.journal == null); + try std.testing.expectEqual(@as(u32, 0), m.nonce_mark); + try std.testing.expectEqual(@as(u64, 0), m.ledger.total_credited_mtri); + } + + var buf: [4096]u8 = undefined; + try std.testing.expectEqualStrings(torn, journalText(j.path, &buf)); + + // Second start: the same named error, not a journal that has since become + // unparsable, and still not a byte written. + { + var m = try Mesh.init(std.testing.allocator, .{}); + defer m.deinit(); + try m.join(Node.initEmulated(0x4E4F4431, "peer-1", .honest), "dmitrii-f-t27", 100000); + try std.testing.expectError(journal_mod.Error.JournalTornTail, m.openJournal(j, "")); + } + var buf2: [4096]u8 = undefined; + try std.testing.expectEqualStrings(torn, journalText(j.path, &buf2)); + + // Read-only inspection still describes the file rather than refusing it: + // one whole credit, the mark, and a tail somebody has to decide about. + var spent: std.AutoHashMapUnmanaged(u64, void) = .empty; + defer spent.deinit(std.testing.allocator); + const seen = try j.restore(std.testing.allocator, &spent); + try std.testing.expect(seen.torn_tail); + try std.testing.expectEqual(@as(u64, 1), seen.credits); + try std.testing.expectEqual(@as(u32, 1024), seen.mark); +} + +test "a journal line that does not parse stops the coordinator, it does not read as empty" { + if (comptime !@import("builtin").link_libc) return error.SkipZigTest; + const j: journal_mod.Journal = .{ .path = "/tmp/trinet-mesh-malformed-906.jsonl" }; + defer _ = std.c.unlink(j.path.ptr); + // A complete line with no node field: we cannot tell what it paid. + try writeJournalText(j.path, "{\"mark\":1024}\n{\"t\":1,\"owner\":\"dmitrii-f-t27\",\"nonce\":1,\"mtri\":1}\n"); + + var m = try Mesh.init(std.testing.allocator, .{}); + defer m.deinit(); + try m.join(Node.initEmulated(0x4E4F4431, "peer-1", .honest), "dmitrii-f-t27", 100000); + try std.testing.expectError(journal_mod.Error.JournalMalformed, m.openJournal(j, "")); + // The journal was not adopted, so nothing here can credit against it. The + // runner turns this error into a refusal to start (main.zig). + try std.testing.expect(m.ledger.journal == null); + try std.testing.expectEqual(@as(u64, 0), m.ledger.total_credited_mtri); +} + test "an unwritable journal issues no nonce and credits nothing" { if (comptime !@import("builtin").link_libc) return error.SkipZigTest; const j: journal_mod.Journal = .{ .path = "/nonexistent-dir-906/journal.jsonl" };