diff --git a/.github/workflows/conformance-frame-alignment.yml b/.github/workflows/conformance-frame-alignment.yml index dfedc82ea..f4c5d9152 100644 --- a/.github/workflows/conformance-frame-alignment.yml +++ b/.github/workflows/conformance-frame-alignment.yml @@ -11,11 +11,14 @@ on: - 'conformance/frame_alignment_check.py' - 'conformance/gfternary_compute_conformance_ax7203.py' - 'conformance/trinet_mac32_conformance_ax7203.py' + # the TRI-NET stack the trinet-tests job runs: a change to it must run them + - 'src/trinet/**' pull_request: paths: - 'conformance/**' - 'fpga/vivado/**' - 'fpga/openxc7-synth/**' + - 'src/trinet/**' schedule: # Daily, so a wrapper edit that silently orphans a host surfaces within a # day rather than at the next flash session. diff --git a/src/trinet/journal.zig b/src/trinet/journal.zig new file mode 100644 index 000000000..af63e1975 --- /dev/null +++ b/src/trinet/journal.zig @@ -0,0 +1,349 @@ +//! The coordinator's durable journal: what it has paid, and how far it has +//! handed out nonces, so that neither is forgotten when the process stops. +//! +//! Spec: gHashTag/t27 specs/trinet/node-work-credit.t27 (gHashTag/t27#8426). +//! Issue: gHashTag/trinity-fpga#906. +//! +//! Before this file the paid (node, nonce) set lived in Ledger memory and the +//! coordinator's next nonce started at 1 on every start, so a restart paid the +//! same (node, nonce) again. The journal fixes both: +//! +//! - a mark line `{"mark":N}` is written BEFORE any nonce at or above the +//! previous mark is issued, so after a restart every nonce that may have +//! gone out lies below the last mark and the coordinator resumes at it; +//! - a credit line is written and flushed to disk BEFORE the credit counts, +//! and the paid set is rebuilt from those lines on start. +//! +//! Reading fails closed. A missing file is a fresh network; a file that exists +//! but cannot be read, or holds a complete line we cannot parse, is an error: +//! "we do not know what was paid" must never read as "nothing was paid". Only a +//! torn LAST line (no newline: the process died mid-write) is skipped, because +//! a credit counts only after its whole line is on disk and a nonce is issued +//! only after its mark is. +//! +//! Skipped for READING is not the same as fit to WRITE. `append` opens with +//! fopen("a"), so the next line a run writes is glued onto the torn one, and +//! the result is worse than the tear: depending on where the process died the +//! glued line either stops parsing (so the journal becomes unreadable and +//! crediting stops for good) or parses as a credit and SWALLOWS the mark that +//! was glued into it (so the next start resumes at an older mark and reissues +//! nonces the previous run may already have handed out). A writing run +//! therefore refuses a torn journal at open -- `Ledger.openJournal` returns +//! `JournalTornTail` before any nonce or credit -- and leaves the file exactly +//! as it found it. Repairing it by truncation is NOT done here: another writer +//! may hold the same path, and an unproven repair of a payment record is worse +//! than a stop. `parse`/`restore` keep reporting `torn_tail` so a person or a +//! read-only tool can still inspect the file. +//! +//! The file is JSON lines so a person can read it and `jq` can sum it. It +//! needs libc for fsync; without libc a journal cannot be opened. + +const std = @import("std"); +const builtin = @import("builtin"); + +/// NONCE_RESERVE_BLOCK in node-work-credit.t27. +pub const reserve_block: u32 = 1024; + +pub const Error = error{ + JournalUnavailable, + JournalUnreadable, + JournalMalformed, + /// The last line is incomplete: a run died mid-write. Readable, but not + /// writable -- see the note at the top of this file. The run stops with + /// the file untouched instead of appending onto the tear. + JournalTornTail, + JournalWrite, + NonceSpaceExhausted, + /// An owner or operator name the journal cannot hold as written. + InvalidHandle, + OutOfMemory, +}; + +/// Who vouches for a credit (record levels in node-work-credit.t27). A credit +/// written by a coordinator whose operator is the node's owner, or whose +/// operator is unknown, is the owner's own word about their node. +pub const Level = enum { + self_reported, + verified, + + pub fn of(operator: []const u8, owner: []const u8) Level { + if (operator.len == 0 or std.mem.eql(u8, operator, owner)) return .self_reported; + return .verified; + } +}; + +/// The spec's next_mark: 0 when the 32-bit nonce space is spent, never a wrap. +pub fn nextMark(mark: u32) u32 { + if (mark > std.math.maxInt(u32) - reserve_block) return 0; + return mark + reserve_block; +} + +pub fn spentKey(node_id: u32, nonce: u32) u64 { + return (@as(u64, node_id) << 32) | nonce; +} + +pub const Restored = struct { + /// The highest mark on disk; 0 for a fresh journal. + mark: u32 = 0, + credits: u64 = 0, + /// The file's last line has no newline, so it was never finished. Reading + /// skips it; a run that intends to WRITE must stop (`JournalTornTail`), + /// because appending glues the next line onto this one. + torn_tail: bool = false, +}; + +const C = struct { + extern "c" fn fsync(fd: c_int) c_int; + extern "c" fn fileno(f: *std.c.FILE) c_int; + extern "c" fn access(path: [*:0]const u8, mode: c_int) c_int; + extern "c" fn fflush(f: *std.c.FILE) c_int; + extern "c" fn ferror(f: *std.c.FILE) c_int; +}; + +pub const Journal = struct { + path: [:0]const u8, + + /// Append one line and put it on disk before returning. + fn append(self: Journal, line: []const u8) Error!void { + if (comptime !builtin.link_libc) return Error.JournalUnavailable; + const f = std.c.fopen(self.path.ptr, "a") orelse return Error.JournalWrite; + var ok = std.c.fwrite(line.ptr, 1, line.len, f) == line.len; + ok = ok and C.fflush(f) == 0; + ok = ok and C.fsync(C.fileno(f)) == 0; + ok = std.c.fclose(f) == 0 and ok; + if (!ok) return Error.JournalWrite; + } + + pub fn writeMark(self: Journal, mark: u32) Error!void { + var buf: [32]u8 = undefined; + const line = std.fmt.bufPrint(&buf, "{{\"mark\":{d}}}\n", .{mark}) catch unreachable; + return self.append(line); + } + + pub const Credit = struct { + operator: []const u8, + owner: []const u8, + node_id: u32, + physical: bool, + nonce: u32, + y: i32, + tag: u64, + tag_kind: []const u8, + level: Level, + mtri: u64, + }; + + pub fn writeCredit(self: Journal, c: Credit) Error!void { + // Handles go into the line unescaped, so only handle characters pass. + if (!isHandle(c.owner) or !(c.operator.len == 0 or isHandle(c.operator))) return Error.JournalWrite; + var ts: std.c.timespec = undefined; + _ = std.c.clock_gettime(.REALTIME, &ts); + var buf: [512]u8 = undefined; + const line = std.fmt.bufPrint(&buf, "{{\"t\":{d},\"operator\":\"{s}\",\"owner\":\"{s}\",\"node\":\"{x:0>8}\",\"physical\":{},\"nonce\":{d},\"y\":{d},\"tag\":\"{x:0>16}\",\"tag_kind\":\"{s}\",\"level\":\"{s}\",\"mtri\":{d}}}\n", .{ + ts.sec, c.operator, c.owner, c.node_id, c.physical, c.nonce, c.y, c.tag, c.tag_kind, @tagName(c.level), c.mtri, + }) catch return Error.JournalWrite; + return self.append(line); + } + + /// Rebuild the paid set and the nonce mark from disk. + pub fn restore(self: Journal, gpa: std.mem.Allocator, spent: *std.AutoHashMapUnmanaged(u64, void)) Error!Restored { + if (comptime !builtin.link_libc) return Error.JournalUnavailable; + if (C.access(self.path.ptr, 0) != 0) return .{}; + const f = std.c.fopen(self.path.ptr, "r") orelse return Error.JournalUnreadable; + defer _ = std.c.fclose(f); + + var text: std.ArrayList(u8) = .empty; + defer text.deinit(gpa); + var chunk: [4096]u8 = undefined; + while (true) { + const n = std.c.fread(&chunk, 1, chunk.len, f); + if (n == 0) break; + try text.appendSlice(gpa, chunk[0..n]); + } + if (C.ferror(f) != 0) return Error.JournalUnreadable; + return parse(gpa, text.items, spent); + } +}; + +/// A GitHub-style handle: letters, digits, `-` and `_`. Names go into the +/// journal unescaped, so only these pass, and the ledger refuses anything else +/// at registration rather than at payout. +pub fn isHandle(s: []const u8) bool { + if (s.len == 0) return false; + for (s) |ch| if (!std.ascii.isAlphanumeric(ch) and ch != '-' and ch != '_') return false; + return true; +} + +/// The text of a journal into the paid set and the mark. Public for tests. +pub fn parse(gpa: std.mem.Allocator, text: []const u8, spent: *std.AutoHashMapUnmanaged(u64, void)) Error!Restored { + var out: Restored = .{}; + var rest = text; + while (rest.len > 0) { + const nl = std.mem.indexOfScalar(u8, rest, '\n') orelse { + out.torn_tail = true; + break; + }; + const line = rest[0..nl]; + rest = rest[nl + 1 ..]; + if (line.len == 0) continue; + + if (std.mem.startsWith(u8, line, "{\"mark\":")) { + const v = numberAfter(line, "{\"mark\":") orelse return Error.JournalMalformed; + const m = std.fmt.parseInt(u32, v, 10) catch return Error.JournalMalformed; + if (m > out.mark) out.mark = m; + continue; + } + const node_hex = stringAfter(line, "\"node\":\"") orelse return Error.JournalMalformed; + const node_id = std.fmt.parseInt(u32, node_hex, 16) catch return Error.JournalMalformed; + const nonce_s = numberAfter(line, "\"nonce\":") orelse return Error.JournalMalformed; + const nonce = std.fmt.parseInt(u32, nonce_s, 10) catch return Error.JournalMalformed; + try spent.put(gpa, spentKey(node_id, nonce), {}); + out.credits += 1; + } + return out; +} + +fn stringAfter(line: []const u8, key: []const u8) ?[]const u8 { + const at = std.mem.indexOf(u8, line, key) orelse return null; + const from = at + key.len; + const end = std.mem.indexOfScalarPos(u8, line, from, '"') orelse return null; + return line[from..end]; +} + +fn numberAfter(line: []const u8, key: []const u8) ?[]const u8 { + const at = std.mem.indexOf(u8, line, key) orelse return null; + const from = at + key.len; + var end = from; + while (end < line.len and std.ascii.isDigit(line[end])) end += 1; + if (end == from) return null; + return line[from..end]; +} + +// --------------------------------------------------------------------------- + +const testing = std.testing; + +/// Lay down a journal by hand, to read back a state a running coordinator +/// cannot produce on purpose. +fn writeText(path: [:0]const u8, text: []const u8) !void { + const f = std.c.fopen(path.ptr, "w") orelse return error.JournalWrite; + const n = std.c.fwrite(text.ptr, 1, text.len, f); + if (std.c.fclose(f) != 0 or n != text.len) return error.JournalWrite; +} + +test "the nonce mark moves in blocks and never wraps" { + try testing.expectEqual(@as(u32, 1024), reserve_block); + try testing.expectEqual(@as(u32, 1025), nextMark(1)); + try testing.expectEqual(@as(u32, 3072), nextMark(2048)); + try testing.expectEqual(@as(u32, std.math.maxInt(u32)), nextMark(std.math.maxInt(u32) - 1024)); + try testing.expectEqual(@as(u32, 0), nextMark(std.math.maxInt(u32) - 1023)); +} + +test "a record the node's owner vouches for is self-reported" { + try testing.expectEqual(Level.self_reported, Level.of("", "dmitrii-f-t27")); + try testing.expectEqual(Level.self_reported, Level.of("dmitrii-f-t27", "dmitrii-f-t27")); + try testing.expectEqual(Level.verified, Level.of("gHashTag", "dmitrii-f-t27")); +} + +test "a journal rebuilds the paid set and the highest mark" { + var spent: std.AutoHashMapUnmanaged(u64, void) = .empty; + defer spent.deinit(testing.allocator); + const text = + "{\"mark\":1024}\n" ++ + "{\"t\":1,\"operator\":\"\",\"owner\":\"a\",\"node\":\"4e4f4431\",\"physical\":false,\"nonce\":1,\"y\":3,\"tag\":\"00\",\"tag_kind\":\"keyed\",\"level\":\"self_reported\",\"mtri\":1}\n" ++ + "{\"mark\":2048}\n"; + const r = try parse(testing.allocator, text, &spent); + try testing.expectEqual(@as(u32, 2048), r.mark); + try testing.expectEqual(@as(u64, 1), r.credits); + try testing.expect(spent.contains(spentKey(0x4E4F4431, 1))); + try testing.expect(!r.torn_tail); +} + +test "a torn last line is skipped, a malformed whole line is refused" { + var spent: std.AutoHashMapUnmanaged(u64, void) = .empty; + defer spent.deinit(testing.allocator); + const torn = "{\"mark\":1024}\n{\"t\":1,\"owner\":\"a\",\"node\":\"4e4f"; + const r = try parse(testing.allocator, torn, &spent); + try testing.expect(r.torn_tail); + try testing.expectEqual(@as(u32, 1024), r.mark); + try testing.expectEqual(@as(u64, 0), r.credits); + + try testing.expectError(Error.JournalMalformed, parse(testing.allocator, "{\"owner\":\"a\"}\n", &spent)); + try testing.expectError(Error.JournalMalformed, parse(testing.allocator, "{\"mark\":x}\n", &spent)); + try testing.expectError(Error.JournalMalformed, parse(testing.allocator, "{\"node\":\"zz\",\"nonce\":1}\n", &spent)); +} + +test "a missing journal is fresh, an unreadable one is an error" { + if (comptime !builtin.link_libc) return error.SkipZigTest; + var spent: std.AutoHashMapUnmanaged(u64, void) = .empty; + defer spent.deinit(testing.allocator); + const missing: Journal = .{ .path = "/tmp/trinet-journal-test-missing-906.jsonl" }; + _ = std.c.unlink(missing.path.ptr); + const r = try missing.restore(testing.allocator, &spent); + try testing.expectEqual(@as(u32, 0), r.mark); + + // A directory exists but cannot be read as a journal. + const dir: Journal = .{ .path = "/tmp" }; + try testing.expectError(Error.JournalUnreadable, dir.restore(testing.allocator, &spent)); +} + +test "writing on top of a torn tail is what corrupts a journal" { + if (comptime !builtin.link_libc) return error.SkipZigTest; + // This is why a torn tail refuses to open for writing instead of being + // appended to: `append` uses fopen("a"), so the next line the coordinator + // writes is glued onto the unfinished one. Both halves below write exactly + // the mark a resuming run would have written next, and neither result is + // survivable. + const head = "{\"mark\":1024}\n"; + + // Torn before the nonce: the glued line stops parsing, so from here on the + // journal is unreadable and crediting stops for good. + { + const j: Journal = .{ .path = "/tmp/trinet-journal-test-glue-early-906.jsonl" }; + _ = std.c.unlink(j.path.ptr); + defer _ = std.c.unlink(j.path.ptr); + try writeText(j.path, head ++ "{\"t\":2,\"operator\":\"\",\"owner\":\"a\",\"node\":\"4e4f4431\",\"physi"); + try j.writeMark(2048); + + var spent: std.AutoHashMapUnmanaged(u64, void) = .empty; + defer spent.deinit(testing.allocator); + try testing.expectError(Error.JournalMalformed, j.restore(testing.allocator, &spent)); + } + + // Torn a few fields later: worse, because it is silent. The glued line + // parses as a paid credit and swallows the mark written into it, so the + // next start resumes at the OLD mark and reissues nonces the previous run + // may already have handed out -- the exact double payment this file exists + // to prevent. + { + const j: Journal = .{ .path = "/tmp/trinet-journal-test-glue-late-906.jsonl" }; + _ = std.c.unlink(j.path.ptr); + defer _ = std.c.unlink(j.path.ptr); + try writeText(j.path, head ++ "{\"t\":2,\"operator\":\"\",\"owner\":\"a\",\"node\":\"4e4f4432\",\"physical\":false,\"nonce\":5,\"y\":3,\"tag\":\"00"); + try j.writeMark(2048); + + var spent: std.AutoHashMapUnmanaged(u64, void) = .empty; + defer spent.deinit(testing.allocator); + const r = try j.restore(testing.allocator, &spent); + try testing.expectEqual(@as(u32, 1024), r.mark); // the 2048 just written is gone + try testing.expect(spent.contains(spentKey(0x4E4F4432, 5))); + try testing.expect(!r.torn_tail); // and it no longer looks damaged + } +} + +test "a mark and a credit written are read back" { + if (comptime !builtin.link_libc) return error.SkipZigTest; + const j: Journal = .{ .path = "/tmp/trinet-journal-test-roundtrip-906.jsonl" }; + _ = std.c.unlink(j.path.ptr); + defer _ = std.c.unlink(j.path.ptr); + try j.writeMark(1024); + try j.writeCredit(.{ .operator = "", .owner = "dmitrii-f-t27", .node_id = 0x5452494E, .physical = true, .nonce = 7, .y = -2, .tag = 0xABCD, .tag_kind = "keyed", .level = .self_reported, .mtri = 1 }); + try testing.expectError(Error.JournalWrite, j.writeCredit(.{ .operator = "", .owner = "a\"b", .node_id = 1, .physical = false, .nonce = 1, .y = 0, .tag = 0, .tag_kind = "keyed", .level = .self_reported, .mtri = 1 })); + + var spent: std.AutoHashMapUnmanaged(u64, void) = .empty; + defer spent.deinit(testing.allocator); + const r = try j.restore(testing.allocator, &spent); + try testing.expectEqual(@as(u32, 1024), r.mark); + try testing.expectEqual(@as(u64, 1), r.credits); + try testing.expect(spent.contains(spentKey(0x5452494E, 7))); +} diff --git a/src/trinet/ledger.zig b/src/trinet/ledger.zig index 54aea75a9..94acacc76 100644 --- a/src/trinet/ledger.zig +++ b/src/trinet/ledger.zig @@ -28,6 +28,7 @@ const std = @import("std"); const protocol = @import("protocol.zig"); +const journal_mod = @import("journal.zig"); /// Credits are tracked in milli-TRI so that per-job rewards stay integral. pub const milli: u64 = 1000; @@ -131,6 +132,11 @@ pub const Outcome = enum { unverifiable_not_charged, /// Node is suspended and should not have been dispatched to. not_eligible, + /// The coordinator could not put the credit on disk, or the nonce was not + /// issued by this coordinator run. Not credited and NOT slashed: both are + /// statements about us. A credit that is not on disk does not count + /// (gHashTag/t27 specs/trinet/node-work-credit.t27 may_credit). + not_recorded, }; pub const Settlement = struct { @@ -146,6 +152,9 @@ pub const Error = error{ UnknownNode, DuplicateNode, InsufficientStake, + /// With a journal open, an owner name the journal cannot hold. Refused at + /// registration: accepted here it would earn work and never be credited. + InvalidOwner, OutOfMemory, }; @@ -160,6 +169,12 @@ pub const Ledger = struct { total_corrupted: u64 = 0, total_unverifiable: u64 = 0, jobs_on_silicon: u64 = 0, + /// The durable record of what was paid (journal.zig). Without one, the + /// paid set lives in memory and a restart forgets it. + journal: ?journal_mod.Journal = null, + /// Who runs this coordinator. Empty, or equal to a node's owner, makes + /// that node's credits self-reported. + operator: []const u8 = "", pub fn init(gpa: std.mem.Allocator, policy: Policy) Error!Ledger { if (!policy.isSound()) return Error.UnsoundPolicy; @@ -171,10 +186,32 @@ pub const Ledger = struct { self.spent_nonces.deinit(self.gpa); } + /// Open the coordinator's journal: rebuild the paid set from it and return + /// the nonce mark the coordinator must resume at. An unreadable journal is + /// an error, never an empty one. + /// + /// This is the spec's `may_start_crediting` (gHashTag/t27 + /// specs/trinet/node-work-credit.t27): nothing is credited and no nonce is + /// issued unless the journal and its mark were read. A torn last line is + /// readable but not writable -- appending would glue the next line onto it + /// -- so it is refused here, before any nonce or credit, and the file is + /// left exactly as it was for an operator to look at. + pub fn openJournal(self: *Ledger, j: journal_mod.Journal, operator: []const u8) journal_mod.Error!journal_mod.Restored { + if (operator.len != 0 and !journal_mod.isHandle(operator)) return journal_mod.Error.InvalidHandle; + var it = self.accounts.valueIterator(); + while (it.next()) |a| if (!journal_mod.isHandle(a.owner)) return journal_mod.Error.InvalidHandle; + const r = try j.restore(self.gpa, &self.spent_nonces); + if (r.torn_tail) return journal_mod.Error.JournalTornTail; + self.journal = j; + self.operator = operator; + return r; + } + /// A developer attaches a node and bonds a stake. This is the whole /// onboarding step: an identity, an owner, and something to lose. pub fn register(self: *Ledger, node_id: u32, owner: []const u8, physical: bool, stake_mtri: u64) Error!void { if (self.accounts.contains(node_id)) return Error.DuplicateNode; + if (self.journal != null and !journal_mod.isHandle(owner)) return Error.InvalidOwner; if (stake_mtri < self.policy.min_stake_mtri) return Error.InsufficientStake; try self.accounts.put(self.gpa, node_id, .{ .node_id = node_id, @@ -292,7 +329,11 @@ pub const Ledger = struct { // Pay once per nonce, per node. Without this a node can resubmit the // same verified receipt forever. - const key = (@as(u64, dispatched_to) << 32) | job.nonceValue(); + // + // The key comes from journal.zig, which is also what `restore` rebuilds + // the set with. Spelling the same shift out twice is how a restart + // quietly stops recognising what it already paid. + const key = journal_mod.spentKey(dispatched_to, job.nonceValue()); if (self.spent_nonces.contains(key)) { return .{ .node_id = dispatched_to, @@ -300,9 +341,39 @@ pub const Ledger = struct { .detail = "receipt for an already-settled nonce", }; } - try self.spent_nonces.put(self.gpa, key, {}); - const reward = self.policy.reward_per_job_mtri; + + // Room for the paid key BEFORE the line goes to disk. The other order -- + // fsync the credit, then allocate -- can fail after the credit is + // durable, leaving a line on disk that this run never marked as paid; a + // later settle of the same (node, nonce) would then write a second line + // for it, which is the one thing the journal exists to prevent. Asking + // for the room first makes the failure happen before anything is + // written, and `putAssumeCapacity` below cannot fail. + try self.spent_nonces.ensureUnusedCapacity(self.gpa, 1); + + // Write before credit: with a journal, a credit counts only once its + // line is on disk, so a crash can lose a credit but never repeat one. + if (self.journal) |j| { + j.writeCredit(.{ + .operator = self.operator, + .owner = acct.owner, + .node_id = dispatched_to, + .physical = acct.physical, + .nonce = job.nonceValue(), + .y = receipt.y, + .tag = receipt.tag, + .tag_kind = @tagName(receipt.kind), + .level = journal_mod.Level.of(self.operator, acct.owner), + .mtri = reward, + }) catch return .{ + .node_id = dispatched_to, + .outcome = .not_recorded, + .detail = "journal write failed: not credited", + }; + } + self.spent_nonces.putAssumeCapacity(key, {}); + acct.credit_mtri += reward; acct.accepted += 1; acct.consecutive_rejections = 0; @@ -565,6 +636,116 @@ test "a verifier holding no key charges nobody" { try std.testing.expectEqual(Status.active, l.get(0xB0A2D).?.status); } +test "a receipt we cannot check is recorded nowhere, not even as verified" { + // `verified` is a statement about who vouched, not about the arithmetic: + // a coordinator run by somebody other than the node's owner writes that + // level on every credit it records. So the question this test asks is + // whether a receipt the coordinator could not judge can reach the journal + // at all under exactly those conditions -- because if it could, the record + // would read "verified" over a receipt nobody verified. + if (comptime !@import("builtin").link_libc) return error.SkipZigTest; + const j: journal_mod.Journal = .{ .path = "/tmp/trinet-ledger-unverifiable-906.jsonl" }; + _ = std.c.unlink(j.path.ptr); + defer _ = std.c.unlink(j.path.ptr); + + var l = try Ledger.init(std.testing.allocator, .{}); + defer l.deinit(); + try l.register(0xB0A2D, "dmitrii-f-t27", true, 100_000); + _ = try l.openJournal(j, "gHashTag"); + try std.testing.expectEqual(journal_mod.Level.verified, journal_mod.Level.of("gHashTag", "dmitrii-f-t27")); + + // A keyed receipt and no key to check it with. + const job = makeJob(1); + var key: [16]u8 = undefined; + for (&key, 0..) |*b, i| b.* = @intCast(0x33 +% i); + const y = protocol.dot(job.w, job.x); + const keyed: protocol.Receipt = .{ + .kind = .siphash24, + .y = y, + .status = protocol.status_ok, + .nonce = job.nonce, + .node_id = 0xB0A2D, + .tag = protocol.receiptTagKeyed(job, y, 0xB0A2D, key), + }; + const verdict = protocol.verifyWithKey(job, keyed, null); + try std.testing.expectEqual(protocol.Verdict.unverifiable, verdict); + const s1 = try l.settle(0xB0A2D, job, keyed, verdict); + try std.testing.expectEqual(Outcome.unverifiable_not_charged, s1.outcome); + + // And a receipt whose tag was altered on the way back. + const job2 = makeJob(2); + var tampered = protocol.execute(job2, 0xB0A2D); + tampered.tag ^= 0x40; + const s2 = try l.settle(0xB0A2D, job2, tampered, protocol.verify(job2, tampered)); + try std.testing.expectEqual(Outcome.corrupt_not_charged, s2.outcome); + + // Neither of them is on disk, at any level. + var spent: std.AutoHashMapUnmanaged(u64, void) = .empty; + defer spent.deinit(std.testing.allocator); + const restored = try j.restore(std.testing.allocator, &spent); + try std.testing.expectEqual(@as(u64, 0), restored.credits); + try std.testing.expectEqual(@as(u64, 0), l.total_credited_mtri); +} + +test "a credit the journal refused is neither paid nor marked as paid" { + if (comptime !@import("builtin").link_libc) return error.SkipZigTest; + var l = try Ledger.init(std.testing.allocator, .{}); + defer l.deinit(); + try l.register(0x5005, "dmitrii-f-t27", true, 5000); + // Opening this is fine -- a journal that is not there yet is a fresh + // network. Writing to it can never work. + _ = try l.openJournal(.{ .path = "/nonexistent-dir-906/journal.jsonl" }, ""); + + const job = makeJob(5); + const r = protocol.execute(job, 0x5005); + const s = try l.settle(0x5005, job, r, protocol.verify(job, r)); + + try std.testing.expectEqual(Outcome.not_recorded, s.outcome); + try std.testing.expectEqual(@as(u64, 0), s.credit_delta_mtri); + // Not a punishment: a journal we cannot write is our failure, not the + // node's. + try std.testing.expectEqual(@as(u64, 0), s.slash_delta_mtri); + try std.testing.expectEqual(@as(u64, 0), l.get(0x5005).?.credit_mtri); + try std.testing.expectEqual(@as(u64, 5000), l.get(0x5005).?.stake_mtri); + try std.testing.expectEqual(@as(u64, 0), l.get(0x5005).?.rejected); + try std.testing.expectEqual(@as(u64, 0), l.total_credited_mtri); + // And not marked paid, so the same work is still payable once the journal + // works again. The line on disk, not this set, is what stops a second + // payment. + try std.testing.expect(!l.spent_nonces.contains(journal_mod.spentKey(0x5005, job.nonceValue()))); +} + +test "no credit reaches the disk that the paid set has no room to record" { + // The write-then-record order has one more failure between its two halves: + // the paid set has to grow. If it grew after the fsync, an allocation + // failure would leave a durable credit this run never marked paid, and the + // next settle of the same (node, nonce) would append a second line for it. + if (comptime !@import("builtin").link_libc) return error.SkipZigTest; + const j: journal_mod.Journal = .{ .path = "/tmp/trinet-ledger-noroom-906.jsonl" }; + _ = std.c.unlink(j.path.ptr); + defer _ = std.c.unlink(j.path.ptr); + + var l = try Ledger.init(std.testing.allocator, .{}); + defer l.deinit(); + try l.register(0x7007, "dmitrii-f-t27", true, 5000); + _ = try l.openJournal(j, ""); + + const job = makeJob(9); + const r = protocol.execute(job, 0x7007); + l.gpa = std.testing.failing_allocator; + try std.testing.expectError(Error.OutOfMemory, l.settle(0x7007, job, r, protocol.verify(job, r))); + // The accounts map was allocated by the real allocator and must be freed by + // it; the paid set never grew, so it owns nothing. + l.gpa = std.testing.allocator; + + var spent: std.AutoHashMapUnmanaged(u64, void) = .empty; + defer spent.deinit(std.testing.allocator); + const restored = try j.restore(std.testing.allocator, &spent); + try std.testing.expectEqual(@as(u64, 0), restored.credits); + try std.testing.expectEqual(@as(u64, 0), l.get(0x7007).?.credit_mtri); + try std.testing.expectEqual(@as(u64, 0), l.total_credited_mtri); +} + test "work we can never pay for eventually stops being dispatched" { // Not punishment — no stake moves — but there is no point sending jobs to a // node whose answers can never be credited. diff --git a/src/trinet/main.zig b/src/trinet/main.zig index 29e80723c..cb35c58d3 100644 --- a/src/trinet/main.zig +++ b/src/trinet/main.zig @@ -285,6 +285,55 @@ var fleet_nodes = [_]FleetNode{ const key_file_env = "TRINET_KEYS"; const key_file_default = "trinet-keys.txt"; +/// The coordinator's journal (journal.zig, gHashTag/trinity-fpga#906). Unset: +/// credits live in memory, as before. Set: paid (node, nonce) pairs and the +/// nonce mark survive a restart, and a credit counts only once it is on disk. +const journal_env = "TRINET_JOURNAL"; +/// The handle a board attached by this process is credited to. +const owner_env = "TRINET_OWNER"; +/// Who runs this coordinator. Unset, or equal to the owner, writes every +/// credit as self-reported (gHashTag/t27 specs/trinet/node-work-credit.t27). +const operator_env = "TRINET_OPERATOR"; + +fn envOr(name: [*:0]const u8, default: []const u8) []const u8 { + const v = std.c.getenv(name) orelse return default; + const s = std.mem.span(v); + return if (s.len == 0) default else s; +} + +fn ownerHandle() []const u8 { + return envOr(owner_env, "operator"); +} + +/// Open TRINET_JOURNAL if it is set. A journal that cannot be read stops the +/// run: crediting without knowing what was already paid is the bug it fixes. +fn openJournalFromEnv(m: *mesh_mod.Mesh) !void { + const p = std.c.getenv(journal_env) orelse return; + const path = std.mem.span(p); + const operator = envOr(operator_env, ""); + const r = m.openJournal(.{ .path = path }, operator) catch |e| { + std.debug.print("journal {s}: {s} -- not crediting without it\n", .{ path, @errorName(e) }); + switch (e) { + // Say what it is, because the file is fine to READ and the fix is + // an operator's decision, not a retry. + error.JournalTornTail => std.debug.print("The last line was never finished: a previous run died mid-write.\n" ++ + "Appending to it would glue the next line onto the tear, so this run\n" ++ + "stops and the file is left exactly as it is. Stop here and keep the\n" ++ + "file unchanged: it holds the paid set and the nonce reserve mark.\n" ++ + "Recover it only under control, after checking its complete credit\n" ++ + "lines and its last complete mark line.\n", .{}), + else => {}, + } + return e; + }; + std.debug.print("journal {s}: {d} paid jobs restored, nonces resume at {d}, credits {s}\n", .{ + path, + r.credits, + m.next_nonce, + if (operator.len == 0) "self-reported (TRINET_OPERATOR unset)" else "by operator", + }); +} + /// The UART divisor the fleet bitstream is built with. The board's line rate is /// CFGMCLK / this, so it is also the only honest way to read CFGMCLK back out of /// a negotiated rate. @@ -352,6 +401,7 @@ fn fleet(gpa: std.mem.Allocator, ports: []const [:0]const u8) !void { var m = try mesh_mod.Mesh.init(gpa, .{}); defer m.deinit(); + try openJournalFromEnv(&m); var attached: usize = 0; var stale_boards: usize = 0; @@ -406,7 +456,7 @@ fn fleet(gpa: std.mem.Allocator, ports: []const [:0]const u8) !void { stale_boards += 1; std.debug.print("{s}: PUBLISHED KEY — receipts carry no evidence. Re-flash with `trinet keygen`.\n", .{p}); } - try m.join(n, "operator", 100000); + try m.join(n, ownerHandle(), 100000); attached += 1; std.debug.print("{s}: identified as {s}, id {x:0>8} at {d} baud{s}\n", .{ p, spec.?.name, spec.?.id, found.baud, @@ -716,11 +766,12 @@ fn bench(gpa: std.mem.Allocator, path: []const u8, n: usize, baud: u32) !void { fn buildMesh(gpa: std.mem.Allocator, serial_path: ?[]const u8, buf: []u8) !mesh_mod.Mesh { var m = try mesh_mod.Mesh.init(gpa, .{}); errdefer m.deinit(); + try openJournalFromEnv(&m); if (serial_path) |p| { const zpath = try std.fmt.bufPrintZ(buf, "{s}", .{p}); if (node_mod.Node.initFpga(protocol.default_node_id, "ax7203-node0", zpath, default_baud)) |fpga| { - try m.join(fpga, "operator", 100000); + try m.join(fpga, ownerHandle(), 100000); std.debug.print("node 0: physical AX7203 on {s}\n", .{p}); } else |e| { std.debug.print("node 0: no board ({s}) — running without a physical node\n", .{@errorName(e)}); diff --git a/src/trinet/mesh.zig b/src/trinet/mesh.zig index af96125e1..1ae1129bd 100644 --- a/src/trinet/mesh.zig +++ b/src/trinet/mesh.zig @@ -18,6 +18,7 @@ const std = @import("std"); const protocol = @import("protocol.zig"); const node_mod = @import("node.zig"); const ledger_mod = @import("ledger.zig"); +const journal_mod = @import("journal.zig"); pub const Node = node_mod.Node; pub const Ledger = ledger_mod.Ledger; @@ -33,7 +34,13 @@ pub const Error = error{ UnknownNode, DuplicateNode, InsufficientStake, + InvalidOwner, UnsoundPolicy, + /// The journal could not take the next nonce mark, so no nonce is issued. + JournalWrite, + JournalUnavailable, + /// The 32-bit nonce space is spent; nodes need a new key epoch. + NonceSpaceExhausted, }; pub const JobOutcome = struct { @@ -68,6 +75,8 @@ pub const Stats = struct { corrupt_jobs: u64 = 0, unverifiable_jobs: u64 = 0, not_eligible_jobs: u64 = 0, + /// Verified but not put on disk, or a nonce this run did not issue. + not_recorded_jobs: u64 = 0, on_silicon: u64 = 0, in_software: u64 = 0, @@ -92,6 +101,10 @@ pub const Mesh = struct { stats: Stats = .{}, cursor: usize = 0, next_nonce: u32 = 1, + /// With a journal: the highest nonce mark on disk, and where this run + /// resumed. Nonces are issued only in [resumed_from, nonce_mark). + nonce_mark: u32 = 0, + resumed_from: u32 = 1, pub fn init(gpa: std.mem.Allocator, policy: ledger_mod.Policy) Error!Mesh { return .{ .gpa = gpa, .ledger = try Ledger.init(gpa, policy) }; @@ -122,12 +135,38 @@ pub const Mesh = struct { return k; } - pub fn freshNonce(self: *Mesh) u32 { + /// Open the coordinator's journal (journal.zig): rebuild what was paid and + /// resume nonces at the last mark, so a restart never reissues a nonce. + pub fn openJournal(self: *Mesh, j: journal_mod.Journal, operator: []const u8) journal_mod.Error!journal_mod.Restored { + const r = try self.ledger.openJournal(j, operator); + self.nonce_mark = r.mark; + self.next_nonce = if (r.mark == 0) 1 else r.mark; + self.resumed_from = self.next_nonce; + return r; + } + + pub fn freshNonce(self: *Mesh) Error!u32 { + if (self.ledger.journal) |j| { + // Reserve before issuing: the mark is on disk before any nonce at + // or above the old mark goes out. + if (self.next_nonce >= self.nonce_mark) { + const mark = journal_mod.nextMark(self.nonce_mark); + if (mark == 0 or mark <= self.next_nonce) return Error.NonceSpaceExhausted; + j.writeMark(mark) catch return Error.JournalWrite; + self.nonce_mark = mark; + } + } const n = self.next_nonce; self.next_nonce +%= 1; return n; } + /// With a journal, only a nonce this run issued may be credited. + fn issuedByThisRun(self: *Mesh, nonce: u32) bool { + if (self.ledger.journal == null) return true; + return nonce >= self.resumed_from and nonce < self.next_nonce; + } + fn pickEligible(self: *Mesh) ?*Node { const count = self.nodes.items.len; if (count == 0) return null; @@ -167,6 +206,18 @@ pub const Mesh = struct { if (job.nonceValue() > n.highest_nonce_issued) n.highest_nonce_issued = job.nonceValue(); + if (!self.issuedByThisRun(job.nonceValue())) { + self.stats.not_recorded_jobs += 1; + return .{ + .node_id = n.id, + .node_name = n.name, + .physical = n.isPhysical(), + .y = receipt.y, + .verdict = protocol.verifyWithKey(job, receipt, n.key), + .settlement = .{ .node_id = n.id, .outcome = .not_recorded, .detail = "nonce not issued by this coordinator run" }, + }; + } + var verdict = protocol.verifyWithKey(job, receipt, n.key); // A nonce mismatch means either a replay attack or a stream that lost a @@ -198,6 +249,7 @@ pub const Mesh = struct { // We declined to use the node. That is our scheduling decision, not // the node's conduct. .not_eligible => self.stats.not_eligible_jobs += 1, + .not_recorded => self.stats.not_recorded_jobs += 1, .rejected_and_slashed, .identity_mismatch => { self.stats.rejected += 1; n.stats.rejected += 1; @@ -320,7 +372,7 @@ pub const Mesh = struct { for (0..take) |k| { const row = next_row + k; - const nonce = self.freshNonce(); + const nonce = try self.freshNonce(); if (nonce > n.highest_nonce_issued) n.highest_nonce_issued = nonce; jobs[k] = protocol.Job.withNonce(nonce, rows[row], x); row_of[k] = row; @@ -350,6 +402,7 @@ pub const Mesh = struct { .corrupt_not_charged => self.stats.corrupt_jobs += 1, .unverifiable_not_charged => self.stats.unverifiable_jobs += 1, .not_eligible => self.stats.not_eligible_jobs += 1, + .not_recorded => self.stats.not_recorded_jobs += 1, .rejected_and_slashed, .identity_mismatch => { self.stats.rejected += 1; n.stats.rejected += 1; @@ -663,3 +716,256 @@ test "a mesh with no eligible node fails loudly instead of silently faking work" defer m.deinit(); try std.testing.expectError(Error.NoEligibleNode, m.dispatch(testJob(0))); } + +// --------------------------------------------------------------------------- +// The coordinator's journal across restarts (gHashTag/trinity-fpga#906, +// gHashTag/t27 specs/trinet/node-work-credit.t27). + +fn journalText(path: [:0]const u8, buf: []u8) []const u8 { + const f = std.c.fopen(path.ptr, "r") orelse return ""; + defer _ = std.c.fclose(f); + return buf[0..std.c.fread(buf.ptr, 1, buf.len, f)]; +} + +fn journaledJob(m: *Mesh, seed: u8) !protocol.Job { + return protocol.Job.withNonce(try m.freshNonce(), @splat(seed), @splat(0x55)); +} + +/// Lay down a journal by hand, to restart on top of one a running coordinator +/// cannot produce on purpose: a torn tail, or a line that does not parse. +fn writeJournalText(path: [:0]const u8, text: []const u8) !void { + const f = std.c.fopen(path.ptr, "w") orelse return error.JournalWrite; + const written = std.c.fwrite(text.ptr, 1, text.len, f); + if (std.c.fclose(f) != 0 or written != text.len) return error.JournalWrite; +} + +test "a restarted coordinator never pays the same (node, nonce) twice" { + if (comptime !@import("builtin").link_libc) return error.SkipZigTest; + const j: journal_mod.Journal = .{ .path = "/tmp/trinet-mesh-restart-906.jsonl" }; + _ = std.c.unlink(j.path.ptr); + defer _ = std.c.unlink(j.path.ptr); + + var saved: protocol.Job = undefined; + { + var m = try Mesh.init(std.testing.allocator, .{}); + defer m.deinit(); + try m.join(Node.initEmulated(0x4E4F4431, "peer-1", .honest), "dmitrii-f-t27", 100000); + const r = try m.openJournal(j, "dmitrii-f-t27"); + try std.testing.expectEqual(@as(u32, 0), r.mark); + for (0..3) |i| { + const job = try journaledJob(&m, @intCast(i)); + if (i == 0) saved = job; + const o = try m.dispatch(job); + try std.testing.expectEqual(ledger_mod.Outcome.credited, o.settlement.outcome); + } + try std.testing.expectEqual(@as(u32, 1), saved.nonceValue()); + } + { + // The restart: a new process, the same journal. + var m = try Mesh.init(std.testing.allocator, .{}); + defer m.deinit(); + try m.join(Node.initEmulated(0x4E4F4431, "peer-1", .honest), "dmitrii-f-t27", 100000); + const r = try m.openJournal(j, "dmitrii-f-t27"); + try std.testing.expectEqual(@as(u64, 3), r.credits); + try std.testing.expectEqual(@as(u32, 1024), r.mark); + + // The job saved from the first run earns nothing now: its nonce lies + // below where this run resumed. + const replayed = try m.dispatch(saved); + try std.testing.expectEqual(ledger_mod.Outcome.not_recorded, replayed.settlement.outcome); + try std.testing.expectEqual(@as(u64, 0), replayed.settlement.credit_delta_mtri); + + // New work resumes at the mark, never at 1. + const fresh = try journaledJob(&m, 9); + try std.testing.expectEqual(@as(u32, 1024), fresh.nonceValue()); + const o = try m.dispatch(fresh); + try std.testing.expectEqual(ledger_mod.Outcome.credited, o.settlement.outcome); + } + var buf: [4096]u8 = undefined; + const text = journalText(j.path, &buf); + try std.testing.expectEqual(@as(usize, 4), std.mem.count(u8, text, "\"mtri\":1}")); + try std.testing.expectEqual(@as(usize, 1), std.mem.count(u8, text, "\"nonce\":1,")); + try std.testing.expect(std.mem.indexOf(u8, text, "{\"mark\":2048}") != null); +} + +test "a coordinator run by the node's owner writes self-reported credits" { + if (comptime !@import("builtin").link_libc) return error.SkipZigTest; + const j: journal_mod.Journal = .{ .path = "/tmp/trinet-mesh-level-906.jsonl" }; + for ([_][]const u8{ "dmitrii-f-t27", "", "gHashTag" }) |operator| { + _ = std.c.unlink(j.path.ptr); + var m = try Mesh.init(std.testing.allocator, .{}); + defer m.deinit(); + try m.join(Node.initEmulated(0x4E4F4431, "peer-1", .honest), "dmitrii-f-t27", 100000); + _ = try m.openJournal(j, operator); + _ = try m.dispatch(try journaledJob(&m, 1)); + var buf: [1024]u8 = undefined; + const text = journalText(j.path, &buf); + const want: []const u8 = if (std.mem.eql(u8, operator, "gHashTag")) "\"level\":\"verified\"" else "\"level\":\"self_reported\""; + try std.testing.expect(std.mem.indexOf(u8, text, want) != null); + } + _ = std.c.unlink(j.path.ptr); +} + +test "a forged answer is refused and never reaches the journal" { + if (comptime !@import("builtin").link_libc) return error.SkipZigTest; + const j: journal_mod.Journal = .{ .path = "/tmp/trinet-mesh-forged-906.jsonl" }; + _ = std.c.unlink(j.path.ptr); + defer _ = std.c.unlink(j.path.ptr); + var m = try Mesh.init(std.testing.allocator, .{}); + defer m.deinit(); + try m.join(Node.initEmulated(0x4E4F4432, "lazy", .lazy), "dmitrii-f-t27", 100000); + _ = try m.openJournal(j, "dmitrii-f-t27"); + const o = try m.dispatch(protocol.Job.withNonce(try m.freshNonce(), @splat(0x55), @splat(0x55))); + try std.testing.expect(o.settlement.outcome != .credited); + var buf: [1024]u8 = undefined; + try std.testing.expectEqual(@as(usize, 0), std.mem.count(u8, journalText(j.path, &buf), "\"mtri\"")); +} + +test "every credit line names the owner its own node was registered to" { + // The handle in the line is what a payout is read off, so it has to be the + // node's owner -- not the coordinator's operator, and not whichever owner + // happened to be settled first. + if (comptime !@import("builtin").link_libc) return error.SkipZigTest; + const j: journal_mod.Journal = .{ .path = "/tmp/trinet-mesh-owner-906.jsonl" }; + _ = std.c.unlink(j.path.ptr); + defer _ = std.c.unlink(j.path.ptr); + + var m = try Mesh.init(std.testing.allocator, .{}); + defer m.deinit(); + try m.join(Node.initEmulated(0x4E4F4431, "peer-1", .honest), "developer-1", 100000); + try m.join(Node.initEmulated(0x4E4F4432, "peer-2", .honest), "developer-2", 100000); + _ = try m.openJournal(j, "gHashTag"); + + // Round-robin: one job each. + for (0..2) |_| { + const o = try m.dispatch(try journaledJob(&m, 1)); + try std.testing.expectEqual(ledger_mod.Outcome.credited, o.settlement.outcome); + } + + var buf: [4096]u8 = undefined; + const text = journalText(j.path, &buf); + try std.testing.expectEqual(@as(usize, 1), std.mem.count(u8, text, "\"owner\":\"developer-1\"")); + try std.testing.expectEqual(@as(usize, 1), std.mem.count(u8, text, "\"owner\":\"developer-2\"")); + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, text, "\"operator\":\"gHashTag\"")); + // The operator is not an owner of anything here. + try std.testing.expectEqual(@as(usize, 0), std.mem.count(u8, text, "\"owner\":\"gHashTag\"")); +} + +test "a journal torn mid-line refuses to open, twice over, and is left untouched" { + // The regression this exists for: the coordinator used to open a torn + // journal successfully and carry on writing. `append` uses fopen("a"), so + // the first mark it wrote was glued onto the unfinished line -- see + // journal.zig "writing on top of a torn tail is what corrupts a journal" + // for what that does. One start cannot see it; the SECOND start is where + // the damage surfaces, so this test performs both. + if (comptime !@import("builtin").link_libc) return error.SkipZigTest; + const j: journal_mod.Journal = .{ .path = "/tmp/trinet-mesh-torn-906.jsonl" }; + defer _ = std.c.unlink(j.path.ptr); + // A mark, one whole credit, and a line the process died inside. + const torn = "{\"mark\":1024}\n" ++ + "{\"t\":1,\"operator\":\"\",\"owner\":\"dmitrii-f-t27\",\"node\":\"4e4f4431\",\"physical\":false,\"nonce\":1,\"y\":3,\"tag\":\"0000000000000000\",\"tag_kind\":\"crc32\",\"level\":\"self_reported\",\"mtri\":1}\n" ++ + "{\"t\":2,\"operator\":\"\",\"owner\":\"dmitrii-f-t27\",\"node\":\"4e4f4431\",\"physi"; + try writeJournalText(j.path, torn); + + // First start after the crash: refused before any nonce or credit. + { + var m = try Mesh.init(std.testing.allocator, .{}); + defer m.deinit(); + try m.join(Node.initEmulated(0x4E4F4431, "peer-1", .honest), "dmitrii-f-t27", 100000); + try std.testing.expectError(journal_mod.Error.JournalTornTail, m.openJournal(j, "")); + try std.testing.expect(m.ledger.journal == null); + try std.testing.expectEqual(@as(u32, 0), m.nonce_mark); + try std.testing.expectEqual(@as(u64, 0), m.ledger.total_credited_mtri); + } + + var buf: [4096]u8 = undefined; + try std.testing.expectEqualStrings(torn, journalText(j.path, &buf)); + + // Second start: the same named error, not a journal that has since become + // unparsable, and still not a byte written. + { + var m = try Mesh.init(std.testing.allocator, .{}); + defer m.deinit(); + try m.join(Node.initEmulated(0x4E4F4431, "peer-1", .honest), "dmitrii-f-t27", 100000); + try std.testing.expectError(journal_mod.Error.JournalTornTail, m.openJournal(j, "")); + } + var buf2: [4096]u8 = undefined; + try std.testing.expectEqualStrings(torn, journalText(j.path, &buf2)); + + // Read-only inspection still describes the file rather than refusing it: + // one whole credit, the mark, and a tail somebody has to decide about. + var spent: std.AutoHashMapUnmanaged(u64, void) = .empty; + defer spent.deinit(std.testing.allocator); + const seen = try j.restore(std.testing.allocator, &spent); + try std.testing.expect(seen.torn_tail); + try std.testing.expectEqual(@as(u64, 1), seen.credits); + try std.testing.expectEqual(@as(u32, 1024), seen.mark); +} + +test "a journal line that does not parse stops the coordinator, it does not read as empty" { + if (comptime !@import("builtin").link_libc) return error.SkipZigTest; + const j: journal_mod.Journal = .{ .path = "/tmp/trinet-mesh-malformed-906.jsonl" }; + defer _ = std.c.unlink(j.path.ptr); + // A complete line with no node field: we cannot tell what it paid. + try writeJournalText(j.path, "{\"mark\":1024}\n{\"t\":1,\"owner\":\"dmitrii-f-t27\",\"nonce\":1,\"mtri\":1}\n"); + + var m = try Mesh.init(std.testing.allocator, .{}); + defer m.deinit(); + try m.join(Node.initEmulated(0x4E4F4431, "peer-1", .honest), "dmitrii-f-t27", 100000); + try std.testing.expectError(journal_mod.Error.JournalMalformed, m.openJournal(j, "")); + // The journal was not adopted, so nothing here can credit against it. The + // runner turns this error into a refusal to start (main.zig). + try std.testing.expect(m.ledger.journal == null); + try std.testing.expectEqual(@as(u64, 0), m.ledger.total_credited_mtri); +} + +test "an unwritable journal issues no nonce and credits nothing" { + if (comptime !@import("builtin").link_libc) return error.SkipZigTest; + const j: journal_mod.Journal = .{ .path = "/nonexistent-dir-906/journal.jsonl" }; + var m = try Mesh.init(std.testing.allocator, .{}); + defer m.deinit(); + try m.join(Node.initEmulated(0x4E4F4431, "peer-1", .honest), "dmitrii-f-t27", 100000); + _ = try m.openJournal(j, "dmitrii-f-t27"); + try std.testing.expectError(Error.JournalWrite, m.freshNonce()); + + // Even a job whose nonce slipped through earns nothing it cannot record. + m.nonce_mark = 10; + m.next_nonce = 2; + m.resumed_from = 1; + const o = try m.dispatch(protocol.Job.withNonce(1, @splat(0x55), @splat(0x55))); + try std.testing.expectEqual(ledger_mod.Outcome.not_recorded, o.settlement.outcome); + try std.testing.expectEqual(@as(u64, 0), m.ledger.total_credited_mtri); +} + +test "an unreadable journal refuses to open" { + if (comptime !@import("builtin").link_libc) return error.SkipZigTest; + var m = try Mesh.init(std.testing.allocator, .{}); + defer m.deinit(); + try std.testing.expectError(journal_mod.Error.JournalUnreadable, m.openJournal(.{ .path = "/tmp" }, "")); +} + +test "an owner name the journal cannot hold is refused at the door, not at payout" { + if (comptime !@import("builtin").link_libc) return error.SkipZigTest; + const j: journal_mod.Journal = .{ .path = "/tmp/trinet-mesh-handle-906.jsonl" }; + _ = std.c.unlink(j.path.ptr); + defer _ = std.c.unlink(j.path.ptr); + { + var m = try Mesh.init(std.testing.allocator, .{}); + defer m.deinit(); + _ = try m.openJournal(j, ""); + try std.testing.expectError(Error.InvalidOwner, m.join(Node.initEmulated(0x4E4F4431, "peer-1", .honest), "developer 1", 100000)); + try m.join(Node.initEmulated(0x4E4F4432, "peer-2", .honest), "developer-2", 100000); + } + { + // Joined before the journal opened: the journal refuses to open. + var m = try Mesh.init(std.testing.allocator, .{}); + defer m.deinit(); + try m.join(Node.initEmulated(0x4E4F4431, "peer-1", .honest), "a\"b", 100000); + try std.testing.expectError(journal_mod.Error.InvalidHandle, m.openJournal(j, "")); + } + { + var m = try Mesh.init(std.testing.allocator, .{}); + defer m.deinit(); + try std.testing.expectError(journal_mod.Error.InvalidHandle, m.openJournal(j, "op erator")); + } +}