From 75a463fe41a68a850be24ad98cfe7d35c123db50 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 14:00:19 +0700 Subject: [PATCH 1/2] t27os keys: every stored key in rotation, failover on overload, restored panes wait for their keys (Closes #11241) The rotation lane's four commits on the T27 OS stack (claude/t27os-keyrot-11241, head 30d60ebc5), applied onto master, whose copies of these five files equal the stack head the lane built on. Co-Authored-By: Claude Sonnet 5.5 Co-Authored-By: Claude Opus 5.5 --- specs/t27os/glass_window.t27 | 78 ++- specs/t27os/keys.t27 | 346 ++++++++++++- specs/t27os/keys_host.t27 | 124 +++-- specs/t27os/pty_logic.t27 | 926 ++++++++++++++++++++++++++++++++++- specs/t27os/pty_server.t27 | 265 ++++++++-- 5 files changed, 1640 insertions(+), 99 deletions(-) diff --git a/specs/t27os/glass_window.t27 b/specs/t27os/glass_window.t27 index 221fd10ff9..2f26aa3a22 100644 --- a/specs/t27os/glass_window.t27 +++ b/specs/t27os/glass_window.t27 @@ -67,6 +67,10 @@ use t27os::layout::cap_mid; use t27os::layout::same_line; use t27os::agents::AGENT_NAMES; use t27os::keys::PROVIDER_COUNT; +use t27os::keys::PROFILE_COUNT; +use t27os::keys::MAX_KEYS; +use t27os::keys::setkey_byte; +use t27os::keys::key_count_text; use t27os::keys::MAX_KEY_CHARS; use t27os::keys::KEYS_WORDS; use t27os::keys::KY_NONE; @@ -112,7 +116,8 @@ use t27os::keys::circuit_fails; use t27os::keys::status_shown; use t27os::keys_host::kc_put; use t27os::keys_host::kc_get; -use t27os::keys_host::kc_del; +use t27os::keys_host::kc_get_n; +use t27os::keys_host::kc_del_all; use t27os::keys_host::kc_has; use t27os::keys_host::probe; use t27os::keys_host::push_keys; @@ -2328,29 +2333,46 @@ fn bytes_w(kit: [48]u64, s: []const u8, n: usize, size: f64, weight: f64, mono: return w; } -/// The Keychain is read here and nowhere else in the window: which providers hold a key, and the masked -/// text of each (16 bytes a provider). The full key is wiped from the buffer before the next one is read. +/// The Keychain is read here and nowhere else in the window: how many keys each provider holds (stored[p], +/// 0 to MAX_KEYS, every numbered account is read) and the masked text of its first key (16 bytes a +/// provider). The full key is wiped from the buffer before the next one is read. fn keys_refresh(stored: []u32, mask: []u8, mlen: []u32) -> void { var key : [256]u8 = undefined; var p : u32 = 0; while (p < PROVIDER_COUNT) { - const n : usize = kc_get(p, key[0..256]); const at : usize = @as(usize, p) * 16; mlen[p] = 0; stored[p] = 0; - if (n > 0) { - stored[p] = 1; - mlen[p] = @as(u32, @intCast(mask_key(key[0..n], mask[at .. at + 16]))); - } - var i : usize = 0; - while (i < 256) { - key[i] = 0; - i = i + 1; + var k : u32 = 0; + while (k < MAX_KEYS) { + const n : usize = kc_get_n(p, k, key[0..256]); + if (n > 0) { + if (stored[p] == 0) { mlen[p] = @as(u32, @intCast(mask_key(key[0..n], mask[at .. at + 16]))); } + stored[p] = stored[p] + 1; + } + var i : usize = 0; + while (i < 256) { + key[i] = 0; + i = i + 1; + } + k = k + 1; } p = p + 1; } } +/// Hand the server every stored key with its number (keys_host push_keys), then take back the first key of each +/// provider whose last check failed, as the check button does: a refresh must not undo that. +fn keys_sync(fd: i32, kst: []const u32, out: []u8) -> void { + push_keys(fd); + const bad : u32 = bad_mask(kst); + var p : u32 = 0; + while (p < PROVIDER_COUNT) { + if ((bad >> p) % 2 == 1) { client_set_key(fd, setkey_byte(p, 0), "", out); } + p = p + 1; + } +} + fn keys_wipe(buf: []u8) -> void { var i : usize = 0; while (i < buf.len) { @@ -2412,11 +2434,18 @@ fn draw_keys_panel(c: u64, kit: [48]u64, sw: f64, sh: f64, mx: f64, my: f64, sto var p : u32 = 0; while (p < PROVIDER_COUNT) { const y : f64 = ky_row_top(t, p); - const has : bool = stored[p] == 1; + const has : bool = stored[p] > 0; const st : u32 = status_shown(has, kst[p], kfail[p], kopen[p], now); cg_fill_rgba(c, 1.0, 1.0, 1.0, TEXT_WHITE); n = provider_name(p, w[0..48]); - draw_bytes(c, kit, w[0..48], n, x, y + 14.0, 11.0, WEIGHT_SEMIBOLD, false); + const nw : f64 = draw_bytes(c, kit, w[0..48], n, x, y + 14.0, 11.0, WEIGHT_SEMIBOLD, false); + // how many keys the provider holds ("zai 7 keys"), right after its name + n = key_count_text(stored[p], w[0..48]); + if (n > 0) { + cg_fill_rgba(c, 1.0, 1.0, 1.0, MUTED_WHITE); + draw_bytes(c, kit, w[0..48], n, x + nw + 8.0, y + 14.0, 9.5, 0.0, false); + } + cg_fill_rgba(c, 1.0, 1.0, 1.0, TEXT_WHITE); draw_dot(c, status_dot(st), x + 112.0, y + 10.0, 1.0); n = csv_field(STATUS_NAMES, st, w[0..48]); cg_fill_rgba(c, 1.0, 1.0, 1.0, LABEL_WHITE); @@ -2454,7 +2483,7 @@ fn draw_keys_panel(c: u64, kit: [48]u64, sw: f64, sh: f64, mx: f64, my: f64, sto var sm : u32 = 0; var pi : u32 = 0; while (pi < PROVIDER_COUNT) { - if (stored[pi] == 1) { sm = sm + (1 << pi); } + if (stored[pi] > 0) { sm = sm + (1 << pi); } pi = pi + 1; } var wy : [96]u8 = undefined; @@ -3761,7 +3790,7 @@ pub fn main() -> void { const kh : u32 = ky_hit(mx, my, kl, QP_W, QP_PAD, WS_TOP); const kq : u32 = kh / 4; if (kh != KY_NONE) { - const kact : u32 = ky_click(kh % 4, kstored[kq] == 1, probe_has(kq), karm, kq); + const kact : u32 = ky_click(kh % 4, kstored[kq] > 0, probe_has(kq), karm, kq); if (kact == KA_EDIT) { kedit = kq; klen = 0; @@ -3771,12 +3800,19 @@ pub fn main() -> void { kedit = KY_NONE; klen = 0; } else if (kact == KA_DELETE) { - kc_del(kq); + // the row's DELETE removes every key of the provider, from the Keychain and the server + kc_del_all(kq); karm = KY_NONE; kst[kq] = ST_STORED; kfail[kq] = 0; keys_refresh(kstored[0..9], kmask[0..144], kmlen[0..9]); - if (tfd >= 0) { client_set_key(tfd, kq, "", tout[0..4100]); } + if (tfd >= 0) { + var kdk : u32 = 0; + while (kdk < MAX_KEYS) { + client_set_key(tfd, setkey_byte(kq, kdk), "", tout[0..4100]); + kdk = kdk + 1; + } + } } else if (kact == KA_CHECK) { kbusy = kq; kpend = 0; @@ -3893,6 +3929,8 @@ pub fn main() -> void { god = false; dp = false; keys_refresh(kstored[0..9], kmask[0..144], kmlen[0..9]); + // the refresh also hands the server every numbered key again (a server that restarted forgot them) + if (tfd >= 0) { keys_sync(tfd, kst[0..9], tout[0..4100]); } const khn : usize = ns_bytes(home_dir(), khome[0..256]); kemask = installed_mask(khome[0..khn]); ktpick = term_pick(term_mask()); @@ -4130,8 +4168,8 @@ pub fn main() -> void { } } // K. pre-flight (M12): once at start, check the stored key behind each claude profile, in the order - // keys.t27 PROFILE_PROVIDER gives, so the first pane already avoids a dead key - if (kbusy == KY_NONE and kpre < 2) { + // keys.t27 PROFILE_PROVIDER gives (all PROFILE_COUNT of them), so the first pane already avoids a dead key + if (kbusy == KY_NONE and kpre < PROFILE_COUNT) { const kpp : u32 = PROFILE_PROVIDER[kpre]; kpre = kpre + 1; if (probe_has(kpp) and kc_has(kpp)) { diff --git a/specs/t27os/keys.t27 b/specs/t27os/keys.t27 index 38d52134fa..d3f4f6ed75 100644 --- a/specs/t27os/keys.t27 +++ b/specs/t27os/keys.t27 @@ -5,7 +5,8 @@ ; Owner, 2026-10-19: a KEYS tab modelled on trios, every key managed by one manager written in .t27. ; Mirrors the trios SR-00 decisions (not the Swift): ModelProvider, ModelKeyRotation (least recently used ; plus cooldown), ProviderCircuitBreaker, ProviderStatusService, KeychainSecrets. -; Storage is the macOS Keychain, service "ai.t27.os", one item per provider (account = provider name). +; Storage is the macOS Keychain, service "ai.t27.os", one item per key (account = provider name for key 1, +; "#2" .. "#8" for the others, keychain_account_n). ; Never a plain file and never the repo. This spec holds no key and no syscall: the Keychain, the probe and ; the child environment are host calls (keys_host.t27, pty_server.t27); only the decisions live here, so ; they run their tests (a module with @extern imports cannot). Every test uses fake values. @@ -822,12 +823,13 @@ pub fn pick_profile(stored: u32, bad: u32) -> u32 { return next_profile(PR_NONE, stored, bad); } -/// How many keys one provider may hold, numbered from 1 (Free Claude Code: "key 1 first"). -pub const MAX_KEYS : u32 = 3; +/// How many keys one provider may hold, numbered from 1 (Free Claude Code: "key 1 first"). Eight, since the +/// owner stores seven z.ai keys and rotates over all of them (gHashTag/t27#11241). +pub const MAX_KEYS : u32 = 8; /// The Keychain item of key number k (0-based; the panel says "key k+1") of provider p. Key 1 keeps the -/// plain label of keychain_label, so a key stored before numbering stays where it is; keys 2 and 3 add -/// "#2" and "#3". 0 for an unknown provider or a number past MAX_KEYS. +/// plain label of keychain_label, so a key stored before numbering stays where it is; keys 2 to 8 add +/// "#2" to "#8". 0 for an unknown provider or a number past MAX_KEYS. pub fn keychain_label_n(p: u32, k: u32, out: []u8) -> usize { if (p >= PROVIDER_COUNT or k >= MAX_KEYS) { return 0; } var n : usize = keychain_label(p, out); @@ -836,6 +838,43 @@ pub fn keychain_label_n(p: u32, k: u32, out: []u8) -> usize { return put_dec(out, n, @as(u64, k) + 1); } +/// The Keychain account of key number k of provider p: the provider's name for key 1 and "#" for +/// the others ("zai", "zai#2", ...); the service is KEYCHAIN_SERVICE for all. 0 when p or k is out of range. +pub fn keychain_account_n(p: u32, k: u32, out: []u8) -> usize { + if (p >= PROVIDER_COUNT or k >= MAX_KEYS) { return 0; } + var n : usize = provider_name(p, out); + if (k == 0) { return n; } + n = put_bytes(out, n, "#"); + return put_dec(out, n, @as(u64, k) + 1); +} + +/// What a KEYS row says next to the provider's name: nothing for no key, "1 key", "7 keys". +pub fn key_count_text(n: u32, out: []u8) -> usize { + if (n == 0) { return 0; } + const c : usize = put_dec(out, 0, @as(u64, n)); + if (n == 1) { return put_bytes(out, c, " key"); } + return put_bytes(out, c, " keys"); +} + +/// A key travels to the PTY server in an OP_SETKEY frame whose pane byte names the provider; the key number +/// rides in the high nibble of that byte (k * 16 + p), so key 1 is the old frame byte for byte and an +/// older server refuses the others instead of misreading them. +pub const SETKEY_STEP : u32 = 16; + +pub fn setkey_byte(p: u32, k: u32) -> u32 { + return k * SETKEY_STEP + p; +} + +/// The provider and the key number an OP_SETKEY pane byte names; the provider is PROVIDER_COUNT or more when +/// the byte is not one the server knows (the caller refuses it), the number is MAX_KEYS or more when too big. +pub fn setkey_provider(b: u32) -> u32 { + return b % SETKEY_STEP; +} + +pub fn setkey_number(b: u32) -> u32 { + return b / SETKEY_STEP; +} + /// The bad mask from the providers' statuses (st[p]): invalid and circuit-open keys are not offered. pub fn bad_mask(st: []const u32) -> u32 { var m : u32 = 0; @@ -883,7 +922,10 @@ pub const FC_NO_BALANCE : u32 = 2; pub const FC_SUBSCRIPTION : u32 = 3; pub const FC_LOGIN_REJECTED : u32 = 4; pub const FC_OTHER : u32 = 5; -pub const FC_WORDS : str = "no error,rate limited (429),out of balance,subscription disabled,login rejected (401/403),other error"; +/// The provider shed the request: HTTP 529, z.ai code 1305 "service may be temporarily overloaded". +/// Measured 2026-10-20: on the free flash model five of seven z.ai keys answered it at once and one answered 200. +pub const FC_OVERLOADED : u32 = 6; +pub const FC_WORDS : str = "no error,rate limited (429),out of balance,subscription disabled,login rejected (401/403),other error,overloaded (529)"; /// What to do about it. pub const FA_KEEP : u32 = 0; @@ -933,18 +975,50 @@ pub fn has_code(hay: []const u8, code: []const u8) -> bool { /// The failure class of one screen line, by plain byte matching. The two phrases are specific enough alone; /// a bare status number counts only on a line that also says "error" (so "Edited 429 lines" is not a 429). -/// "Insufficient balance" wins over 429: the provider sends both for an empty account. An "API Error" line -/// that matches nothing is FC_OTHER; any other line is FC_NONE. +/// "Insufficient balance" (or its code, 1113) wins over 429: the provider sends both for an empty account. +/// An overload (529, code 1305, or the word "overloaded") wins over 429 too: z.ai sends 1305 under either +/// status, and a throttled key rests longer than a busy one. An "API Error" line that matches nothing is +/// FC_OTHER; any other line is FC_NONE. pub fn classify_line(line: []const u8) -> u32 { if (has_bytes(line, "disabled Claude subscription")) { return FC_SUBSCRIPTION; } if (has_bytes(line, "Insufficient balance")) { return FC_NO_BALANCE; } if (has_bytes(line, "rror") == false) { return FC_NONE; } + if (has_code(line, "1113")) { return FC_NO_BALANCE; } + if (has_code(line, "529") or has_code(line, "1305") or has_bytes(line, "verloaded")) { return FC_OVERLOADED; } if (has_code(line, "429")) { return FC_RATE_LIMITED; } if (has_code(line, "401") or has_code(line, "403")) { return FC_LOGIN_REJECTED; } if (has_bytes(line, "API Error")) { return FC_OTHER; } return FC_NONE; } +/// Does `hay` start with `needle`? +pub fn begins_with(hay: []const u8, needle: []const u8) -> bool { + if (needle.len > hay.len) { return false; } + var i : usize = 0; + while (i < needle.len) { + if (hay[i] != needle[i]) { return false; } + i = i + 1; + } + return true; +} + +/// Is this screen line the error row Claude Code prints, and the final one? classify_line may be run on it +/// only then, never on the body of the transcript: a diff or a quoted line that shows "Insufficient balance" +/// or "API Error: 429" starts with something else (a +, a quote, a word) and would otherwise make the pane +/// leave a healthy provider. Leading blanks are skipped. A row that says "Retrying" is Claude Code's own retry +/// in flight and the turn is still alive; the failover waits for the final row, when nothing is left to lose. +pub fn error_row(line: []const u8) -> bool { + var a : usize = 0; + while (a < line.len and line[a] == 32) { a = a + 1; } + const t : []const u8 = line[a .. line.len]; + if (has_bytes(t, "Retrying")) { return false; } + if (begins_with(t, "API Error")) { return true; } + if (begins_with(t, "Error")) { return true; } + if (begins_with(t, "Request failed")) { return true; } + if (begins_with(t, "Your organization has disabled")) { return true; } + return begins_with(t, "Insufficient balance"); +} + /// The bad mask after a failure of class `class` on profile pr: the profile's provider joins it. A line that /// is no failure (FC_NONE, FC_OTHER) or a pane with no profile changes nothing. pub fn fo_bad(class: u32, pr: u32, bad: u32) -> u32 { @@ -959,13 +1033,13 @@ pub fn fo_bad(class: u32, pr: u32, bad: u32) -> u32 { /// started without a key, which is where the "disabled Claude subscription access" line comes from). /// `stored` and `bad` are the provider masks (bad: providers that failed for this pane or their check), /// `done` the failovers this pane has had, `alt` whether another key of pr's provider is usable now -/// (rotate_pick_except over the keys, the failed one skipped). A throttled key (429) with another key at hand -/// swaps the key and keeps the profile; every other failure leaves the profile for the next one in the chain; -/// when none is left, or the cap is reached, the pane stays as it is. +/// (rotate_pick_except over the keys, the failed one skipped). A throttled (429) or overloaded (529) key with +/// another key at hand swaps the key and keeps the profile; every other failure leaves the profile for the +/// next one in the chain; when none is left, or the cap is reached, the pane stays as it is. pub fn fo_action(class: u32, pr: u32, stored: u32, bad: u32, done: u32, alt: bool) -> u32 { if (class == FC_NONE or class == FC_OTHER) { return FA_KEEP; } if (done >= MAX_FAILOVERS) { return FA_GIVE_UP; } - if (class == FC_RATE_LIMITED and alt and pr < PROFILE_COUNT) { return FA_RESPAWN_KEY; } + if ((class == FC_RATE_LIMITED or class == FC_OVERLOADED) and alt and pr < PROFILE_COUNT) { return FA_RESPAWN_KEY; } if (next_profile(pr, stored, fo_bad(class, pr, bad)) == PR_NONE) { return FA_GIVE_UP; } return FA_RESPAWN_NEXT; } @@ -1004,6 +1078,45 @@ pub fn fo_why(class: u32, pr: u32, act: u32, nxt: u32, done: u32, out: []u8) -> return put_bytes(out, n, ", no profile left in the chain, the pane stays"); } +/// How long a key rests after it caused a failure of class `class` (seconds): a throttled key 30 s, an +/// overloaded one 60 s (the service is busy, not the key, but the same key would meet the same queue), a key +/// with no balance 10 min, a rejected key an hour; a failure that names no key rests none. +pub const OVERLOAD_COOLDOWN_S : u64 = 60; +pub const BALANCE_REST_S : u64 = 600; +pub const REJECT_REST_S : u64 = 3600; + +pub fn key_rest_after(class: u32) -> u64 { + if (class == FC_RATE_LIMITED) { return RATE_COOLDOWN_S; } + if (class == FC_OVERLOADED) { return OVERLOAD_COOLDOWN_S; } + if (class == FC_NO_BALANCE) { return BALANCE_REST_S; } + if (class == FC_LOGIN_REJECTED) { return REJECT_REST_S; } + return 0; +} + +/// The rest_until of a key slot that holds no key: no clock reaches it, so rotation never picks it. +pub const KEY_NEVER : u64 = 99999999999; + +/// The key a new pane starts with: rotate_pick_except, and when every key is resting, the one whose rest ends +/// first (a pane with a resting key is better than a pane with none). Slots that hold no key carry KEY_NEVER +/// and are never taken. Returns n when the provider holds no usable key but the skipped one. +pub fn rotate_pick_soft(n: u32, skip: u32, last_used: []const u64, rest_until: []const u64, now: u64) -> u32 { + const k : u32 = rotate_pick_except(n, skip, last_used, rest_until, now); + if (k != n) { return k; } + var best : u32 = n; + var i : u32 = 0; + while (i < n) { + if (rest_until[i] < KEY_NEVER and i != skip) { + if (best == n) { + best = i; + } else if (rest_until[i] < rest_until[best]) { + best = i; + } + } + i = i + 1; + } + return best; +} + /// z.ai has no cheap GET that proves a key opens the Anthropic-protocol route, so its check is one /// one-token message to the flash model (free); the answer is read by status only. pub fn probe_post(p: u32) -> bool { @@ -1243,13 +1356,220 @@ test a_screen_line_is_sorted_into_a_failure_class { assert classify_line("Your organization has disabled Claude subscription access") == FC_SUBSCRIPTION; assert classify_line("API Error: 401 invalid credentials") == FC_LOGIN_REJECTED; assert classify_line("Request failed with error 403 forbidden") == FC_LOGIN_REJECTED; - assert classify_line("API Error: 500 upstream overloaded") == FC_OTHER; + assert classify_line("API Error: 500 upstream failure") == FC_OTHER; assert classify_line("Edited 429 lines in 3 files") == FC_NONE; assert classify_line("error: read 14290 tokens") == FC_NONE; assert classify_line("Reading specs/t27os/keys.t27") == FC_NONE; assert classify_line("") == FC_NONE; } +test an_overload_is_its_own_class_and_the_balance_code_still_marks_the_key_bad { + // what z.ai answered on the free flash model, 2026-10-20 (code 1305, HTTP 529 or 429) + assert classify_line("API Error: 529 {\"error\":{\"code\":\"1305\",\"message\":\"The service may be temporarily overloaded\"}}") == FC_OVERLOADED; + assert classify_line("API Error: 429 {\"error\":{\"code\":\"1305\",\"message\":\"overloaded, retry\"}}") == FC_OVERLOADED; + assert classify_line("API Error (529 {\"type\":\"error\",\"error\":{\"type\":\"overloaded_error\",\"message\":\"Overloaded\"}})") == FC_OVERLOADED; + assert classify_line("Request failed with error 1305") == FC_OVERLOADED; + assert classify_line("error 529") == FC_OVERLOADED; + // the paid models answered 1113: still out of balance, even next to a 429 or the word overloaded + assert classify_line("API Error: 429 {\"error\":{\"code\":\"1113\",\"message\":\"Insufficient balance\"}}") == FC_NO_BALANCE; + assert classify_line("API Error: 429 {\"error\":{\"code\":\"1113\"") == FC_NO_BALANCE; + assert classify_line("API Error: 429 code 1113 overloaded") == FC_NO_BALANCE; + // a number glued to digits, or a line that is no error, is no overload + assert classify_line("error 5290") == FC_NONE; + assert classify_line("error 13050") == FC_NONE; + assert classify_line("Edited 529 lines in 3 files") == FC_NONE; + assert classify_line("the service was overloaded earlier") == FC_NONE; + assert classify_line("API Error: 429 rate limit reached") == FC_RATE_LIMITED; +} + +test only_an_error_row_is_ever_classified_never_the_body_of_the_transcript { + assert error_row("API Error: 529 {\"error\":{\"code\":\"1305\"}}"); + assert error_row(" API Error: 429 Insufficient balance"); + assert error_row("Error: 401 invalid credentials"); + assert error_row("Request failed with error 403"); + assert error_row("Your organization has disabled Claude subscription access"); + assert error_row("Insufficient balance or no resource package"); + // a diff or a quoted line that merely shows the phrases + assert error_row("+ assert classify_line(\"API Error: 429 Insufficient balance\") == FC_NO_BALANCE;") == false; + assert error_row("The log said API Error: 529 yesterday") == false; + assert error_row(" - Error handling for 429 added") == false; + assert error_row("Reading specs/t27os/keys.t27") == false; + assert error_row("") == false; + assert error_row(" ") == false; + // Claude Code's own retry in flight: the turn is alive, the failover waits + assert error_row("API Error (529 overloaded_error) Retrying in 4 seconds (attempt 2/10)") == false; + assert begins_with("abc", "ab"); + assert begins_with("abc", "abcd") == false; + assert begins_with("abc", ""); + assert begins_with("", "a") == false; +} + +test an_overloaded_key_with_a_spare_swaps_the_key_and_keeps_the_profile { + assert fo_action(FC_OVERLOADED, PR_ZAI, 57, 0, 0, true) == FA_RESPAWN_KEY; + assert fo_target(FC_OVERLOADED, PR_ZAI, 57, 0, 0, true) == PR_ZAI; + // no other key usable now: the pane leaves the profile, as for a throttled key with no spare + assert fo_action(FC_OVERLOADED, PR_ZAI, 57, 0, 0, false) == FA_RESPAWN_NEXT; + assert fo_target(FC_OVERLOADED, PR_ZAI, 57, 0, 0, false) == PR_ANTHROPIC; + assert fo_action(FC_OVERLOADED, PR_ZAI, 16, 0, 0, false) == FA_GIVE_UP; + assert fo_action(FC_OVERLOADED, PR_NONE, 57, 0, 0, true) == FA_RESPAWN_NEXT; + assert fo_action(FC_OVERLOADED, PR_ZAI, 57, 0, MAX_FAILOVERS, true) == FA_GIVE_UP; + var done : u32 = 0; + while (done < 40 and fo_action(FC_OVERLOADED, PR_ZAI, 57, 0, done, true) == FA_RESPAWN_KEY) { + done = done + 1; + } + assert done == MAX_FAILOVERS; +} + +test the_overload_why_line_names_the_class_and_never_a_key { + var o : [128]u8 = undefined; + const n : usize = fo_why(FC_OVERLOADED, PR_ZAI, FA_RESPAWN_KEY, PR_ZAI, 1, o[0..128]); + assert has_bytes(o[0..n], "claude: zai overloaded (529), same profile with the next key"); + const m : usize = fo_why(FC_OVERLOADED, PR_ZAI, FA_RESPAWN_NEXT, PR_ANTHROPIC, 1, o[0..128]); + assert has_bytes(o[0..m], "zai overloaded (529), respawn on anthropic-api"); + assert csv_fields(FC_WORDS) == FC_OVERLOADED + 1; +} + +test a_failed_key_rests_by_the_kind_of_failure { + assert key_rest_after(FC_RATE_LIMITED) == RATE_COOLDOWN_S; + assert key_rest_after(FC_OVERLOADED) == OVERLOAD_COOLDOWN_S; + assert key_rest_after(FC_NO_BALANCE) == BALANCE_REST_S; + assert key_rest_after(FC_LOGIN_REJECTED) == REJECT_REST_S; + assert key_rest_after(FC_NONE) == 0; + assert key_rest_after(FC_OTHER) == 0; + assert key_rest_after(FC_SUBSCRIPTION) == 0; + assert key_rest_after(FC_OVERLOADED) > key_rest_after(FC_RATE_LIMITED); + assert key_rest_after(FC_NO_BALANCE) > key_rest_after(FC_OVERLOADED); + assert key_rest_after(FC_LOGIN_REJECTED) > key_rest_after(FC_NO_BALANCE); +} + +test a_pane_takes_a_resting_key_only_when_no_key_is_free { + var used : [4]u64 = [50, 10, 30, 0]; + var rest : [4]u64 = [0, 0, 0, KEY_NEVER]; + // slot 3 holds no key: never taken; the key used longest ago wins + assert rotate_pick_soft(4, 4, used[0..4], rest[0..4], 100) == 1; + assert rotate_pick_soft(4, 1, used[0..4], rest[0..4], 100) == 2; + // every present key rests: the one that is free first + rest[0] = 300; + rest[1] = 200; + rest[2] = 250; + assert rotate_pick_except(4, 4, used[0..4], rest[0..4], 100) == 4; + assert rotate_pick_soft(4, 4, used[0..4], rest[0..4], 100) == 1; + assert rotate_pick_soft(4, 1, used[0..4], rest[0..4], 100) == 2; + // only the skipped key is left: none + rest[0] = KEY_NEVER; + rest[2] = KEY_NEVER; + assert rotate_pick_soft(4, 1, used[0..4], rest[0..4], 100) == 4; + assert rotate_pick_soft(4, 4, used[0..4], rest[0..4], 100) == 1; + rest[1] = KEY_NEVER; + assert rotate_pick_soft(4, 4, used[0..4], rest[0..4], 100) == 4; +} + +test eight_keys_rotate_so_each_is_used_once_before_any_is_used_twice { + var used : [8]u64 = [0, 0, 0, 0, 0, 0, 0, 0]; + var rest : [8]u64 = [0, 0, 0, 0, 0, 0, 0, KEY_NEVER]; + var seen : u32 = 0; + var t : u64 = 1; + var round : u32 = 0; + while (round < 7) { + const k : u32 = rotate_pick(8, used[0..8], rest[0..8], t); + assert k < 7; + assert (seen >> k) % 2 == 0; + seen = seen + (1 << k); + used[k] = t; + t = t + 1; + round = round + 1; + } + assert seen == 127; + assert rotate_pick(8, used[0..8], rest[0..8], t) == 0; +} + +test the_account_of_key_k_is_the_provider_name_with_a_number { + var a : [32]u8 = undefined; + assert keychain_account_n(P_ZAI, 0, a[0..32]) == 3; + assert a[0] == 122 and a[2] == 105; + assert keychain_account_n(P_ZAI, 1, a[0..32]) == 5; + assert a[3] == 35 and a[4] == 50; + assert keychain_account_n(P_ZAI, 6, a[0..32]) == 5; + assert a[3] == 35 and a[4] == 55; + assert keychain_account_n(P_ANTHROPIC, 7, a[0..32]) == 11; + assert a[9] == 35 and a[10] == 56; + assert keychain_account_n(P_ZAI, MAX_KEYS, a[0..32]) == 0; + assert keychain_account_n(PROVIDER_COUNT, 0, a[0..32]) == 0; + assert MAX_KEYS == 8; + var l : [48]u8 = undefined; + assert keychain_label_n(P_ZAI, 7, l[0..48]) == 15; + assert l[13] == 35 and l[14] == 56; +} + +test each_agent_cli_gets_the_variable_of_every_provider_it_reads_and_no_other { + // The pane's environment is built from AGENT_PROVIDERS and PROVIDER_ENVS only (pty_server inject_keys), one + // variable per provider the cell reads, carrying the key the rotation picked. These are the pairs. + var o : [40]u8 = undefined; + var n : usize = provider_env(P_OPENAI, o[0..40]); + assert agent_uses(1, P_OPENAI) and has_bytes(o[0..n], "OPENAI_API_KEY") and n == 14; + n = provider_env(P_GEMINI, o[0..40]); + assert agent_uses(2, P_GEMINI) and has_bytes(o[0..n], "GEMINI_API_KEY") and n == 14; + n = provider_env(P_ZAI, o[0..40]); + assert agent_uses(5, P_ZAI) and has_bytes(o[0..n], "ZAI_API_KEY") and n == 11; + n = provider_env(P_CURSOR, o[0..40]); + assert agent_uses(6, P_CURSOR) and has_bytes(o[0..n], "CURSOR_API_KEY") and n == 14; + n = provider_env(P_GITHUB, o[0..40]); + assert agent_uses(8, P_GITHUB) and has_bytes(o[0..n], "GITHUB_TOKEN") and n == 12; + n = provider_env(P_OPENROUTER, o[0..40]); + assert agent_uses(3, P_OPENROUTER) and agent_uses(7, P_OPENROUTER) and has_bytes(o[0..n], "OPENROUTER_API_KEY"); + // a cell never gets a variable of a provider it does not read + assert agent_uses(1, P_ZAI) == false; + assert agent_uses(2, P_OPENAI) == false; + assert agent_uses(8, P_ANTHROPIC) == false; + assert agent_uses(6, P_GITHUB) == false; + // Claude Code is the one cell with profile variables, and its z.ai profile reads the z.ai provider's keys + assert profile_agent(0) and profile_agent(5) == false; + assert PROFILE_PROVIDER[PR_ZAI] == P_ZAI; + // every provider a cell reads has a variable name + var a : u32 = 0; + while (a < AGENT_COUNT) { + var p : u32 = 0; + while (p < PROVIDER_COUNT) { + if (agent_uses(a, p)) { assert provider_env(p, o[0..40]) > 0; } + p = p + 1; + } + a = a + 1; + } +} + +test a_keys_row_says_how_many_keys_the_provider_holds { + var o : [24]u8 = undefined; + assert key_count_text(0, o[0..24]) == 0; + assert key_count_text(1, o[0..24]) == 5; + assert o[0] == 49 and o[2] == 107 and o[4] == 121; + assert key_count_text(7, o[0..24]) == 6; + assert o[0] == 55 and o[5] == 115; + assert key_count_text(MAX_KEYS, o[0..24]) == 6; + assert o[0] == 56; +} + +test the_key_number_rides_in_the_high_nibble_of_the_setkey_byte { + var p : u32 = 0; + while (p < PROVIDER_COUNT) { + var k : u32 = 0; + while (k < MAX_KEYS) { + const b : u32 = setkey_byte(p, k); + assert b < 256; + assert setkey_provider(b) == p; + assert setkey_number(b) == k; + k = k + 1; + } + p = p + 1; + } + // key 1 is the old frame byte + assert setkey_byte(P_ZAI, 0) == P_ZAI; + assert setkey_number(P_GONKA) == 0; + // a byte from a newer client that this table cannot hold reads as unknown + assert setkey_provider(setkey_byte(11, 0)) >= PROVIDER_COUNT; + assert PROVIDER_COUNT <= SETKEY_STEP; + assert MAX_KEYS * SETKEY_STEP <= 256; +} + test a_failure_marks_the_provider_bad_for_this_pane { assert fo_bad(FC_NO_BALANCE, PR_ZAI, 0) == 16; assert fo_bad(FC_LOGIN_REJECTED, PR_ANTHROPIC, 16) == 17; diff --git a/specs/t27os/keys_host.t27 b/specs/t27os/keys_host.t27 index d7923ee5fc..0f90dec80f 100644 --- a/specs/t27os/keys_host.t27 +++ b/specs/t27os/keys_host.t27 @@ -1,12 +1,14 @@ // SPDX-License-Identifier: Apache-2.0 // T27 OS keys_host: where the key manager (keys.t27) touches the machine. Three jobs, no decisions: -// - the macOS Keychain: kc_put / kc_get / kc_del, one generic-password item per provider under the -// service "ai.t27.os" (SecItemAdd, SecItemCopyMatching, SecItemUpdate, SecItemDelete). A key is never -// written to a file and never reaches the repo. +// - the macOS Keychain: kc_put_n / kc_get_n / kc_del_n, one generic-password item per key (up to MAX_KEYS a +// provider; account = provider name, then "#2" .. "#8") under the service "ai.t27.os" +// (SecItemAdd, SecItemCopyMatching, SecItemUpdate, SecItemDelete); kc_put / kc_get / kc_del are key 1. +// A key is never written to a file and never reaches the repo. // - the check button: probe() sends the cheapest authenticated request of a provider (a GET; for z.ai one one-token POST) (table: keys.t27) and // hands back the HTTP status; keys.t27 status_after_probe decides what it means. The key travels in a // request header, never in the URL. -// - push_keys(): hand every stored key to the PTY server (OP_SETKEY) so a pane it spawns is logged in. +// - push_keys(): hand every stored key, with its number, to the PTY server (OP_SETKEY) so a pane it spawns +// is logged in and the server can rotate over the keys of a provider. // // The kSec* constants are global CFStringRef variables, not functions, so an @extern call cannot read them: // dlsym finds the variable's address and sys_peek copies the pointer out. Specs holding @extern cannot run @@ -16,9 +18,11 @@ module T27osKeysHost; use t27os::keys::PROVIDER_COUNT; +use t27os::keys::MAX_KEYS; use t27os::keys::KEYCHAIN_SERVICE; use t27os::keys::PROBE_TIMEOUT_S; -use t27os::keys::provider_name; +use t27os::keys::keychain_account_n; +use t27os::keys::setkey_byte; use t27os::keys::probe_url; use t27os::keys::probe_has; use t27os::keys::probe_header_name; @@ -107,13 +111,14 @@ fn set_text(d: u64, key_name: []const u8, text: []const u8) -> void { cf_release(s); } -/// The query that names provider p's item: generic password, service "ai.t27.os", account = provider name. -fn item_query(p: u32) -> u64 { +/// The query that names key k of provider p (k = 0 is key 1): generic password, service "ai.t27.os", account +/// = the provider name, then "#2" .. "#8" for the others (keys.t27 keychain_account_n). +fn item_query(p: u32, k: u32) -> u64 { const d : u64 = new_dict(); cf_dict_set(d, global_ref("kSecClass"), global_ref("kSecClassGenericPassword")); set_text(d, "kSecAttrService", KEYCHAIN_SERVICE); - var nm : [24]u8 = undefined; - const n : usize = provider_name(p, nm[0..24]); + var nm : [32]u8 = undefined; + const n : usize = keychain_account_n(p, k, nm[0..32]); set_text(d, "kSecAttrAccount", nm[0..n]); return d; } @@ -124,16 +129,16 @@ fn set_data(d: u64, key: []const u8) -> void { cf_release(data); } -/// Store provider p's key (replacing an earlier one). True when the Keychain accepted it. -pub fn kc_put(p: u32, key: []const u8) -> bool { - if (p >= PROVIDER_COUNT) { return false; } +/// Store key k (0-based) of provider p, replacing an earlier one. True when the Keychain accepted it. +pub fn kc_put_n(p: u32, k: u32, key: []const u8) -> bool { + if (p >= PROVIDER_COUNT or k >= MAX_KEYS) { return false; } if (key.len == 0) { return false; } - const a : u64 = item_query(p); + const a : u64 = item_query(p, k); set_data(a, key); var st : i32 = sec_add(a, 0); cf_release(a); if (st == SEC_DUPLICATE) { - const q : u64 = item_query(p); + const q : u64 = item_query(p, k); const u : u64 = new_dict(); set_data(u, key); st = sec_update(q, u); @@ -143,10 +148,15 @@ pub fn kc_put(p: u32, key: []const u8) -> bool { return st == 0; } -/// Read provider p's key into out; its length, 0 when none is stored. -pub fn kc_get(p: u32, out: []u8) -> usize { - if (p >= PROVIDER_COUNT) { return 0; } - const q : u64 = item_query(p); +/// Store provider p's first key. +pub fn kc_put(p: u32, key: []const u8) -> bool { + return kc_put_n(p, 0, key); +} + +/// Read key k (0-based) of provider p into out; its length, 0 when none is stored. +pub fn kc_get_n(p: u32, k: u32, out: []u8) -> usize { + if (p >= PROVIDER_COUNT or k >= MAX_KEYS) { return 0; } + const q : u64 = item_query(p, k); cf_dict_set(q, global_ref("kSecReturnData"), global_ref("kCFBooleanTrue")); cf_dict_set(q, global_ref("kSecMatchLimit"), global_ref("kSecMatchLimitOne")); var res : [8]u8 = undefined; @@ -162,19 +172,63 @@ pub fn kc_get(p: u32, out: []u8) -> usize { return @as(usize, n); } -/// Delete provider p's item. True when it is gone (also when none was there). -pub fn kc_del(p: u32) -> bool { - if (p >= PROVIDER_COUNT) { return false; } - const q : u64 = item_query(p); +/// Read provider p's first key into out. +pub fn kc_get(p: u32, out: []u8) -> usize { + return kc_get_n(p, 0, out); +} + +/// Delete key k (0-based) of provider p. True when it is gone (also when none was there). +pub fn kc_del_n(p: u32, k: u32) -> bool { + if (p >= PROVIDER_COUNT or k >= MAX_KEYS) { return false; } + const q : u64 = item_query(p, k); const st : i32 = sec_delete(q); cf_release(q); return st == 0 or st == -25300; } -/// True when the Keychain holds a key for provider p (reads it, then drops the copy). +/// Delete provider p's first key. +pub fn kc_del(p: u32) -> bool { + return kc_del_n(p, 0); +} + +/// Delete every key of provider p (a KEYS row's DELETE removes the provider's whole set). True when all are gone. +pub fn kc_del_all(p: u32) -> bool { + var ok : bool = true; + var k : u32 = 0; + while (k < MAX_KEYS) { + if (kc_del_n(p, k) == false) { ok = false; } + k = k + 1; + } + return ok; +} + +/// True when the Keychain holds a first key for provider p (reads it, then drops the copy). pub fn kc_has(p: u32) -> bool { var b : [256]u8 = undefined; - return kc_get(p, b[0..256]) > 0; + const n : usize = kc_get(p, b[0..256]); + var i : usize = 0; + while (i < 256) { + b[i] = 0; + i = i + 1; + } + return n > 0; +} + +/// How many of the provider's MAX_KEYS accounts hold a key (a gap in the numbers does not stop the count). +pub fn kc_count(p: u32) -> u32 { + var b : [256]u8 = undefined; + var c : u32 = 0; + var k : u32 = 0; + while (k < MAX_KEYS) { + if (kc_get_n(p, k, b[0..256]) > 0) { c = c + 1; } + k = k + 1; + } + var i : usize = 0; + while (i < 256) { + b[i] = 0; + i = i + 1; + } + return c; } fn add_header(req: u64, name: []const u8, value: []const u8) -> void { @@ -230,20 +284,32 @@ pub fn probe(p: u32, key: []const u8) -> u32 { return @as(u32, @intCast(msg0(r, sel("statusCode")))); } -/// Hand every stored key to the PTY server on `fd`; the count pushed. Called on connect and before a spawn, -/// because a server that restarted has forgotten them. +/// Hand every stored key to the PTY server on `fd`, each with its number; the count pushed. Every numbered +/// account of every provider is read, up to MAX_KEYS, and the number rides in the OP_SETKEY pane byte +/// (keys.t27 setkey_byte), so the server holds all of them and rotates over them. Called on connect, when +/// the KEYS tab opens and before a spawn, because a server that restarted has forgotten them. The copy of +/// each key in this stack frame is wiped when the call ends. pub fn push_keys(fd: i32) -> u32 { var key : [256]u8 = undefined; var reply : [MAX_FRAME]u8 = undefined; var pushed : u32 = 0; var p : u32 = 0; while (p < PROVIDER_COUNT) { - const n : usize = kc_get(p, key[0..256]); - if (n > 0) { - if (client_set_key(fd, p, key[0..n], reply[0..MAX_FRAME]) > 0) { pushed = pushed + 1; } + var k : u32 = 0; + while (k < MAX_KEYS) { + const n : usize = kc_get_n(p, k, key[0..256]); + if (n > 0) { + if (client_set_key(fd, setkey_byte(p, k), key[0..n], reply[0..MAX_FRAME]) > 0) { pushed = pushed + 1; } + } + k = k + 1; } p = p + 1; } + var i : usize = 0; + while (i < 256) { + key[i] = 0; + i = i + 1; + } return pushed; } diff --git a/specs/t27os/pty_logic.t27 b/specs/t27os/pty_logic.t27 index c3523485e7..8e8ca12b42 100644 --- a/specs/t27os/pty_logic.t27 +++ b/specs/t27os/pty_logic.t27 @@ -88,6 +88,45 @@ use t27os::pty_proto::ring_copy; use t27os::pty_proto::put_list_entry; use t27os::pty_proto::spawn_ok; use t27os::pty_proto::socket_path; +use t27os::keys::PROVIDER_COUNT; +use t27os::keys::MAX_KEYS; +use t27os::keys::KEY_NEVER; +use t27os::keys::PROFILE_COUNT; +use t27os::keys::PROFILE_PROVIDER; +use t27os::keys::PR_NONE; +use t27os::keys::PR_ZAI; +use t27os::keys::PR_ANTHROPIC; +use t27os::keys::MAX_FAILOVERS; +use t27os::keys::agent_uses; +use t27os::keys::spawn_mask; +use t27os::keys::P_ZAI; +use t27os::keys::P_ANTHROPIC; +use t27os::keys::P_OPENAI; +use t27os::keys::FC_OTHER; +use t27os::keys::FC_LOGIN_REJECTED; +use t27os::keys::profile_agent; +use t27os::keys::pick_profile; +use t27os::keys::rotate_pick_soft; +use t27os::keys::rotate_pick_except; +use t27os::keys::key_rest_after; +use t27os::keys::fo_action; +use t27os::keys::fo_target; +use t27os::keys::fo_bad; +use t27os::keys::fo_why; +use t27os::keys::classify_line; +use t27os::keys::error_row; +use t27os::keys::has_bytes; +use t27os::keys::FC_NONE; +use t27os::keys::FC_RATE_LIMITED; +use t27os::keys::FC_NO_BALANCE; +use t27os::keys::FC_SUBSCRIPTION; +use t27os::keys::FC_OVERLOADED; +use t27os::keys::FA_KEEP; +use t27os::keys::FA_RESPAWN_NEXT; +use t27os::keys::FA_RESPAWN_KEY; +use t27os::keys::FA_GIVE_UP; +use t27os::keys::setkey_provider; +use t27os::keys::setkey_number; use t27os::editors::TERM_COUNT; use t27os::editors::TERM_NONE; use t27os::editors::TERM_PATH_MAX; @@ -125,7 +164,42 @@ pub const G_CFD : usize = 232; pub const G_CLEN : usize = 240; /// Each pane's last RESIZE, two panes per word (pty_proto ws_pack); 8 words cover MAX_PANES. pub const G_WINSZ : usize = 248; -pub const ST_SIZE : usize = 256; +/// The table grew from 256 to 512 words for the key rotation (gHashTag/t27#11241): per key slot of every +/// provider (PROVIDER_COUNT * MAX_KEYS = 72) when it was last used (ms) and until when it rests (s, KEY_NEVER +/// for a slot that holds no key), then PK words per pane: the key numbers the pane holds, its profile, its bad +/// provider mask, its failover count, the key to skip at the next launch, the time until which its screen +/// is not read (after a give-up) and its restore state (W_*). Key slot (p, k) is word p * MAX_KEYS + k. +/// G_START is the millisecond the server started at (a restored pane waits for keys from then on). +pub const G_START : usize = 227; +pub const G_KLAST : usize = 256; +pub const G_KREST : usize = 328; +pub const G_PANEK : usize = 400; +pub const KEY_SLOTS : usize = 72; +pub const PK : usize = 7; +pub const PK_KEYS : usize = 0; +pub const PK_PROF : usize = 1; +pub const PK_BAD : usize = 2; +pub const PK_DONE : usize = 3; +pub const PK_SKIP : usize = 4; +pub const PK_STOP : usize = 5; +pub const PK_WAIT : usize = 6; +/// PK_PROF of a pane that has not chosen a profile yet. +pub const PK_UNSET : u64 = 254; +/// PK_WAIT: 0 nothing; 1 + plan a restored pane waiting for a key of its provider (its launch is deferred); +/// W_KEYLESS a restored pane that started with no key (its one respawn is due when a key arrives); W_DONE +/// that respawn was made. +pub const W_NONE : u64 = 0; +pub const W_KEYLESS : u64 = 8; +pub const W_DONE : u64 = 9; +/// How long a restored pane waits for the window's keys before it starts as it always did. +pub const RESTORE_WAIT_MS : u64 = 20000; +/// A pane's screen is not read for this long after it starts: a resumed Claude Code session replays its +/// transcript, which may hold an earlier error row. +pub const SCAN_GRACE_MS : u64 = 10000; +/// After a give-up the pane's screen is read again only after this long, so one error row on screen is not +/// answered with a line in the scrollback at every redraw. +pub const GIVEUP_QUIET_MS : u64 = 60000; +pub const ST_SIZE : usize = 512; pub const MAX_CONNS : usize = 8; pub const SID_CAP : usize = 64; @@ -145,9 +219,17 @@ pub const O_BIN : usize = 92400; pub const MODEL_CAP : usize = 64; pub const O_MODEL : usize = 97280; pub const O_RING : usize = 98304; +/// The key table: KEY_SLOTS slots of KEY_CAP bytes (a length byte, the key, a NUL), slot (p, k) at +/// O_KEYS + (p * MAX_KEYS + k) * KEY_CAP (key_slot). Then each pane's unfinished screen line (a length byte +/// and up to CARRY_CAP bytes, CARRY_SLOT apart) and the scratch where a chunk's plain text is built. pub const O_KEYS : usize = 1146880; pub const KEY_CAP : usize = 256; -pub const MEM_SIZE : usize = 1149184; +pub const O_CARRY : usize = 1165312; +pub const CARRY_CAP : usize = 128; +pub const CARRY_SLOT : usize = 132; +pub const O_SCAN : usize = 1167424; +pub const SCAN_CAP : usize = 4232; +pub const MEM_SIZE : usize = 1171712; pub const TICK_MS : i32 = 500; pub const SIGHUP_RING : u64 = 65536; @@ -357,6 +439,419 @@ pub fn c_path(out: []u8, src: []const u8) -> usize { return n; } +// ---- key rotation (gHashTag/t27#11241): the server's key table, the keys a pane holds, the failover ---- +// The server holds up to MAX_KEYS keys per provider in memory (never on disk) with when each was last used +// and until when it rests. A pane spawn picks its key with rotate_pick_soft (the key used longest ago that is +// not resting; when all rest, the one whose rest ends first). The pane keeps the key NUMBER, never the key. +// When the pane's screen shows a failure the key rests and the pane is respawned by keys.t27 fo_action, on the +// next key of the same provider (rotate_pick_except, the failed key skipped) or on the next profile. + +fn kidx(p: u32, k: u32) -> usize { + return @as(usize, p) * @as(usize, MAX_KEYS) + @as(usize, k); +} + +fn pow16(n: u32) -> u64 { + var m : u64 = 1; + var j : u32 = 0; + while (j < n) { + m = m * 16; + j = j + 1; + } + return m; +} + +/// The offset in the arena of key slot (p, k): a length byte, the key, a NUL (OP_SETKEY fills it). +pub fn key_slot(p: u32, k: u32) -> usize { + return O_KEYS + kidx(p, k) * KEY_CAP; +} + +/// A pane that has chosen nothing: no keys, no profile, no failures, nothing to skip, nothing waiting. +pub fn pane_keys_reset(st: []u64, i: usize) -> void { + const pk : usize = G_PANEK + i * PK; + st[pk + PK_KEYS] = 0; + st[pk + PK_PROF] = PK_UNSET; + st[pk + PK_BAD] = 0; + st[pk + PK_DONE] = 0; + st[pk + PK_SKIP] = 0; + st[pk + PK_STOP] = 0; + st[pk + PK_WAIT] = W_NONE; +} + +/// The empty table: every slot rests until KEY_NEVER (holds no key), every pane is reset. +pub fn keys_init(st: []u64) -> void { + var s : usize = 0; + while (s < KEY_SLOTS) { + st[G_KLAST + s] = 0; + st[G_KREST + s] = KEY_NEVER; + s = s + 1; + } + var i : usize = 0; + while (i < 16) { + pane_keys_reset(st, i); + i = i + 1; + } +} + +pub fn key_present(st: []const u64, p: u32, k: u32) -> bool { + return st[G_KREST + kidx(p, k)] < KEY_NEVER; +} + +/// How many slots of provider p hold a key. +pub fn key_count(st: []const u64, p: u32) -> u32 { + var c : u32 = 0; + var k : u32 = 0; + while (k < MAX_KEYS) { + if (key_present(st, p, k)) { c = c + 1; } + k = k + 1; + } + return c; +} + +/// The providers that hold at least one key, as a bit mask. +pub fn keys_stored(st: []const u64) -> u32 { + var m : u32 = 0; + var p : u32 = 0; + while (p < PROVIDER_COUNT) { + if (key_count(st, p) > 0) { m = m + (1 << p); } + p = p + 1; + } + return m; +} + +/// OP_SETKEY filled (present) or emptied slot (p, k). `same`: the bytes are those already held, so the key keeps +/// its last-use and rest times (a window that sends every key again must not end a cooldown). +pub fn key_put(st: []u64, p: u32, k: u32, present: bool, same: bool) -> void { + const ix : usize = kidx(p, k); + if (present == false) { + st[G_KLAST + ix] = 0; + st[G_KREST + ix] = KEY_NEVER; + return; + } + if (same and st[G_KREST + ix] < KEY_NEVER) { return; } + st[G_KLAST + ix] = 0; + st[G_KREST + ix] = 0; +} + +/// The key of provider p a new pane takes (MAX_KEYS when it holds none); `skip` is a key not to take. +pub fn key_pick(st: []const u64, p: u32, skip: u32, now_ms: u64) -> u32 { + const at : usize = kidx(p, 0); + return rotate_pick_soft(MAX_KEYS, skip, st[G_KLAST + at .. G_KLAST + at + 8], st[G_KREST + at .. G_KREST + at + 8], now_ms / 1000); +} + +/// A key of provider p that is usable right now and is not `skip` (MAX_KEYS when none): the test for a swap. +pub fn key_pick_free(st: []const u64, p: u32, skip: u32, now_ms: u64) -> u32 { + const at : usize = kidx(p, 0); + return rotate_pick_except(MAX_KEYS, skip, st[G_KLAST + at .. G_KLAST + at + 8], st[G_KREST + at .. G_KREST + at + 8], now_ms / 1000); +} + +pub fn key_used(st: []u64, p: u32, k: u32, now_ms: u64) -> void { + st[G_KLAST + kidx(p, k)] = now_ms; +} + +/// Key (p, k) rests until `until_s`; a longer rest it already has stays, and a slot that holds no key stays empty. +pub fn key_rest(st: []u64, p: u32, k: u32, until_s: u64) -> void { + const ix : usize = G_KREST + kidx(p, k); + if (st[ix] >= KEY_NEVER) { return; } + if (until_s > st[ix]) { st[ix] = until_s; } +} + +/// The key number + 1 pane i holds for provider p (0 = none): one nibble per provider. +pub fn pane_key_get(st: []const u64, i: usize, p: u32) -> u32 { + return @as(u32, @intCast(st[G_PANEK + i * PK + PK_KEYS] / pow16(p) % 16)); +} + +pub fn pane_key_set(st: []u64, i: usize, p: u32, v: u32) -> void { + const at : usize = G_PANEK + i * PK + PK_KEYS; + const m : u64 = pow16(p); + st[at] = st[at] - @as(u64, pane_key_get(st, i, p)) * m + @as(u64, v) * m; +} + +/// The key number + 1 a LIST shows for pane i (0 = none): for the claude cell the key of its profile's provider, +/// for every other agent the first provider it reads that it holds a key for. +pub fn pane_key_shown(st: []const u64, i: usize, a: u32) -> u32 { + if (profile_agent(a)) { + const pr : u64 = st[G_PANEK + i * PK + PK_PROF]; + if (pr >= @as(u64, PROFILE_COUNT)) { return 0; } + return pane_key_get(st, i, PROFILE_PROVIDER[@as(u32, @intCast(pr))]); + } + var p : u32 = 0; + while (p < PROVIDER_COUNT) { + const v : u32 = pane_key_get(st, i, p); + if (v > 0) { return v; } + p = p + 1; + } + return 0; +} + +/// The flags byte of a LIST entry with the pane's key number + 1 in its high nibble (0 to 8). Bits 0 to 2 are +/// list_flags as before, so a client that reads only those is not disturbed. +pub fn list_flags_key(live: bool, w: u32, key1: u32) -> u32 { + return list_flags(live, w) + 16 * key1; +} + +/// Choose the keys of pane i about to launch (agent a, `now_ms`): fills the pane's key numbers and profile +/// from the table, marks each picked key used, clears the skip. The claude cell takes the profile the pane +/// asked for (a failover set it) or pick_profile over the providers that hold a key and have not failed for +/// this pane, then the key of that provider by rotation, skipping the key that just failed. Every other agent +/// takes a key of each provider it reads, by rotation. Nothing here holds a key value. +pub fn plan_pane_keys(st: []u64, i: usize, a: u32, now_ms: u64) -> void { + const pk : usize = G_PANEK + i * PK; + st[pk + PK_KEYS] = 0; + st[pk + PK_STOP] = 0; + if (profile_agent(a)) { + var pr : u32 = PR_NONE; + if (st[pk + PK_PROF] < @as(u64, PROFILE_COUNT)) { + pr = @as(u32, @intCast(st[pk + PK_PROF])); + } else { + pr = pick_profile(keys_stored(st), @as(u32, @intCast(st[pk + PK_BAD]))); + } + var skip : u32 = MAX_KEYS; + if (st[pk + PK_SKIP] > 0) { skip = @as(u32, @intCast(st[pk + PK_SKIP])) - 1; } + if (pr < PROFILE_COUNT) { + const prov : u32 = PROFILE_PROVIDER[pr]; + const k : u32 = key_pick(st, prov, skip, now_ms); + if (k < MAX_KEYS) { + key_used(st, prov, k, now_ms); + pane_key_set(st, i, prov, k + 1); + } else { + pr = PR_NONE; + } + } + st[pk + PK_PROF] = @as(u64, pr); + st[pk + PK_SKIP] = 0; + return; + } + var p : u32 = 0; + while (p < PROVIDER_COUNT) { + if (agent_uses(a, p) and key_count(st, p) > 0) { + const k : u32 = key_pick(st, p, MAX_KEYS, now_ms); + if (k < MAX_KEYS) { + key_used(st, p, k, now_ms); + pane_key_set(st, i, p, k + 1); + } + } + p = p + 1; + } +} + +/// The failover for pane i whose screen showed failure `class` at `now_ms`. The key the pane used rests by the +/// kind of failure (keys.t27 key_rest_after), for every pane, not just this one. fo_action decides; on a respawn +/// the pane's profile, bad mask, failover count and key to skip are set for plan_pane_keys, and the why line +/// (keys.t27 fo_why, 128 bytes are enough) goes to `why` with its length in wl[0]. After a give-up the screen +/// is not read again for GIVEUP_QUIET_MS. Returns the action; FA_RESPAWN_* means "stop the process, launch again". +pub fn plan_failover(st: []u64, i: usize, class: u32, now_ms: u64, why: []u8, wl: []u64) -> u32 { + const pk : usize = G_PANEK + i * PK; + var pr : u32 = PR_NONE; + if (st[pk + PK_PROF] < @as(u64, PROFILE_COUNT)) { pr = @as(u32, @intCast(st[pk + PK_PROF])); } + const stored : u32 = keys_stored(st); + const bad : u32 = @as(u32, @intCast(st[pk + PK_BAD])); + const done : u32 = @as(u32, @intCast(st[pk + PK_DONE])); + var cur : u32 = MAX_KEYS; + var alt : bool = false; + if (pr < PROFILE_COUNT) { + const prov : u32 = PROFILE_PROVIDER[pr]; + const held : u32 = pane_key_get(st, i, prov); + if (held > 0) { cur = held - 1; } + const rest : u64 = key_rest_after(class); + if (cur < MAX_KEYS and rest > 0) { key_rest(st, prov, cur, now_ms / 1000 + rest); } + alt = key_pick_free(st, prov, cur, now_ms) != MAX_KEYS; + } + const act : u32 = fo_action(class, pr, stored, bad, done, alt); + var nxt : u32 = PR_NONE; + if (act == FA_RESPAWN_KEY) { + nxt = pr; + st[pk + PK_SKIP] = @as(u64, cur) + 1; + } + if (act == FA_RESPAWN_NEXT) { + nxt = fo_target(class, pr, stored, bad, done, alt); + st[pk + PK_BAD] = @as(u64, fo_bad(class, pr, bad)); + } + if (act == FA_RESPAWN_KEY or act == FA_RESPAWN_NEXT) { + st[pk + PK_PROF] = @as(u64, nxt); + st[pk + PK_DONE] = @as(u64, done) + 1; + } + if (act == FA_GIVE_UP) { st[pk + PK_STOP] = now_ms + GIVEUP_QUIET_MS; } + wl[0] = @as(u64, fo_why(class, pr, act, nxt, done, why)); + return act; +} + +// ---- reading a pane's screen: plain text lines out of terminal bytes ---- + +fn put_break(out: []u8, n: usize) -> usize { + if (n == 0) { return 0; } + if (out[n - 1] == 10) { return n; } + out[n] = 10; + return n + 1; +} + +/// Plain text of terminal output: printable ASCII kept, tab a space, CR and LF one line break (no empty +/// lines), cursor moves up, down or to a position a line break, a move right a space; every other escape +/// sequence (colours, modes, titles), control byte and non-ASCII byte dropped. out needs src.len + 2 bytes. +/// The result is a rough picture of the screen's rows, enough for keys.t27 to read an error row from. +pub fn screen_text(src: []const u8, out: []u8) -> usize { + var n : usize = 0; + var i : usize = 0; + while (i < src.len and n + 1 < out.len) { + const b : u8 = src[i]; + if (b == 27) { + var j : usize = i + 1; + if (j >= src.len) { + i = src.len; + } else if (src[j] == 91) { + j = j + 1; + while (j < src.len and (src[j] < 64 or src[j] > 126)) { j = j + 1; } + if (j < src.len) { + const f : u8 = src[j]; + if (f == 65 or f == 66 or f == 69 or f == 70 or f == 72 or f == 100 or f == 102 or f == 74) { + n = put_break(out, n); + } else if (f == 67 or f == 71) { + out[n] = 32; + n = n + 1; + } + } + i = j + 1; + } else if (src[j] == 93) { + j = j + 1; + while (j < src.len and src[j] != 7 and src[j] != 27) { j = j + 1; } + if (j < src.len and src[j] == 27) { j = j + 2; } else { j = j + 1; } + i = j; + } else if (src[j] >= 40 and src[j] <= 43) { + i = j + 2; + } else { + i = j + 1; + } + } else if (b == 10 or b == 13) { + n = put_break(out, n); + i = i + 1; + } else if (b == 9) { + out[n] = 32; + n = n + 1; + i = i + 1; + } else if (b >= 32 and b < 127) { + out[n] = b; + n = n + 1; + i = i + 1; + } else { + i = i + 1; + } + } + return n; +} + +/// The failure class of one row: keys.t27 classify_line, but only on an error row (error_row). +pub fn row_class(line: []const u8) -> u32 { + if (error_row(line) == false) { return FC_NONE; } + return classify_line(line); +} + +/// The failure class of the rows of `text` (lines split at LF): the last row that is a failure; FC_NONE when +/// there is none (a row classed FC_OTHER is no failure to act on). +pub fn text_class(text: []const u8) -> u32 { + var cls : u32 = FC_NONE; + var a : usize = 0; + var i : usize = 0; + while (i <= text.len) { + if (i == text.len or text[i] == 10) { + if (i > a) { + const c : u32 = row_class(text[a .. i]); + if (c != FC_NONE and c != FC_OTHER) { cls = c; } + } + a = i + 1; + } + i = i + 1; + } + return cls; +} + +/// Where the unfinished last row of `text` starts and how long it is, cut at CARRY_CAP: f[0], f[1]. The next +/// chunk is read with this row in front, so an error row split across two reads is still one row. +pub fn carry_cut(text: []const u8, f: []u64) -> void { + var a : usize = 0; + var i : usize = 0; + while (i < text.len) { + if (text[i] == 10) { a = i + 1; } + i = i + 1; + } + var l : usize = text.len - a; + if (l > CARRY_CAP) { l = CARRY_CAP; } + f[0] = @as(u64, a); + f[1] = @as(u64, l); +} + +/// Only the claude cell is read: its error rows are the ones keys.t27 knows, and only it has profiles. +pub fn scan_applies(a: u32) -> bool { + return profile_agent(a); +} + +/// Is pane i's screen read now? Not in the first SCAN_GRACE_MS after it starts (a resumed session replays its +/// transcript) and not while a give-up is quiet. +pub fn scan_ready(st: []const u64, i: usize, born: u64, now: u64) -> bool { + if (now < born + SCAN_GRACE_MS) { return false; } + return now >= st[G_PANEK + i * PK + PK_STOP]; +} + +// ---- a restored pane waits for its keys ---- +// After a server restart the panes come back from their records at once, but the window sends the keys only +// when it connects. A restored claude pane would start with the subscription variables stripped and no +// profile ("Not logged in"). So a restored pane whose agent reads a provider key waits for a key of that +// provider, or RESTORE_WAIT_MS since the server started, and then starts as it always did; a restored pane +// that started with no key is respawned once when its first key arrives. + +/// Does a pane of agent a read some provider's key (the shell cell and an editor read none)? +pub fn needs_key(a: u32) -> bool { + return spawn_mask(a, 511) != 0; +} + +/// Does the table hold a key a pane of agent a can start with? The claude cell reads the key of a profile's +/// provider (z.ai, Anthropic, OpenRouter, Ollama cloud), every other agent a provider of AGENT_PROVIDERS. +pub fn has_key_for(st: []const u64, a: u32) -> bool { + const stored : u32 = keys_stored(st); + if (profile_agent(a)) { return pick_profile(stored, 0) != PR_NONE; } + return spawn_mask(a, stored) != 0; +} + +/// A pane being restored waits instead of launching. +pub fn restore_waits(st: []const u64, a: u32) -> bool { + return needs_key(a) and has_key_for(st, a) == false; +} + +/// The wait ends: a key of the agent's provider arrived, or the time is up (`start` is G_START). +pub fn restore_release(st: []const u64, a: u32, now: u64, start: u64) -> bool { + return has_key_for(st, a) or now >= start + RESTORE_WAIT_MS; +} + +/// PK_WAIT of a pane that waits with launch plan `plan`, and the plan out of it (R_SKIP when not waiting). +pub fn wait_word(plan: u32) -> u64 { + return @as(u64, plan) + 1; +} + +pub fn waiting(w: u64) -> bool { + return w >= 1 and w <= 3; +} + +pub fn wait_plan(w: u64) -> u32 { + if (waiting(w) == false) { return R_SKIP; } + return @as(u32, @intCast(w - 1)); +} + +/// PK_WAIT after a deferred launch: nothing when a key was there, W_KEYLESS when the wait ran out. +pub fn after_wait(st: []const u64, a: u32) -> u64 { + if (has_key_for(st, a)) { return W_NONE; } + return W_KEYLESS; +} + +/// A restored pane that started keyless gets its one respawn when its first key arrives. +pub fn keyless_respawn(w: u64, st: []const u64, a: u32) -> bool { + return w == W_KEYLESS and has_key_for(st, a); +} + +/// The plan of that respawn: resume when the session is known, else relaunch fresh. +pub fn keyless_plan(a: u32, sid: []const u8) -> u32 { + const p : u32 = restore_plan(a, 0, sid); + if (p == R_SKIP) { return R_RELAUNCH; } + return p; +} + // ---- tests: each runs and asserts on values the decisions compute ---- @@ -643,3 +1138,430 @@ test an_editor_pane_survives_a_restart_in_its_record { // the path limit is the pane cwd's assert TERM_PATH_MAX == CWD_CAP - 1; } + +// ---- tests of the key rotation. Every key here is a slot number; no test holds a key value, real or fake. ---- + +fn t_table(st: []u64) -> void { + var z : usize = 0; + while (z < 512) { + st[z] = 0; + z = z + 1; + } + keys_init(st); +} + +fn t_key(st: []u64, p: u32, k: u32) -> void { + key_put(st, p, k, true, false); +} + +/// The profile pane i has chosen (PR_NONE when it holds no real profile). +fn t_prof(st: []const u64, i: usize) -> u32 { + return @as(u32, @intCast(st[G_PANEK + i * PK + PK_PROF])); +} + +/// The table of the owner's setup, in miniature: three z.ai keys and one Anthropic key. +fn t_zai3(st: []u64) -> void { + t_table(st); + t_key(st, P_ZAI, 0); + t_key(st, P_ZAI, 1); + t_key(st, P_ZAI, 2); + t_key(st, P_ANTHROPIC, 0); +} + +test the_server_tables_and_the_arena_line_up_for_eight_keys_a_provider { + assert KEY_SLOTS == @as(usize, PROVIDER_COUNT) * @as(usize, MAX_KEYS); + assert G_KLAST + KEY_SLOTS <= G_KREST; + assert G_KREST + KEY_SLOTS <= G_PANEK; + assert G_PANEK + 16 * PK <= ST_SIZE; + assert G_START < G_CFD; + assert G_START > G_BINLEN; + assert 16 * PF <= G_LISTEN; + assert key_slot(0, 0) == O_KEYS; + assert key_slot(4, 2) == O_KEYS + (4 * 8 + 2) * KEY_CAP; + assert key_slot(PROVIDER_COUNT - 1, MAX_KEYS - 1) + KEY_CAP <= O_CARRY; + assert O_CARRY + 16 * CARRY_SLOT <= O_SCAN; + assert O_SCAN + SCAN_CAP <= MEM_SIZE; + assert CARRY_SLOT == CARRY_CAP + 4; + assert SCAN_CAP >= CARRY_CAP + 4096 + 2; + assert MAX_KEYS <= 8; +} + +test an_empty_table_holds_no_key_and_a_put_fills_one_slot { + var st : [512]u64 = undefined; + t_table(st[0..512]); + assert keys_stored(st[0..512]) == 0; + assert key_count(st[0..512], P_ZAI) == 0; + assert key_present(st[0..512], P_ZAI, 3) == false; + t_key(st[0..512], P_ZAI, 3); + assert key_present(st[0..512], P_ZAI, 3); + assert key_count(st[0..512], P_ZAI) == 1; + assert keys_stored(st[0..512]) == 16; + t_key(st[0..512], P_OPENAI, 7); + assert keys_stored(st[0..512]) == 18; + key_put(st[0..512], P_ZAI, 3, false, false); + assert keys_stored(st[0..512]) == 2; + assert st[G_KREST + 4 * 8 + 3] == KEY_NEVER; +} + +test sending_the_same_key_again_keeps_its_cooldown_and_a_new_key_starts_fresh { + var st : [512]u64 = undefined; + t_table(st[0..512]); + t_key(st[0..512], P_ZAI, 0); + key_used(st[0..512], P_ZAI, 0, 777); + key_rest(st[0..512], P_ZAI, 0, 5000); + key_put(st[0..512], P_ZAI, 0, true, true); + assert st[G_KREST + 32] == 5000; + assert st[G_KLAST + 32] == 777; + key_put(st[0..512], P_ZAI, 0, true, false); + assert st[G_KREST + 32] == 0; + assert st[G_KLAST + 32] == 0; + // a rest never shortens, and an empty slot never gets one + key_rest(st[0..512], P_ZAI, 0, 900); + key_rest(st[0..512], P_ZAI, 0, 400); + assert st[G_KREST + 32] == 900; + key_rest(st[0..512], P_ZAI, 1, 900); + assert st[G_KREST + 33] == KEY_NEVER; +} + +test new_claude_panes_take_the_z_ai_keys_in_rotation { + var st : [512]u64 = undefined; + t_zai3(st[0..512]); + var n : usize = 0; + var want : [4]u32 = [1, 2, 3, 1]; + while (n < 4) { + plan_pane_keys(st[0..512], n, 0, 1000000 + @as(u64, n) * 100); + assert t_prof(st[0..512], n) == PR_ZAI; + assert pane_key_get(st[0..512], n, P_ZAI) == want[n]; + assert pane_key_get(st[0..512], n, P_ANTHROPIC) == 0; + assert pane_key_shown(st[0..512], n, 0) == want[n]; + n = n + 1; + } + // a key that rests is passed over + key_rest(st[0..512], P_ZAI, 1, 2000); + plan_pane_keys(st[0..512], 5, 0, 1000500); + assert pane_key_get(st[0..512], 5, P_ZAI) == 3; + // every key resting: the one whose rest ends first + key_rest(st[0..512], P_ZAI, 0, 3000); + key_rest(st[0..512], P_ZAI, 2, 2500); + plan_pane_keys(st[0..512], 6, 0, 1000600); + assert pane_key_get(st[0..512], 6, P_ZAI) == 2; +} + +test every_other_agent_gets_a_key_of_each_provider_it_reads { + var st : [512]u64 = undefined; + t_zai3(st[0..512]); + t_key(st[0..512], P_OPENAI, 0); + t_key(st[0..512], P_OPENAI, 1); + // OpenCode (cell 3) reads Anthropic, OpenAI, OpenRouter: OpenRouter holds none + plan_pane_keys(st[0..512], 0, 3, 1000000); + assert pane_key_get(st[0..512], 0, P_ANTHROPIC) == 1; + assert pane_key_get(st[0..512], 0, P_OPENAI) == 1; + assert pane_key_get(st[0..512], 0, 3) == 0; + assert pane_key_get(st[0..512], 0, P_ZAI) == 0; + // Codex (cell 1) reads OpenAI only, and takes the key used longest ago + plan_pane_keys(st[0..512], 1, 1, 1000100); + assert pane_key_get(st[0..512], 1, P_OPENAI) == 2; + assert pane_key_get(st[0..512], 1, P_ANTHROPIC) == 0; + assert pane_key_shown(st[0..512], 1, 1) == 2; + // Crush (cell 5) reads ZAI_API_KEY among others + plan_pane_keys(st[0..512], 2, 5, 1000200); + assert pane_key_get(st[0..512], 2, P_ZAI) > 0; + assert pane_key_get(st[0..512], 2, P_OPENAI) > 0; + // a shell and an editor read none; an agent with no key of its provider gets nothing + plan_pane_keys(st[0..512], 3, 9, 1000300); + assert pane_key_shown(st[0..512], 3, 9) == 0; + assert st[G_PANEK + 3 * PK + PK_KEYS] == 0; + plan_pane_keys(st[0..512], 4, 2, 1000400); + assert pane_key_shown(st[0..512], 4, 2) == 0; + // the key numbers of one pane do not touch another's + assert pane_key_get(st[0..512], 0, P_OPENAI) == 1; +} + +test an_overloaded_key_swaps_to_the_next_key_rests_and_is_passed_over_by_other_panes { + var st : [512]u64 = undefined; + var why : [128]u8 = undefined; + var wl : [2]u64 = undefined; + t_zai3(st[0..512]); + plan_pane_keys(st[0..512], 0, 0, 1000000); + assert pane_key_get(st[0..512], 0, P_ZAI) == 1; + const act : u32 = plan_failover(st[0..512], 0, FC_OVERLOADED, 1005000, why[0..128], wl[0..2]); + assert act == FA_RESPAWN_KEY; + assert st[G_KREST + 32] == 1005 + 60; + assert st[G_PANEK + PK_SKIP] == 1; + assert st[G_PANEK + PK_DONE] == 1; + assert t_prof(st[0..512], 0) == PR_ZAI; + assert has_bytes(why[0 .. @as(usize, @intCast(wl[0]))], "claude: zai overloaded (529), same profile with the next key"); + plan_pane_keys(st[0..512], 0, 0, 1006000); + assert pane_key_get(st[0..512], 0, P_ZAI) == 2; + assert st[G_PANEK + PK_SKIP] == 0; + // another pane started meanwhile does not take the resting key + plan_pane_keys(st[0..512], 1, 0, 1010000); + assert pane_key_get(st[0..512], 1, P_ZAI) == 3; + // and a rate limit rests the key for thirty seconds + const b : u32 = plan_failover(st[0..512], 1, FC_RATE_LIMITED, 1011000, why[0..128], wl[0..2]); + assert b == FA_RESPAWN_KEY; + assert st[G_KREST + 34] == 1011 + 30; +} + +test when_every_z_ai_key_is_overloaded_the_pane_moves_to_the_next_profile_then_gives_up { + var st : [512]u64 = undefined; + var why : [128]u8 = undefined; + var wl : [2]u64 = undefined; + t_zai3(st[0..512]); + plan_pane_keys(st[0..512], 0, 0, 1000000); + assert pane_key_get(st[0..512], 0, P_ZAI) == 1; + var a : u32 = plan_failover(st[0..512], 0, FC_OVERLOADED, 1001000, why[0..128], wl[0..2]); + assert a == FA_RESPAWN_KEY; + plan_pane_keys(st[0..512], 0, 0, 1001500); + assert pane_key_get(st[0..512], 0, P_ZAI) == 2; + a = plan_failover(st[0..512], 0, FC_OVERLOADED, 1002000, why[0..128], wl[0..2]); + assert a == FA_RESPAWN_KEY; + plan_pane_keys(st[0..512], 0, 0, 1002500); + assert pane_key_get(st[0..512], 0, P_ZAI) == 3; + // the third key is overloaded too and the other two still rest: next profile, z.ai marked bad + a = plan_failover(st[0..512], 0, FC_OVERLOADED, 1003000, why[0..128], wl[0..2]); + assert a == FA_RESPAWN_NEXT; + assert t_prof(st[0..512], 0) == PR_ANTHROPIC; + assert st[G_PANEK + PK_BAD] == 16; + assert has_bytes(why[0 .. @as(usize, @intCast(wl[0]))], "zai overloaded (529), respawn on anthropic-api"); + plan_pane_keys(st[0..512], 0, 0, 1003500); + assert pane_key_get(st[0..512], 0, P_ANTHROPIC) == 1; + assert pane_key_get(st[0..512], 0, P_ZAI) == 0; + // the Anthropic key is overloaded: no profile is left, the pane stays and its screen is not read for a minute + a = plan_failover(st[0..512], 0, FC_OVERLOADED, 1004000, why[0..128], wl[0..2]); + assert a == FA_GIVE_UP; + assert st[G_PANEK + PK_DONE] == 3; + assert st[G_PANEK + PK_STOP] == 1004000 + GIVEUP_QUIET_MS; + assert has_bytes(why[0 .. @as(usize, @intCast(wl[0]))], "no profile left in the chain"); + assert scan_ready(st[0..512], 0, 1003500, 1020000) == false; + assert scan_ready(st[0..512], 0, 1003500, 1064001); +} + +test an_empty_balance_leaves_the_provider_at_once_and_the_key_rests_ten_minutes { + var st : [512]u64 = undefined; + var why : [128]u8 = undefined; + var wl : [2]u64 = undefined; + t_zai3(st[0..512]); + plan_pane_keys(st[0..512], 0, 0, 1000000); + const a : u32 = plan_failover(st[0..512], 0, FC_NO_BALANCE, 1005000, why[0..128], wl[0..2]); + assert a == FA_RESPAWN_NEXT; + assert st[G_KREST + 32] == 1005 + 600; + assert st[G_PANEK + PK_BAD] == 16; + assert t_prof(st[0..512], 0) == PR_ANTHROPIC; + assert has_bytes(why[0 .. @as(usize, @intCast(wl[0]))], "zai out of balance, respawn on anthropic-api"); + // the failure cap ends any loop + st[G_PANEK + PK_DONE] = @as(u64, MAX_FAILOVERS); + const g : u32 = plan_failover(st[0..512], 0, FC_OVERLOADED, 1006000, why[0..128], wl[0..2]); + assert g == FA_GIVE_UP; + assert has_bytes(why[0 .. @as(usize, @intCast(wl[0]))], "failover limit reached"); +} + +test a_pane_that_started_without_a_key_takes_the_first_stored_profile_on_a_subscription_row { + var st : [512]u64 = undefined; + var why : [128]u8 = undefined; + var wl : [2]u64 = undefined; + t_table(st[0..512]); + plan_pane_keys(st[0..512], 0, 0, 1000000); + assert t_prof(st[0..512], 0) == PR_NONE; + assert pane_key_shown(st[0..512], 0, 0) == 0; + // nothing stored: nothing to move to + assert plan_failover(st[0..512], 0, FC_SUBSCRIPTION, 1001000, why[0..128], wl[0..2]) == FA_GIVE_UP; + // the window's keys arrive; the pane asks again after its quiet time + t_key(st[0..512], P_ZAI, 4); + st[G_PANEK + PK_STOP] = 0; + const a : u32 = plan_failover(st[0..512], 0, FC_SUBSCRIPTION, 1002000, why[0..128], wl[0..2]); + assert a == FA_RESPAWN_NEXT; + assert t_prof(st[0..512], 0) == PR_ZAI; + assert has_bytes(why[0 .. @as(usize, @intCast(wl[0]))], "no profile subscription disabled, respawn on zai"); + plan_pane_keys(st[0..512], 0, 0, 1003000); + assert pane_key_get(st[0..512], 0, P_ZAI) == 5; +} + +test a_new_spawn_starts_with_a_clean_pane_record { + var st : [512]u64 = undefined; + t_zai3(st[0..512]); + plan_pane_keys(st[0..512], 2, 0, 1000000); + st[G_PANEK + 2 * PK + PK_DONE] = 5; + st[G_PANEK + 2 * PK + PK_BAD] = 16; + pane_keys_reset(st[0..512], 2); + assert st[G_PANEK + 2 * PK + PK_DONE] == 0; + assert st[G_PANEK + 2 * PK + PK_BAD] == 0; + assert st[G_PANEK + 2 * PK + PK_PROF] == PK_UNSET; + assert st[G_PANEK + 2 * PK + PK_KEYS] == 0; + assert st[G_PANEK + 2 * PK + PK_WAIT] == W_NONE; + plan_pane_keys(st[0..512], 2, 0, 1000100); + assert t_prof(st[0..512], 2) == PR_ZAI; +} + +test a_list_entry_carries_the_key_number_in_the_high_nibble { + const w : u32 = pane_step(0, P_SPAWNED); + assert list_flags_key(true, w, 0) == list_flags(true, w); + assert list_flags_key(true, w, 3) % 16 == list_flags(true, w); + assert list_flags_key(true, w, 3) / 16 == 3; + assert list_flags_key(false, w, MAX_KEYS) / 16 == MAX_KEYS; + assert list_flags_key(true, hooked_status(w, "Notification"), MAX_KEYS) < 256; + // a client that reads bit 0, 1 or 2 only sees the old byte + assert list_flags_key(true, hooked_status(w, "Notification"), 5) % 2 == 1; +} + +test terminal_bytes_become_plain_rows { + var s : [200]u8 = undefined; + var o : [210]u8 = undefined; + var n : usize = put_bytes(s[0..200], 0, "ab"); + s[n] = 27; + n = put_bytes(s[0..200], n + 1, "[31;1mAPI Error: 529"); + s[n] = 27; + n = put_bytes(s[0..200], n + 1, "[0m"); + s[n] = 13; + s[n + 1] = 10; + n = put_bytes(s[0..200], n + 2, "next"); + var t : usize = screen_text(s[0..n], o[0..210]); + assert same_bytes(o[0..t], "abAPI Error: 529\nnext"); + // a cursor move up or to a position is a line break, one to the right a space + n = put_bytes(s[0..200], 0, "one"); + s[n] = 27; + n = put_bytes(s[0..200], n + 1, "[2K"); + s[n] = 27; + n = put_bytes(s[0..200], n + 1, "[1Atwo"); + s[n] = 27; + n = put_bytes(s[0..200], n + 1, "[3Cx"); + t = screen_text(s[0..n], o[0..210]); + assert same_bytes(o[0..t], "one\ntwo x"); + // a title (OSC) up to BEL, a charset switch, a tab, a bare ESC at the end, bytes above 127 + n = put_bytes(s[0..200], 0, "a"); + s[n] = 27; + n = put_bytes(s[0..200], n + 1, "]0;title"); + s[n] = 7; + s[n + 1] = 27; + n = put_bytes(s[0..200], n + 2, "(Bb"); + s[n] = 9; + s[n + 1] = 200; + s[n + 2] = 99; + s[n + 3] = 27; + t = screen_text(s[0..n + 4], o[0..210]); + assert same_bytes(o[0..t], "ab c"); + assert screen_text("", o[0..210]) == 0; + // the output never runs past its room + assert screen_text("abcdefgh", o[0..5]) == 4; + // empty lines collapse and a leading break is dropped + t = screen_text("\n\nx\n\ny", o[0..210]); + assert same_bytes(o[0..t], "x\ny"); +} + +test only_an_error_row_on_the_screen_names_a_failure { + assert text_class("API Error: 529 {\"error\":{\"code\":\"1305\"}}") == FC_OVERLOADED; + assert text_class("reading\nAPI Error: 429 Insufficient balance\nprompt") == FC_NO_BALANCE; + assert text_class("API Error: 401 bad key\nAPI Error: 529 busy") == FC_OVERLOADED; + // a diff or a quotation that shows the phrases is not a failure + assert text_class("+ assert classify_line(\"API Error: 429 Insufficient balance\") == FC_NO_BALANCE;") == FC_NONE; + assert text_class("the log said API Error: 529\nok") == FC_NONE; + // Claude Code's own retry is not final + assert text_class("API Error (529 overloaded_error) Retrying in 4 seconds") == FC_NONE; + assert text_class("API Error: 500 upstream failure") == FC_NONE; + assert text_class("") == FC_NONE; + assert text_class("\n\n") == FC_NONE; + assert row_class(" Error: 403 forbidden") == FC_LOGIN_REJECTED; +} + +test an_error_row_split_across_two_reads_is_still_one_row { + var o : [300]u8 = undefined; + var c : [300]u8 = undefined; + var f : [2]u64 = undefined; + // read 1 ends in the middle of the row; its unfinished row is carried + var t : usize = screen_text("Working...\nAPI Err", o[0..300]); + assert text_class(o[0..t]) == FC_NONE; + carry_cut(o[0..t], f[0..2]); + assert f[0] == 11; + assert f[1] == 7; + var cl : usize = put_bytes(c[0..300], 0, o[@as(usize, @intCast(f[0])) .. @as(usize, @intCast(f[0])) + @as(usize, @intCast(f[1]))]); + // read 2 finishes it + const t2 : usize = screen_text("or: 529 overloaded\n", o[0..300]); + cl = put_bytes(c[0..300], cl, o[0..t2]); + assert text_class(c[0..cl]) == FC_OVERLOADED; + // a long unfinished row is cut at the carry limit; a finished last row carries nothing + var big : [200]u8 = undefined; + var i : usize = 0; + while (i < 200) { + big[i] = 97; + i = i + 1; + } + carry_cut(big[0..200], f[0..2]); + assert f[0] == 0; + assert f[1] == CARRY_CAP; + carry_cut("abc\n", f[0..2]); + assert f[0] == 4; + assert f[1] == 0; +} + +test a_pane_screen_is_not_read_in_its_first_seconds_nor_while_a_give_up_is_quiet { + var st : [512]u64 = undefined; + t_table(st[0..512]); + assert scan_ready(st[0..512], 1, 1000, 5000) == false; + assert scan_ready(st[0..512], 1, 1000, 11000); + st[G_PANEK + PK + PK_STOP] = 70000; + assert scan_ready(st[0..512], 1, 1000, 60000) == false; + assert scan_ready(st[0..512], 1, 1000, 70000); + assert scan_applies(0); + assert scan_applies(1) == false; + assert scan_applies(SHELL_CELL) == false; +} + +test a_restored_pane_waits_for_a_key_of_its_provider_and_then_starts_with_it { + var st : [512]u64 = undefined; + t_table(st[0..512]); + // the window has not connected: claude, codex, queen wait; a shell and an editor do not + assert needs_key(0); + assert needs_key(1); + assert needs_key(4); + assert needs_key(SHELL_CELL) == false; + assert restore_waits(st[0..512], 0); + assert restore_waits(st[0..512], 1); + assert restore_waits(st[0..512], SHELL_CELL) == false; + assert wait_plan(wait_word(R_RESUME)) == R_RESUME; + assert wait_plan(wait_word(R_RELAUNCH)) == R_RELAUNCH; + assert waiting(wait_word(R_RESUME)) and waiting(wait_word(R_RELAUNCH)); + assert waiting(W_NONE) == false; + assert waiting(W_KEYLESS) == false; + assert waiting(W_DONE) == false; + assert wait_plan(W_NONE) == R_SKIP; + // 2 s after the server started, still nothing + assert restore_release(st[0..512], 0, 1002000, 1000000) == false; + // an OpenAI key arrives: claude reads no OpenAI key, codex does + t_key(st[0..512], P_OPENAI, 0); + assert restore_release(st[0..512], 0, 1003000, 1000000) == false; + assert restore_release(st[0..512], 1, 1003000, 1000000); + // a z.ai key arrives: claude's profile has a provider now, and the pane starts on it + t_key(st[0..512], P_ZAI, 0); + assert restore_release(st[0..512], 0, 1003500, 1000000); + assert after_wait(st[0..512], 0) == W_NONE; + plan_pane_keys(st[0..512], 3, 0, 1003600); + assert t_prof(st[0..512], 3) == PR_ZAI; + assert pane_key_get(st[0..512], 3, P_ZAI) == 1; +} + +test a_restored_pane_whose_wait_runs_out_starts_keyless_and_is_respawned_once_when_a_key_arrives { + var st : [512]u64 = undefined; + t_table(st[0..512]); + assert restore_release(st[0..512], 0, 1019999, 1000000) == false; + assert restore_release(st[0..512], 0, 1020000, 1000000); + assert after_wait(st[0..512], 0) == W_KEYLESS; + plan_pane_keys(st[0..512], 0, 0, 1020100); + assert t_prof(st[0..512], 0) == PR_NONE; + st[G_PANEK + PK_WAIT] = W_KEYLESS; + // still no key: no respawn + assert keyless_respawn(st[G_PANEK + PK_WAIT], st[0..512], 0) == false; + // the first key arrives + t_key(st[0..512], P_ZAI, 0); + assert keyless_respawn(st[G_PANEK + PK_WAIT], st[0..512], 0); + // the respawn: the pane takes the profile and the key, and the one respawn is spent + plan_pane_keys(st[0..512], 0, 0, 1030000); + st[G_PANEK + PK_WAIT] = W_DONE; + assert t_prof(st[0..512], 0) == PR_ZAI; + assert pane_key_shown(st[0..512], 0, 0) == 1; + assert keyless_respawn(st[G_PANEK + PK_WAIT], st[0..512], 0) == false; + assert keyless_respawn(W_NONE, st[0..512], 0) == false; + // resume when the session is known, else a fresh start + assert keyless_plan(0, "5f1c9a00-aaaa-bbbb-cccc-0123456789ab") == R_RESUME; + assert keyless_plan(0, "-") == R_RELAUNCH; + assert keyless_plan(0, "") == R_RELAUNCH; +} diff --git a/specs/t27os/pty_server.t27 b/specs/t27os/pty_server.t27 index d8416e7978..d0b3d02c14 100644 --- a/specs/t27os/pty_server.t27 +++ b/specs/t27os/pty_server.t27 @@ -18,6 +18,14 @@ // T27OS_INSTALL=1 write ~/Library/LaunchAgents/ai.t27.os-server.plist for this binary: it starts // the server at login, so panes come back (by session id, model kept) after a reboot. // +// Keys (gHashTag/t27#11241): the server holds up to MAX_KEYS numbered keys for each provider, in memory only, +// sent by the window with OP_SETKEY (the key number rides in the high nibble of the pane byte). Every agent +// pane picks its key by rotation at each start; the pane record keeps the key NUMBER, never the key. The +// claude cell's screen is read for an error row (pty_logic screen_text, keys.t27 error_row and classify_line): +// the failed key rests, the pane is respawned (resumed) on the next key or profile, or stays with a line in its +// scrollback. A restored pane waits for the window's keys (or 20 s) before it starts. T27OS_CTL=list prints +// "key N" for each pane. +// // Every decision is a fn with a test that runs; the host loop only moves bytes through pty_sys.t27. // // Refs gHashTag/t27#10782. @@ -145,16 +153,54 @@ use t27os::pty_sys::sys_unsetenv; use t27os::pty_logic::O_KEYS; use t27os::pty_logic::KEY_CAP; use t27os::pty_proto::OP_SETKEY; +use t27os::pty_logic::G_PANEK; +use t27os::pty_logic::G_START; +use t27os::pty_logic::PK; +use t27os::pty_logic::PK_PROF; +use t27os::pty_logic::PK_WAIT; +use t27os::pty_logic::W_DONE; +use t27os::pty_logic::O_CARRY; +use t27os::pty_logic::CARRY_SLOT; +use t27os::pty_logic::O_SCAN; +use t27os::pty_logic::SCAN_CAP; +use t27os::pty_logic::CARRY_CAP; +use t27os::pty_logic::key_slot; +use t27os::pty_logic::keys_init; +use t27os::pty_logic::pane_keys_reset; +use t27os::pty_logic::pane_key_get; +use t27os::pty_logic::pane_key_shown; +use t27os::pty_logic::list_flags_key; +use t27os::pty_logic::key_put; +use t27os::pty_logic::plan_pane_keys; +use t27os::pty_logic::plan_failover; +use t27os::pty_logic::screen_text; +use t27os::pty_logic::text_class; +use t27os::pty_logic::carry_cut; +use t27os::pty_logic::scan_applies; +use t27os::pty_logic::scan_ready; +use t27os::pty_logic::restore_waits; +use t27os::pty_logic::restore_release; +use t27os::pty_logic::wait_word; +use t27os::pty_logic::waiting; +use t27os::pty_logic::wait_plan; +use t27os::pty_logic::after_wait; +use t27os::pty_logic::keyless_respawn; +use t27os::pty_logic::keyless_plan; use t27os::keys::PROVIDER_COUNT; +use t27os::keys::MAX_KEYS; +use t27os::keys::PROFILE_COUNT; +use t27os::keys::FC_NONE; +use t27os::keys::FA_KEEP; +use t27os::keys::FA_GIVE_UP; +use t27os::keys::setkey_provider; +use t27os::keys::setkey_number; use t27os::keys::agent_uses; use t27os::keys::provider_env; use t27os::keys::env_stripped; use t27os::keys::key_fits; use t27os::keys::profile_agent; use t27os::keys::claude_strips; -use t27os::keys::pick_profile; use t27os::keys::PROFILE_PROVIDER; -use t27os::keys::PR_NONE; use t27os::keys::profile_var_count; use t27os::keys::profile_var_name; use t27os::keys::profile_var_is_key; @@ -385,19 +431,18 @@ fn strip_claude_env(mem: []u8, agent: u32) -> void { } } -/// In the child, for the claude agent: pick the profile (keys.t27 pick_profile over the stored keys; the -/// window withholds a key that failed its check) and set exactly that profile's variables. No stored key -/// leaves the pane without one, and none of the other profile's variables is set. -fn inject_profile(mem: []u8) -> void { - var stored : u32 = 0; - var p : u32 = 0; - while (p < PROVIDER_COUNT) { - if (mem[O_KEYS + @as(usize, p) * KEY_CAP] != 0) { stored = stored + (1 << p); } - p = p + 1; - } - const pr : u32 = pick_profile(stored, 0); - if (pr == PR_NONE) { return; } - const at : usize = O_KEYS + @as(usize, PROFILE_PROVIDER[pr]) * KEY_CAP; +/// In the child, for the claude agent: set exactly the variables of the profile the parent chose for pane i +/// (pty_logic plan_pane_keys: pick_profile over the providers that hold a key, or the profile a failover moved +/// to) with the key of the number it picked by rotation. No usable key leaves the pane without one, and none of +/// the other profiles' variables is set. The key is read from its slot; its number is all the pane record holds. +fn inject_profile(st: []const u64, mem: []u8, i: usize) -> void { + const pr64 : u64 = st[G_PANEK + i * PK + PK_PROF]; + if (pr64 >= @as(u64, PROFILE_COUNT)) { return; } + const pr : u32 = @as(u32, @intCast(pr64)); + const prov : u32 = PROFILE_PROVIDER[pr]; + const kn : u32 = pane_key_get(st, i, prov); + if (kn == 0) { return; } + const at : usize = key_slot(prov, kn - 1); var k : u32 = 0; while (k < profile_var_count(pr)) { const nl : usize = profile_var_name(pr, k, mem[O_TMP2 .. O_TMP2 + 40]); @@ -413,19 +458,24 @@ fn inject_profile(mem: []u8) -> void { } } -/// In the child: set the environment variable of every stored key the agent reads (OP_SETKEY filled them). -fn inject_keys(mem: []u8, agent: u32) -> void { +/// In the child: set the environment variable (keys.t27 PROVIDER_ENVS) of each provider the agent reads +/// (AGENT_PROVIDERS) to the key plan_pane_keys picked for pane i from that provider's numbered keys. A +/// provider that holds no key leaves its variable unset. +fn inject_keys(st: []const u64, mem: []u8, i: usize, agent: u32) -> void { if (profile_agent(agent)) { - inject_profile(mem); + inject_profile(st, mem, i); return; } var p : u32 = 0; while (p < PROVIDER_COUNT) { - const at : usize = O_KEYS + @as(usize, p) * KEY_CAP; - if (agent_uses(agent, p) and mem[at] != 0) { - const nl : usize = provider_env(p, mem[O_TMP2 .. O_TMP2 + 40]); - mem[O_TMP2 + nl] = 0; - sys_setenv(mem[O_TMP2 .. O_TMP2 + nl + 1], mem[at + 1 .. at + 2 + @as(usize, mem[at])], 1); + const kn : u32 = pane_key_get(st, i, p); + if (agent_uses(agent, p) and kn > 0) { + const at : usize = key_slot(p, kn - 1); + if (mem[at] != 0) { + const nl : usize = provider_env(p, mem[O_TMP2 .. O_TMP2 + 40]); + mem[O_TMP2 + nl] = 0; + sys_setenv(mem[O_TMP2 .. O_TMP2 + nl + 1], mem[at + 1 .. at + 2 + @as(usize, mem[at])], 1); + } } p = p + 1; } @@ -440,6 +490,10 @@ fn launch(st: []u64, mem: []u8, i: usize, plan: u32) -> bool { const un : usize = uuid_text(mem[O_MISC + 64 .. O_MISC + 80], mem[O_SID + i * SID_CAP .. O_SID + i * SID_CAP + 40]); st[base + PF_SIDLEN] = @as(u64, un); } + // the keys of this start: the profile and the key number of each provider the agent reads, by rotation + // over the numbered keys (a failover has set what to skip); the unfinished screen row of the old process goes + plan_pane_keys(st, i, agent, now_ms(mem)); + mem[O_CARRY + i * CARRY_SLOT] = 0; const home : []const u8 = home_slice(st, mem); const cn : usize = launch_command(agent, plan, sid_of(st, mem, i), model_of(mem, i), home, cwd_of(st, mem, i), mem[O_TMP1 + 4096 .. O_TMP1 + 8000]); var f : [4]u64 = undefined; @@ -470,7 +524,7 @@ fn launch(st: []u64, mem: []u8, i: usize, plan: u32) -> bool { sys_chdir(mem[O_TMP1 + 3000 .. O_TMP1 + 3000 + cw + 1]); sys_setenv("T27OS_PANE", mem[O_TMP1 + 3300 .. O_TMP1 + 3300 + idn + 1], 1); strip_claude_env(mem, agent); - inject_keys(mem, agent); + inject_keys(st, mem, i, agent); sys_execv(mem[O_TMP1 .. O_TMP1 + 8], argv); sys_exit(127); return false; @@ -510,7 +564,8 @@ fn reap(st: []u64, mem: []u8, i: usize) -> void { save_pane(st, mem, i); } -fn kill_pane(st: []u64, mem: []u8, i: usize) -> void { +/// Stop pane i's process (SIGKILL, wait, close the master); the pane's record and slot stay. +fn stop_pane_process(st: []u64, i: usize) -> void { const base : usize = i * PF; if (st[base + PF_LIVE] != 0) { sys_kill(@as(i32, @intCast(st[base + PF_PID])), 9); @@ -520,11 +575,84 @@ fn kill_pane(st: []u64, mem: []u8, i: usize) -> void { sys_waitpid(@as(i32, @intCast(st[base + PF_PID])), ws, 0); sys_close(@as(i32, @intCast(st[base + PF_FD]))); } - drop_record(st, mem, i); st[base + PF_LIVE] = 0; +} + +fn kill_pane(st: []u64, mem: []u8, i: usize) -> void { + const base : usize = i * PF; + stop_pane_process(st, i); + drop_record(st, mem, i); st[base + PF_USED] = 0; } +/// Start pane i again under `plan` (a failover onto another key or profile, or the one respawn of a restored +/// pane that started without a key). The scrollback and its running count stay, so a window that reads the pane +/// goes on from where it was; launch sets the count to 0 and it is put back. +fn respawn_pane(st: []u64, mem: []u8, i: usize, plan: u32) -> void { + const total : u64 = st[i * PF + PF_TOTAL]; + stop_pane_process(st, i); + launch(st, mem, i, plan); + st[i * PF + PF_TOTAL] = total; + save_pane(st, mem, i); +} + +/// Pane i's screen showed failure `class`: pty_logic plan_failover decides (the key rests, the pane moves to the +/// next key or profile, or stays), the why line goes into the pane's scrollback, and on a respawn the pane is +/// started again (resumed when its session is known). A give-up writes the same line and leaves the pane as it is. +fn failover(st: []u64, mem: []u8, i: usize, class: u32, now: u64) -> void { + const base : usize = i * PF; + var wl : [2]u64 = undefined; + wl[0] = 0; + wl[1] = 0; + const act : u32 = plan_failover(st, i, class, now, mem[O_OUT + 2 .. O_OUT + 130], wl[0..2]); + if (act == FA_KEEP) { return; } + const n : usize = @as(usize, @intCast(wl[0])); + mem[O_OUT] = 13; + mem[O_OUT + 1] = 10; + mem[O_OUT + 2 + n] = 13; + mem[O_OUT + 3 + n] = 10; + st[base + PF_TOTAL] = ring_append(ring_of(mem, i), st[base + PF_TOTAL], mem[O_OUT .. O_OUT + n + 4], n + 4); + if (act == FA_GIVE_UP) { return; } + var plan : u32 = R_RELAUNCH; + if (st[base + PF_SIDLEN] > 0) { plan = R_RESUME; } + respawn_pane(st, mem, i, plan); +} + +/// Read the n new bytes of pane i (at O_MISC + 128) for a provider error row: only the claude cell, never in +/// the first seconds after it starts, never while a give-up is quiet. The bytes become plain rows (screen_text), +/// the unfinished last row of the previous read is put in front and the new one is kept for the next, and the +/// rows go through keys.t27 error_row and classify_line (pty_logic text_class). +fn scan_pane(st: []u64, mem: []u8, i: usize, n: usize) -> void { + const base : usize = i * PF; + if (scan_applies(@as(u32, @intCast(st[base + PF_AGENT]))) == false) { return; } + const now : u64 = now_ms(mem); + const ca : usize = O_CARRY + i * CARRY_SLOT; + if (scan_ready(st, i, st[base + PF_BORN], now) == false) { + mem[ca] = 0; + return; + } + const cl : usize = @as(usize, mem[ca]); + var k : usize = 0; + while (k < cl) { + mem[O_SCAN + k] = mem[ca + 1 + k]; + k = k + 1; + } + const tl : usize = screen_text(mem[O_MISC + 128 .. O_MISC + 128 + n], mem[O_SCAN + cl .. O_SCAN + cl + n + 2]); + const total : usize = cl + tl; + const cls : u32 = text_class(mem[O_SCAN .. O_SCAN + total]); + var f : [2]u64 = undefined; + carry_cut(mem[O_SCAN .. O_SCAN + total], f[0..2]); + const fs : usize = @as(usize, @intCast(f[0])); + const fl : usize = @as(usize, @intCast(f[1])); + mem[ca] = @as(u8, @intCast(fl)); + k = 0; + while (k < fl) { + mem[ca + 1 + k] = mem[O_SCAN + fs + k]; + k = k + 1; + } + if (cls != FC_NONE) { failover(st, mem, i, cls, now); } +} + fn reply(fd: i32, mem: []u8, op: u8, pane: u8, plen: usize) -> void { put_header(mem[O_OUT .. O_OUT + 8], op, pane, plen); sys_write(fd, mem[O_OUT .. O_OUT + HEADER + plen], @as(u64, HEADER + plen)); @@ -543,7 +671,9 @@ fn handle_frame(st: []u64, mem: []u8, fd: i32, off: usize) -> void { const b : usize = @as(usize, i) * PF; if (st[b + PF_USED] != 0) { const w : u32 = @as(u32, @intCast(st[b + PF_STATUS])); - at = put_list_entry(mem[O_OUT .. O_OUT + 128], at - O_OUT, i, @as(u32, @intCast(st[b + PF_AGENT])), shown_state(w), list_flags(st[b + PF_LIVE] != 0, w)) + O_OUT; + // the flags byte carries the pane's key number + 1 in its high nibble (0 = no key); bits 0 to 2 as before + const kshown : u32 = pane_key_shown(st, @as(usize, i), @as(u32, @intCast(st[b + PF_AGENT]))); + at = put_list_entry(mem[O_OUT .. O_OUT + 128], at - O_OUT, i, @as(u32, @intCast(st[b + PF_AGENT])), shown_state(w), list_flags_key(st[b + PF_LIVE] != 0, w, kshown)) + O_OUT; } i = i + 1; } @@ -570,6 +700,7 @@ fn handle_frame(st: []u64, mem: []u8, fd: i32, off: usize) -> void { st[sb + PF_SLOT] = @as(u64, mem[p0 + 2]); st[sb + PF_SIDLEN] = 0; st[sb + PF_CWDLEN] = @as(u64, cl); + pane_keys_reset(st, @as(usize, slot)); st[G_WINSZ + @as(usize, slot) / 2] = ws_pack(st[G_WINSZ + @as(usize, slot) / 2], slot % 2, 0, 0); var k : usize = 0; while (k < cl) { @@ -586,16 +717,29 @@ fn handle_frame(st: []u64, mem: []u8, fd: i32, off: usize) -> void { return; } if (op == OP_SETKEY) { - if (pane >= PROVIDER_COUNT or plen > KEY_CAP - 2) { reply(fd, mem, RE_ERR, 0, 0); return; } + // the pane byte is the provider in its low nibble and the key number (0 = key 1) in the high one + // (keys.t27 setkey_byte): key 1 is the frame this op always had, an older client changes nothing + const sp : u32 = setkey_provider(pane); + const sk : u32 = setkey_number(pane); + if (sp >= PROVIDER_COUNT or sk >= MAX_KEYS or plen > KEY_CAP - 2) { reply(fd, mem, RE_ERR, 0, 0); return; } if (plen > 0 and !key_fits(mem[p0 .. p0 + plen])) { reply(fd, mem, RE_ERR, 0, 0); return; } - const ka : usize = O_KEYS + @as(usize, pane) * KEY_CAP; - mem[ka] = @as(u8, @intCast(plen)); + const ka : usize = key_slot(sp, sk); + var same : bool = plen > 0 and @as(usize, mem[ka]) == plen; var j : usize = 0; while (j < plen) { + if (mem[ka + 1 + j] != mem[p0 + j]) { same = false; } mem[ka + 1 + j] = mem[p0 + j]; j = j + 1; } - mem[ka + 1 + plen] = 0; + mem[ka] = @as(u8, @intCast(plen)); + // the rest of the slot is cleared, so a shorter or deleted key leaves none of the old one behind + var wp : usize = plen + 1; + while (wp < KEY_CAP) { + mem[ka + wp] = 0; + wp = wp + 1; + } + // the table of times and rests follows the slot; the same key sent again keeps its cooldown + key_put(st, sp, sk, plen > 0, same); reply(fd, mem, RE_OK, @as(u8, @intCast(pane)), 0); return; } @@ -716,6 +860,35 @@ fn serve_pane(st: []u64, mem: []u8, i: usize) -> void { st[base + PF_TOTAL] = ring_append(ring_of(mem, i), st[base + PF_TOTAL], mem[O_MISC + 128 .. O_MISC + 128 + 4096], @as(usize, @intCast(n))); st[base + PF_LAST] = now_ms(mem); st[base + PF_STATUS] = @as(u64, pane_step(@as(u32, @intCast(st[base + PF_STATUS])), P_OUTPUT)); + // last, because a failover starts the pane again + scan_pane(st, mem, i, @as(usize, @intCast(n))); +} + +/// Once a tick: a restored pane that waits for a key of its provider starts when one arrived or the wait ran +/// out (a pane that ran out starts as it always did, without a key); a restored pane that started keyless is +/// started again, once, when its first key arrives. pty_logic holds both decisions. +fn wait_tick(st: []u64, mem: []u8) -> void { + const now : u64 = now_ms(mem); + var i : usize = 0; + while (i < MAX_PANES) { + const base : usize = i * PF; + const wk : usize = G_PANEK + i * PK + PK_WAIT; + const ag : u32 = @as(u32, @intCast(st[base + PF_AGENT])); + if (st[base + PF_USED] != 0) { + const w : u64 = st[wk]; + if (waiting(w)) { + if (restore_release(st, ag, now, st[G_START])) { + st[wk] = after_wait(st, ag); + launch(st, mem, i, wait_plan(w)); + save_pane(st, mem, i); + } + } else if (st[base + PF_LIVE] != 0 and keyless_respawn(w, st, ag)) { + st[wk] = W_DONE; + respawn_pane(st, mem, i, keyless_plan(ag, sid_of(st, mem, i))); + } + } + i = i + 1; + } } fn quiet_tick(st: []u64, mem: []u8) -> void { @@ -768,6 +941,7 @@ fn restore_all(st: []u64, mem: []u8) -> u32 { st[b + PF_SLOT] = f[F_SLOT]; st[b + PF_SIDLEN] = @as(u64, sl); st[b + PF_CWDLEN] = @as(u64, cl); + pane_keys_reset(st, @as(usize, i)); // the record's model (a v1 record has none), then the newer one the transcript names mem[O_MODEL + @as(usize, i) * MODEL_CAP] = 0; set_model(mem, @as(usize, i), O_TMP2 + 512 + @as(usize, @intCast(f[F_MODEL_AT])), @as(usize, @intCast(f[F_MODEL_LEN]))); @@ -777,6 +951,13 @@ fn restore_all(st: []u64, mem: []u8) -> u32 { var plan : u32 = restore_plan(@as(u32, @intCast(f[F_AGENT])), @as(u32, @intCast(f[F_STATE])), sidv); if (plan == R_SKIP) { st[b + PF_STATUS] = @as(u64, @intCast(f[F_STATE])); + } else if (restore_waits(st, @as(u32, @intCast(f[F_AGENT])))) { + // the window has not sent its keys yet: a pane started now would run without the profile and + // show "Not logged in". It waits for a key of its provider, or RESTORE_WAIT_MS since the + // server started (wait_tick), and starts then. + st[b + PF_STATUS] = @as(u64, @intCast(f[F_STATE])); + st[G_PANEK + @as(usize, i) * PK + PK_WAIT] = wait_word(plan); + back = back + 1; } else { if (launch(st, mem, @as(usize, i), plan)) { back = back + 1; } save_pane(st, mem, @as(usize, i)); @@ -895,6 +1076,7 @@ fn serve_loop(st: []u64, mem: []u8) -> void { } } quiet_tick(st, mem); + wait_tick(st, mem); } } @@ -904,6 +1086,8 @@ fn run_server(st: []u64, mem: []u8) -> void { say("t27os-server: cannot open the socket\n"); return; } + // the moment a restored pane starts its wait for the window's keys from (pty_logic RESTORE_WAIT_MS) + st[G_START] = now_ms(mem); const back : u32 = restore_all(st, mem); say("t27os-server: listening, restored "); var nb : [8]u8 = undefined; @@ -961,8 +1145,11 @@ fn run_ctl(st: []u64, mem: []u8, verb: u8) -> void { print_dec(@as(u64, out[at + 1]), mem); say(" state "); print_dec(@as(u64, out[at + 2]), mem); + // bits 0 to 2 of the flags byte as before; its high nibble is the key number (1 to 8) the pane holds say(" flags "); - print_dec(@as(u64, out[at + 3]), mem); + print_dec(@as(u64, out[at + 3] % 16), mem); + say(" key "); + if (out[at + 3] / 16 == 0) { say("-"); } else { print_dec(@as(u64, out[at + 3] / 16), mem); } say("\n"); at = at + 4; } @@ -1016,13 +1203,14 @@ fn run_install(st: []u64, mem: []u8) -> void { } pub fn main() -> void { - var st : [256]u64 = undefined; - var mem : [1149184]u8 = undefined; + var st : [512]u64 = undefined; + var mem : [1171712]u8 = undefined; var z : usize = 0; while (z < ST_SIZE) { st[z] = 0; z = z + 1; } + keys_init(st[0..ST_SIZE]); sys_signal(13, 1); st[G_HLEN] = @as(u64, env_text("HOME", mem[0 .. MEM_SIZE], O_HOME)); var bsz : [2]u32 = undefined; @@ -1054,5 +1242,12 @@ test the_arena_regions_do_not_overlap { assert O_MISC + 328 <= O_BIN; assert O_BIN + 256 <= O_RING; assert O_RING + 16 * 65536 <= O_KEYS; - assert O_KEYS + 9 * KEY_CAP <= MEM_SIZE; + // the key table holds MAX_KEYS keys for each of the PROVIDER_COUNT providers, then the screen carry rows + // and the scratch for a chunk's plain text; main() sizes its arrays with these two literals + assert O_KEYS + @as(usize, PROVIDER_COUNT) * @as(usize, MAX_KEYS) * KEY_CAP <= O_CARRY; + assert O_CARRY + 16 * CARRY_SLOT <= O_SCAN; + assert O_SCAN + SCAN_CAP <= MEM_SIZE; + assert MEM_SIZE == 1171712; + assert ST_SIZE == 512; + assert CARRY_SLOT == CARRY_CAP + 4; } From e92e9778e52e5feb6db0da9b4af71204bc26dd47 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 14:01:09 +0700 Subject: [PATCH 2/2] T27 OS keys: every stored key in rotation, failover on overload, restored panes wait for their keys (Closes #11241) Merge of master plus sealing, gates run on the merged tree with t27c from master: tests clean with vacuous passes 0, seal --verify MATCH, tri types ratchet CLEAN, dupe_scan failure set a subset of master's. Co-Authored-By: Claude Sonnet 5.5 --- .trinity/seals/t27os_T27osKeys.json | 37 +++++++++++++++++ .trinity/seals/t27os_T27osPtyLogic.json | 53 +++++++++++++++++++++++++ 2 files changed, 90 insertions(+) create mode 100644 .trinity/seals/t27os_T27osKeys.json create mode 100644 .trinity/seals/t27os_T27osPtyLogic.json diff --git a/.trinity/seals/t27os_T27osKeys.json b/.trinity/seals/t27os_T27osKeys.json new file mode 100644 index 0000000000..cc3464fce5 --- /dev/null +++ b/.trinity/seals/t27os_T27osKeys.json @@ -0,0 +1,37 @@ +{ + "built_by": "t27c-bootstrap@0.5.2+42a4860c4", + "closure": [ + { + "spec": "specs/t27os/agents.t27", + "spec_hash": "sha256:c4b89c1bd1dcb751fada2ae1f0330e250d17c5085ea73acdc38fac13548277ee" + }, + { + "spec": "specs/t27os/session_store.t27", + "spec_hash": "sha256:7d05d7dda14587458a77291c3d587e3f08516d8e678f0a4bd06dcaab7f6d1f2f" + } + ], + "config": "gen=c,rust,verilog,zig;tests=zig", + "gen_hash_c": "sha256:4d2ce37475c5fd9cfdbe564d68df7c92327f40be8d60faa04aa8382217dc60c5", + "gen_hash_rust": "sha256:65387d614d4714c7dff0227fb378c33bc7886c695cc42f4ea24458223de573c8", + "gen_hash_verilog": "sha256:9e29c63b98f00cd92e6b0466d356a9d3911bf444f26a652d5e2e6e6b576c6ade", + "gen_hash_zig": "sha256:d244fcdfb0f95fcf8a2742598dbc84d7e66c6c3aec8b4bc632483753564f8230", + "module": "T27osKeys", + "ring": 12, + "seal_schema": 2, + "sealed_at": "2026-10-20T07:00:49Z", + "sealed_by": "t27c-bootstrap@0.5.2", + "spec_hash": "sha256:bc12487689ca25d46a0155a6cbb30e36983c4898a7e28486266d084c570f8d43", + "spec_path": "specs/t27os/keys.t27", + "tests": { + "failed": 0, + "failing": [], + "forced": false, + "passed": 63, + "total": 63, + "vacuous": 0 + }, + "toolchain": { + "test_runner": "sha256:8512b80da533b79289110fca4a65f3336abd7a059ca4b72ea595d5e33ff8ad10", + "zig": "0.16.0" + } +} \ No newline at end of file diff --git a/.trinity/seals/t27os_T27osPtyLogic.json b/.trinity/seals/t27os_T27osPtyLogic.json new file mode 100644 index 0000000000..7e60f7c435 --- /dev/null +++ b/.trinity/seals/t27os_T27osPtyLogic.json @@ -0,0 +1,53 @@ +{ + "built_by": "t27c-bootstrap@0.5.2+42a4860c4", + "closure": [ + { + "spec": "specs/t27os/agents.t27", + "spec_hash": "sha256:c4b89c1bd1dcb751fada2ae1f0330e250d17c5085ea73acdc38fac13548277ee" + }, + { + "spec": "specs/t27os/editors.t27", + "spec_hash": "sha256:9b4e3c083e7aa0d3d3b42ac77f6349820e123d34617abbdfc85358cd22420ede" + }, + { + "spec": "specs/t27os/keys.t27", + "spec_hash": "sha256:bc12487689ca25d46a0155a6cbb30e36983c4898a7e28486266d084c570f8d43" + }, + { + "spec": "specs/t27os/pane_state.t27", + "spec_hash": "sha256:c7c7ca610b0f69c66dd5518c93f045bfa3f666c1849c12ca2de0c6542d73d1fc" + }, + { + "spec": "specs/t27os/pty_proto.t27", + "spec_hash": "sha256:fe9dee96f99eccaa6273124bb9022a66ca916bc9a0f6c3f415c2f4b6ab964ab1" + }, + { + "spec": "specs/t27os/session_store.t27", + "spec_hash": "sha256:7d05d7dda14587458a77291c3d587e3f08516d8e678f0a4bd06dcaab7f6d1f2f" + } + ], + "config": "gen=c,rust,verilog,zig;tests=zig", + "gen_hash_c": "sha256:a1ba1cc53e826cbfe4123efe840b6b7d8ba3b3b9bcec14bcea82a64e6d77bad7", + "gen_hash_rust": "sha256:b1f9544872abbca303b2b4565a357e4134da53bf232b7efcf2d729d13c3a987a", + "gen_hash_verilog": "sha256:9c28b828e7de42515be178c72ec47fea0b3a20cb6840e9c1a4fbf8085862e08d", + "gen_hash_zig": "sha256:39ac7d141b3d9832cc4ec5e92e9f73e2cf030c4379c4f3253696d46f159efb62", + "module": "T27osPtyLogic", + "ring": 12, + "seal_schema": 2, + "sealed_at": "2026-10-20T07:00:53Z", + "sealed_by": "t27c-bootstrap@0.5.2", + "spec_hash": "sha256:401e66c84055044c98546c093cbdb677ae5cb5203d524b01e172b37a7afc1e33", + "spec_path": "specs/t27os/pty_logic.t27", + "tests": { + "failed": 0, + "failing": [], + "forced": false, + "passed": 34, + "total": 34, + "vacuous": 0 + }, + "toolchain": { + "test_runner": "sha256:8512b80da533b79289110fca4a65f3336abd7a059ca4b72ea595d5e33ff8ad10", + "zig": "0.16.0" + } +} \ No newline at end of file