diff --git a/.trinity/seals/t27os_T27osKeys.json b/.trinity/seals/t27os_T27osKeys.json index 07f7a832f9..9a3b0dad56 100644 --- a/.trinity/seals/t27os_T27osKeys.json +++ b/.trinity/seals/t27os_T27osKeys.json @@ -1,5 +1,5 @@ { - "built_by": "t27c-bootstrap@0.5.2+e3bb9ffec", + "built_by": "t27c-bootstrap@0.5.2+2d3203e90", "closure": [ { "spec": "specs/t27os/agents.t27", @@ -15,23 +15,23 @@ } ], "config": "gen=c,rust,verilog,zig;tests=zig", - "gen_hash_c": "sha256:4da05e755130405b7f85bda4fc5d3456c3638bf567211955a00c2b35fb3448af", - "gen_hash_rust": "sha256:51146ffb9d4d9e2fefc266395eb6ed161971ab4dc9ca949b0650cd008c19e9ce", - "gen_hash_verilog": "sha256:2f1f76f240eb66cd9bee6e5718325f36892e3721ec4d848a670a50c034a484de", - "gen_hash_zig": "sha256:b6aa8439522547b063b1468735ade4eeda04afb2f5624f8a27e59857b8030867", + "gen_hash_c": "sha256:b423ffe1f6419b847847470552e43a3ec33ebee2d84e96e81d4e87064729ac1f", + "gen_hash_rust": "sha256:215fefc28d29127a94fbb24ae3dc56cc7a9ca1e44e97ee4dca1132fbfc1c296e", + "gen_hash_verilog": "sha256:0a213597aed4550a80b99d34c22367b58af23c1221e58a8e67dd83f768de08bf", + "gen_hash_zig": "sha256:8057d21cc4de8ffc9989b4f5848fc72a155f337939c9bb7671590c488d316679", "module": "T27osKeys", "ring": 12, "seal_schema": 2, - "sealed_at": "2026-10-20T07:50:02Z", + "sealed_at": "2026-10-20T09:05:17Z", "sealed_by": "t27c-bootstrap@0.5.2", - "spec_hash": "sha256:35efa3efc95753da265df018e8931d7de5366081a54649e153e074dfd1bae697", + "spec_hash": "sha256:3e51f65c621494d2e4ef87f89b592ed350831804a46822684f624db7891b9577", "spec_path": "specs/t27os/keys.t27", "tests": { "failed": 0, "failing": [], "forced": false, - "passed": 81, - "total": 81, + "passed": 86, + "total": 86, "vacuous": 0 }, "toolchain": { diff --git a/.trinity/seals/t27os_T27osPtyLogic.json b/.trinity/seals/t27os_T27osPtyLogic.json index 1e509380b1..06b37d4a2c 100644 --- a/.trinity/seals/t27os_T27osPtyLogic.json +++ b/.trinity/seals/t27os_T27osPtyLogic.json @@ -1,5 +1,5 @@ { - "built_by": "t27c-bootstrap@0.5.2+1417e3bee", + "built_by": "t27c-bootstrap@0.5.2+2d3203e90", "closure": [ { "spec": "specs/t27os/agents.t27", @@ -11,7 +11,7 @@ }, { "spec": "specs/t27os/keys.t27", - "spec_hash": "sha256:35efa3efc95753da265df018e8931d7de5366081a54649e153e074dfd1bae697" + "spec_hash": "sha256:3e51f65c621494d2e4ef87f89b592ed350831804a46822684f624db7891b9577" }, { "spec": "specs/t27os/pane_state.t27", @@ -27,23 +27,23 @@ } ], "config": "gen=c,rust,verilog,zig;tests=zig", - "gen_hash_c": "sha256:ea264f26eee4a971f4fbf7cd9f7de6de72f55cfe82fa9ae98e5ccaa3cfe9f922", - "gen_hash_rust": "sha256:54815c023a8034256634bffc1894fb481027faba8e8915c0c674363efa8877db", - "gen_hash_verilog": "sha256:2ac6b6aa132ffa0bd68f68b896247fb411d29e9d736eb39ab725cd42c563a553", - "gen_hash_zig": "sha256:f5f02c802c02af613dd90592c3e77dd525cc64da594a57c9274f66208a2cbbf3", + "gen_hash_c": "sha256:08b4f5a76814d98a09ecf6ca924ac7d47ec59c52950c40e7af7dfe6faeaa7b92", + "gen_hash_rust": "sha256:446e88354984b34b59cc6166aa86d5ae52e7b6f83d72277d7a1279458a6f439b", + "gen_hash_verilog": "sha256:f63ab08ec6b9dc43336c69a4a8bb6ddd103039433c5815914b55ee7fe17a412d", + "gen_hash_zig": "sha256:17f6f770abf17c72f25b1c103e6d6f4235ea228db300cfce8ebf5db6c352b742", "module": "T27osPtyLogic", "ring": 12, "seal_schema": 2, - "sealed_at": "2026-10-20T08:04:30Z", + "sealed_at": "2026-10-20T09:05:23Z", "sealed_by": "t27c-bootstrap@0.5.2", - "spec_hash": "sha256:2167ea0959df3a657b97cf7055bf89f622f379b71e73852a3da1d0bacf1e53fa", + "spec_hash": "sha256:c2559b3d076400e0c401bb0bf8ea44eaef9b713ce57a8400e8d0c610dfb6d355", "spec_path": "specs/t27os/pty_logic.t27", "tests": { "failed": 0, "failing": [], "forced": false, - "passed": 37, - "total": 37, + "passed": 45, + "total": 45, "vacuous": 0 }, "toolchain": { diff --git a/specs/t27os/keys.t27 b/specs/t27os/keys.t27 index 08109a1512..7affbaf903 100644 --- a/specs/t27os/keys.t27 +++ b/specs/t27os/keys.t27 @@ -660,12 +660,28 @@ pub const TI_OPUS : u32 = 4; pub const TIER_COUNT : u32 = 5; /// The variable each tier is set through (ANTHROPIC_SMALL_FAST_MODEL is the older name of the haiku slot). pub const TIER_VARS : str = "ANTHROPIC_MODEL,ANTHROPIC_SMALL_FAST_MODEL,ANTHROPIC_DEFAULT_HAIKU_MODEL,ANTHROPIC_DEFAULT_SONNET_MODEL,ANTHROPIC_DEFAULT_OPUS_MODEL"; -/// Only the flash model answered on the second z.ai key (owner, 2026-10-19), so every tier of the zai -/// profile names it. The z.ai page https://docs.z.ai/devpack/tool/claude (read 2026-10-20) lists bigger -/// models for the opus and sonnet tiers; they join here as one data edit when a key that answers them is stored. +/// Only the free flash models answer on the owner's z.ai keys, so every tier of the zai profile names the +/// free flash tier's models; the paid ids (glm-5.2 and the like) answer 1113 "Insufficient balance or no +/// resource package" on every key. The z.ai page https://docs.z.ai/devpack/tool/claude (read 2026-10-20) +/// lists bigger models for the opus and sonnet tiers; they join here as one data edit when a key that +/// answers them is stored. +/// MEASURED 2026-10-20T08:49Z on the bench Mac (gHashTag/t27#11284): one 1-token call per Keychain key of +/// service ai.t27.os, accounts zai and zai#2 to zai#7 (seven keys, no key value is written here): +/// glm-4.5-flash HTTP 200 on 7 of 7 keys -> the first model of the tier, the default +/// glm-4.7-flash HTTP 200 on 2 of 7 keys (zai#3 and zai#6), HTTP 529 code 1305 (overloaded) on the other +/// 5 of 7 -> the second model of the tier +/// glm-5.2 and the other paid ids: 1113 on all keys (earlier measurement) -> not in the map at all. +/// A pane (new or restored) starts on the default; on 429, 1113, 1305 or 401/403 it rotates over every key of +/// the provider (the failed key resting), then takes the next model of the tier (MODELS_SECOND) and rotates +/// over the keys again, then leaves for the next profile of the chain (fo_action). /// Fields: zai, anthropic-api, openrouter, ollama-cloud. OpenRouter's page gives the three tier names and no /// default (so ANTHROPIC_MODEL stays unset there); Ollama's page names one cloud model, used for every tier. -pub const MODELS_DEFAULT : str = "glm-4.7-flash,,,glm-5.3-flash"; +pub const MODELS_DEFAULT : str = "glm-4.5-flash,,,glm-5.3-flash"; +/// The second model of each profile's tier (empty = the profile has one model): what a pane moves to when +/// every key of its provider failed on the default. The same fields as MODELS_DEFAULT. +pub const MODELS_SECOND : str = "glm-4.7-flash,,,"; +/// How many models one profile's tier map can hold: step 0 is MODELS_DEFAULT, step 1 is MODELS_SECOND. +pub const MODEL_STEPS : u32 = 2; pub const MODELS_HAIKU : str = ",,~anthropic/claude-haiku-latest,"; pub const MODELS_SONNET : str = ",,~anthropic/claude-sonnet-latest[1m],"; pub const MODELS_OPUS : str = ",,~anthropic/claude-opus-latest[1m],"; @@ -726,6 +742,63 @@ pub fn tier_model(pr: u32, ti: u32, out: []u8) -> usize { return csv_field(MODELS_DEFAULT, pr, out); } +/// The model profile pr serves for tier ti at model step `step` (0 = MODELS_DEFAULT: tier_model; 1 = the +/// second model of the tier, MODELS_SECOND, which every tier of that profile names); 0 when there is none. +pub fn tier_model_step(pr: u32, ti: u32, step: u32, out: []u8) -> usize { + if (step == 0) { return tier_model(pr, ti, out); } + if (step >= MODEL_STEPS or pr >= PROFILE_COUNT or ti >= TIER_COUNT) { return 0; } + return csv_field(MODELS_SECOND, pr, out); +} + +/// How many models profile pr's tier map holds: 1 or 2 (0 for no profile). +pub fn profile_models(pr: u32) -> u32 { + if (pr >= PROFILE_COUNT) { return 0; } + var tmp : [64]u8 = undefined; + if (csv_field(MODELS_SECOND, pr, tmp[0..64]) > 0) { return 2; } + return 1; +} + +/// Does the model at `step` have a successor in profile pr's tier map? +pub fn model_left(pr: u32, step: u32) -> bool { + return step + 1 < profile_models(pr); +} + +/// Does another profile (not pr) serve exactly `model` at some step and tier? Such an id is a gateway's own +/// name for a model, not something profile pr can answer. +fn served_elsewhere(pr: u32, model: []const u8) -> bool { + var tmp : [64]u8 = undefined; + var q : u32 = 0; + while (q < PROFILE_COUNT) { + var s : u32 = 0; + while (s < MODEL_STEPS and q != pr) { + var t : u32 = 0; + while (t < TIER_COUNT) { + const n : usize = tier_model_step(q, t, s, tmp[0..64]); + if (n == model.len and n > 0 and begins_with(model, tmp[0..n])) { return true; } + t = t + 1; + } + s = s + 1; + } + q = q + 1; + } + return false; +} + +/// The model a claude pane that is restored, or started again by a failover, runs on under profile pr at +/// model step `step`: the one a fresh spawn would compute (the tier map's default model at that step), copied +/// into out (64 bytes are enough); never the model a saved record or an earlier profile left behind. A +/// profile that names no model of its own (the direct Anthropic API, OpenRouter) lets Claude Code keep its own +/// names, so the saved model stays unless another profile's tier map serves it (a z.ai id has no place on the +/// Anthropic API). No profile (a pane without a key) keeps the saved model. 0 = pass no model. +pub fn model_for_profile(pr: u32, step: u32, saved: []const u8, out: []u8) -> usize { + if (pr < PROFILE_COUNT) { + const n : usize = tier_model_step(pr, TI_DEFAULT, step, out); + if (n > 0) { return n; } + if (served_elsewhere(pr, saved)) { return 0; } + } + return put_bytes(out, 0, saved); +} + /// The base URL of profile pr (0 = none: the direct Anthropic API). pub fn profile_base_url(pr: u32, out: []u8) -> usize { if (pr >= PROFILE_COUNT) { return 0; } @@ -785,9 +858,14 @@ pub fn profile_var_is_key(pr: u32, k: u32) -> bool { /// The constant value of variable k (0 for the key variable; a variable that is present with value 0 is set /// to the empty string). pub fn profile_var_text(pr: u32, k: u32, out: []u8) -> usize { + return profile_var_text_step(pr, k, 0, out); +} + +/// profile_var_text for a pane at model step `step`: the tier variables name that step's model. +pub fn profile_var_text_step(pr: u32, k: u32, step: u32, out: []u8) -> usize { const s : u32 = profile_slot(pr, k); if (s == PS_BASE) { return profile_base_url(pr, out); } - if (s >= PS_TIER0 and s < PS_TIER0 + TIER_COUNT) { return tier_model(pr, s - PS_TIER0, out); } + if (s >= PS_TIER0 and s < PS_TIER0 + TIER_COUNT) { return tier_model_step(pr, s - PS_TIER0, step, out); } return 0; } @@ -948,9 +1026,13 @@ pub const FA_KEEP : u32 = 0; pub const FA_RESPAWN_NEXT : u32 = 1; pub const FA_RESPAWN_KEY : u32 = 2; pub const FA_GIVE_UP : u32 = 3; +/// The same profile and the next model of its tier (MODELS_SECOND): every key of the provider failed on the +/// model the pane ran. Only pty_logic plan_failover gives it (fo_next_model); fo_action never does. +pub const FA_RESPAWN_MODEL : u32 = 4; -/// At most this many failovers per pane: every profile once, and a few key swaps inside a provider. -pub const MAX_FAILOVERS : u32 = 12; +/// At most this many failovers per pane: every key of a provider on each of the MODEL_STEPS models of its tier +/// (MAX_KEYS * 2 = 16), then every profile once, and a few spare. +pub const MAX_FAILOVERS : u32 = 24; /// The index of the first occurrence of needle in hay at or after `from`; hay.len + 1 when absent. fn find_bytes(hay: []const u8, needle: []const u8, from: usize) -> usize { @@ -1018,16 +1100,8 @@ pub fn begins_with(hay: []const u8, needle: []const u8) -> bool { return true; } -/// Is this screen line the error row Claude Code prints, and the final one? classify_line may be run on it -/// only then, never on the body of the transcript: a diff or a quoted line that shows "Insufficient balance" -/// or "API Error: 429" starts with something else (a +, a quote, a word) and would otherwise make the pane -/// leave a healthy provider. Leading blanks are skipped. A row that says "Retrying" is Claude Code's own retry -/// in flight and the turn is still alive; the failover waits for the final row, when nothing is left to lose. -pub fn error_row(line: []const u8) -> bool { - var a : usize = 0; - while (a < line.len and line[a] == 32) { a = a + 1; } - const t : []const u8 = line[a .. line.len]; - if (has_bytes(t, "Retrying")) { return false; } +/// Does this line start the way Claude Code's error rows do? +fn error_start(t: []const u8) -> bool { if (begins_with(t, "API Error")) { return true; } if (begins_with(t, "Error")) { return true; } if (begins_with(t, "Request failed")) { return true; } @@ -1035,6 +1109,24 @@ pub fn error_row(line: []const u8) -> bool { return begins_with(t, "Insufficient balance"); } +/// Is this screen line the error row Claude Code prints? classify_line may be run on it only then, never on +/// the body of the transcript: a diff or a quoted line that shows "Insufficient balance" or "API Error: 429" +/// starts with something else (a +, a quote, a word) and would otherwise make the pane leave a healthy +/// provider. Leading blanks are skipped. A row that says "Retrying" is Claude Code's own retry in flight and +/// the turn is still alive, so for a throttle or an overload the failover waits for the final row. A retry +/// cannot cure an empty account, a disabled subscription or a rejected key, though: z.ai answers an empty +/// account with HTTP 429 and code 1113, Claude Code retries that ten times (minutes), and the retry row is the +/// only row on the screen meanwhile (gHashTag/t27#11284). Such a row counts at once. +pub fn error_row(line: []const u8) -> bool { + var a : usize = 0; + while (a < line.len and line[a] == 32) { a = a + 1; } + const t : []const u8 = line[a .. line.len]; + if (error_start(t) == false) { return false; } + if (has_bytes(t, "Retrying") == false) { return true; } + const c : u32 = classify_line(t); + return c == FC_NO_BALANCE or c == FC_SUBSCRIPTION or c == FC_LOGIN_REJECTED; +} + /// The bad mask after a failure of class `class` on profile pr: the profile's provider joins it. A line that /// is no failure (FC_NONE, FC_OTHER) or a pane with no profile changes nothing. pub fn fo_bad(class: u32, pr: u32, bad: u32) -> u32 { @@ -1049,13 +1141,15 @@ pub fn fo_bad(class: u32, pr: u32, bad: u32) -> u32 { /// started without a key, which is where the "disabled Claude subscription access" line comes from). /// `stored` and `bad` are the provider masks (bad: providers that failed for this pane or their check), /// `done` the failovers this pane has had, `alt` whether another key of pr's provider is usable now -/// (rotate_pick_except over the keys, the failed one skipped). A throttled (429) or overloaded (529) key with -/// another key at hand swaps the key and keeps the profile; every other failure leaves the profile for the -/// next one in the chain; when none is left, or the cap is reached, the pane stays as it is. +/// (rotate_pick_except over the keys, the failed one skipped). A throttled (429), overloaded (529), empty +/// (1113) or rejected (401/403) key with another key at hand swaps the key and keeps the profile: the keys +/// are separate accounts (the owner stores several z.ai keys), so one empty key says nothing about the next. +/// Every other failure, and a pane with no key of its provider left to try, leaves the profile for the next +/// one in the chain; when none is left, or the cap is reached, the pane stays as it is. pub fn fo_action(class: u32, pr: u32, stored: u32, bad: u32, done: u32, alt: bool) -> u32 { if (class == FC_NONE or class == FC_OTHER) { return FA_KEEP; } if (done >= MAX_FAILOVERS) { return FA_GIVE_UP; } - if ((class == FC_RATE_LIMITED or class == FC_OVERLOADED) and alt and pr < PROFILE_COUNT) { return FA_RESPAWN_KEY; } + if (class != FC_SUBSCRIPTION and alt and pr < PROFILE_COUNT) { return FA_RESPAWN_KEY; } if (next_profile(pr, stored, fo_bad(class, pr, bad)) == PR_NONE) { return FA_GIVE_UP; } return FA_RESPAWN_NEXT; } @@ -1069,6 +1163,17 @@ pub fn fo_target(class: u32, pr: u32, stored: u32, bad: u32, done: u32, alt: boo return PR_NONE; } +/// Should a pane that ran model `step` of profile pr move to the next model of the same profile? Yes when +/// the failure is one a key or a model answers (throttled, overloaded, empty, rejected), no key of the +/// provider is left to try on this model (`alt` false), the tier map has a next model (model_left) and the +/// cap is not reached. fo_action has already chosen a key swap when a key was left; this is the step before it +/// leaves the profile. +pub fn fo_next_model(class: u32, pr: u32, step: u32, done: u32, alt: bool) -> bool { + if (class == FC_NONE or class == FC_OTHER or class == FC_SUBSCRIPTION) { return false; } + if (alt or done >= MAX_FAILOVERS) { return false; } + return model_left(pr, step); +} + /// The "why" line for a failover, for the KEYS tab and the pane's title (128 bytes are enough): names the /// profile, the failure and the move. No key value, ever. pub fn fo_why(class: u32, pr: u32, act: u32, nxt: u32, done: u32, out: []u8) -> usize { @@ -1085,6 +1190,7 @@ pub fn fo_why(class: u32, pr: u32, act: u32, nxt: u32, done: u32, out: []u8) -> const w : usize = csv_field(FC_WORDS, class, tmp[0..32]); n = put_bytes(out, n, tmp[0..w]); if (act == FA_RESPAWN_KEY) { return put_bytes(out, n, ", same profile with the next key"); } + if (act == FA_RESPAWN_MODEL) { return put_bytes(out, n, ", every key tried, same profile on the next model"); } if (act == FA_RESPAWN_NEXT) { n = put_bytes(out, n, ", respawn on "); const m : usize = profile_name(nxt, tmp[0..32]); @@ -1201,6 +1307,108 @@ test every_tier_of_the_zai_profile_names_the_flash_model { assert tier_model(PR_NONE, TI_DEFAULT, o[0..64]) == 0; } +/// The model of tier ti at step as text, for an assertion (64 bytes are enough). +fn t_model(pr: u32, ti: u32, step: u32, o: []u8) -> usize { + return tier_model_step(pr, ti, step, o); +} + +test the_default_z_ai_model_is_the_free_flash_that_answered_on_every_key { + // measured 2026-10-20T08:49Z: glm-4.5-flash answered 200 on 7 of 7 keys, glm-4.7-flash on 2 of 7 (529 on 5) + var o : [64]u8 = undefined; + var ti : u32 = 0; + while (ti < TIER_COUNT) { + const n : usize = t_model(PR_ZAI, ti, 0, o[0..64]); + assert n == 13; + assert begins_with(o[0..n], "glm-4.5-flash"); + ti = ti + 1; + } + const m : usize = tier_model(PR_ZAI, TI_DEFAULT, o[0..64]); + assert m == 13 and begins_with(o[0..m], "glm-4.5-flash"); + // the second model of the tier, on every tier, only at step 1 + ti = 0; + while (ti < TIER_COUNT) { + const k : usize = t_model(PR_ZAI, ti, 1, o[0..64]); + assert k == 13; + assert begins_with(o[0..k], "glm-4.7-flash"); + ti = ti + 1; + } + // no paid id is in the map at any step or tier + var pr : u32 = 0; + while (pr < PROFILE_COUNT) { + var st : u32 = 0; + while (st < MODEL_STEPS) { + var t : u32 = 0; + while (t < TIER_COUNT) { + const l : usize = t_model(pr, t, st, o[0..64]); + assert has_bytes(o[0..l], "glm-5.2") == false; + t = t + 1; + } + st = st + 1; + } + pr = pr + 1; + } + assert t_model(PR_ZAI, TI_DEFAULT, MODEL_STEPS, o[0..64]) == 0; + assert t_model(PR_ANTHROPIC, TI_DEFAULT, 1, o[0..64]) == 0; + assert t_model(PR_NONE, TI_DEFAULT, 0, o[0..64]) == 0; +} + +test a_profile_knows_how_many_models_its_tier_holds { + assert profile_models(PR_ZAI) == 2; + assert profile_models(PR_ANTHROPIC) == 1; + assert profile_models(PR_OPENROUTER) == 1; + assert profile_models(PR_OLLAMA_CLOUD) == 1; + assert profile_models(PR_NONE) == 0; + assert model_left(PR_ZAI, 0); + assert model_left(PR_ZAI, 1) == false; + assert model_left(PR_ANTHROPIC, 0) == false; + assert model_left(PR_NONE, 0) == false; + assert MODEL_STEPS == 2; + assert csv_fields(MODELS_SECOND) == PROFILE_COUNT; +} + +test the_tier_variables_of_a_pane_name_the_model_of_its_step { + var o : [64]u8 = undefined; + // zai: variable 2 is ANTHROPIC_MODEL; every tier variable follows the pane's step + const a : usize = profile_var_text_step(PR_ZAI, 2, 0, o[0..64]); + assert a == 13 and begins_with(o[0..a], "glm-4.5-flash"); + const b : usize = profile_var_text_step(PR_ZAI, 6, 1, o[0..64]); + assert b == 13 and begins_with(o[0..b], "glm-4.7-flash"); + const c : usize = profile_var_text(PR_ZAI, 2, o[0..64]); + assert c == 13 and begins_with(o[0..c], "glm-4.5-flash"); + // the base URL does not depend on the step + assert profile_var_text_step(PR_ZAI, 0, 1, o[0..64]) == 30; +} + +test a_restored_or_failed_over_pane_runs_the_model_its_profile_serves_never_the_saved_one { + var o : [64]u8 = undefined; + // the record of the live test said glm-5.2: every z.ai key answers 1113 for it, so it is not used + var n : usize = model_for_profile(PR_ZAI, 0, "glm-5.2", o[0..64]); + assert n == 13 and begins_with(o[0..n], "glm-4.5-flash"); + n = model_for_profile(PR_ZAI, 0, "glm-4.7-flash", o[0..64]); + assert n == 13 and begins_with(o[0..n], "glm-4.5-flash"); + n = model_for_profile(PR_ZAI, 0, "", o[0..64]); + assert n == 13 and begins_with(o[0..n], "glm-4.5-flash"); + // the next model of the tier, once every key failed on the first + n = model_for_profile(PR_ZAI, 1, "glm-4.5-flash", o[0..64]); + assert n == 13 and begins_with(o[0..n], "glm-4.7-flash"); + n = model_for_profile(PR_OLLAMA_CLOUD, 0, "glm-5.2", o[0..64]); + assert n == 13 and begins_with(o[0..n], "glm-5.3-flash"); + // a profile with no model of its own keeps a saved model of Anthropic's, and drops one a gateway serves + n = model_for_profile(PR_ANTHROPIC, 0, "claude-opus-5-5", o[0..64]); + assert n == 15 and begins_with(o[0..n], "claude-opus-5-5"); + n = model_for_profile(PR_ANTHROPIC, 0, "glm-4.5-flash", o[0..64]); + assert n == 0; + n = model_for_profile(PR_ANTHROPIC, 0, "glm-4.7-flash", o[0..64]); + assert n == 0; + n = model_for_profile(PR_OPENROUTER, 0, "glm-5.3-flash", o[0..64]); + assert n == 0; + n = model_for_profile(PR_ANTHROPIC, 0, "", o[0..64]); + assert n == 0; + // no profile (a pane that has no key): nothing to move it to, the saved model stays + n = model_for_profile(PR_NONE, 0, "glm-5.2", o[0..64]); + assert n == 7 and begins_with(o[0..n], "glm-5.2"); +} + test the_anthropic_profile_names_no_model_so_claude_code_keeps_its_own { var o : [64]u8 = undefined; var ti : u32 = 0; @@ -1414,6 +1622,14 @@ test only_an_error_row_is_ever_classified_never_the_body_of_the_transcript { assert error_row(" ") == false; // Claude Code's own retry in flight: the turn is alive, the failover waits assert error_row("API Error (529 overloaded_error) Retrying in 4 seconds (attempt 2/10)") == false; + assert error_row("API Error: 429 rate limit reached - Retrying in 8 seconds (attempt 9/10)") == false; + // a retry cannot cure an empty account, a disabled subscription or a rejected key: the row counts at once + assert error_row("API Error: 429 {\"error\":{\"code\":\"1113\",\"message\":\"Insufficient balance or no resource package\"}} - Retrying in 8 seconds (attempt 9/10)"); + assert error_row(" API Error (401 invalid credentials) Retrying in 2 seconds (attempt 1/10)"); + assert error_row("Your organization has disabled Claude subscription access - Retrying in 2 seconds"); + // a diff or a quote that shows a retry row is still no error row + assert error_row("+ \"API Error: 429 1113 Insufficient balance Retrying in 8 seconds (attempt 9/10)\"") == false; + assert error_row("The log said API Error: 429 Insufficient balance Retrying") == false; assert begins_with("abc", "ab"); assert begins_with("abc", "abcd") == false; assert begins_with("abc", ""); @@ -1616,9 +1832,18 @@ test a_throttled_key_with_a_spare_swaps_the_key_and_keeps_the_profile { assert fo_action(FC_RATE_LIMITED, PR_ZAI, 57, 0, 0, true) == FA_RESPAWN_KEY; assert fo_target(FC_RATE_LIMITED, PR_ZAI, 57, 0, 0, true) == PR_ZAI; assert fo_bad(FC_RATE_LIMITED, PR_ZAI, 0) == 16; - assert fo_action(FC_NO_BALANCE, PR_ZAI, 57, 0, 0, true) == FA_RESPAWN_NEXT; - assert fo_action(FC_LOGIN_REJECTED, PR_ZAI, 57, 0, 0, true) == FA_RESPAWN_NEXT; + // an empty or a rejected key is one account's: another key of the provider is tried before the profile is left + assert fo_action(FC_NO_BALANCE, PR_ZAI, 57, 0, 0, true) == FA_RESPAWN_KEY; + assert fo_target(FC_NO_BALANCE, PR_ZAI, 57, 0, 0, true) == PR_ZAI; + assert fo_action(FC_LOGIN_REJECTED, PR_ZAI, 57, 0, 0, true) == FA_RESPAWN_KEY; + // no spare key (or a subscription row, which names no key): the next profile, as before + assert fo_action(FC_NO_BALANCE, PR_ZAI, 57, 0, 0, false) == FA_RESPAWN_NEXT; + assert fo_action(FC_LOGIN_REJECTED, PR_ZAI, 57, 0, 0, false) == FA_RESPAWN_NEXT; + assert fo_action(FC_SUBSCRIPTION, PR_ZAI, 57, 0, 0, true) == FA_RESPAWN_NEXT; assert fo_action(FC_RATE_LIMITED, PR_NONE, 57, 0, 0, true) == FA_RESPAWN_NEXT; + // with only z.ai keys stored an empty key used to end in a give-up although other keys were at hand + assert fo_action(FC_NO_BALANCE, PR_ZAI, 16, 0, 0, true) == FA_RESPAWN_KEY; + assert fo_action(FC_NO_BALANCE, PR_ZAI, 16, 0, 0, false) == FA_GIVE_UP; } test a_pane_without_a_key_takes_the_first_stored_profile { @@ -1690,12 +1915,37 @@ test the_failover_why_line_names_the_profile_and_the_move_and_never_a_key { assert has_bytes(o[0..n], "failover limit reached"); } +test a_pane_with_no_key_left_on_its_model_takes_the_next_model_of_the_tier_before_the_next_profile { + // z.ai has two models: key swaps first (fo_action), then the next model, then fo_action's profile chain + assert fo_action(FC_NO_BALANCE, PR_ZAI, 57, 0, 3, true) == FA_RESPAWN_KEY; + assert fo_next_model(FC_NO_BALANCE, PR_ZAI, 0, 3, true) == false; + assert fo_next_model(FC_NO_BALANCE, PR_ZAI, 0, 3, false); + assert fo_next_model(FC_RATE_LIMITED, PR_ZAI, 0, 3, false); + assert fo_next_model(FC_OVERLOADED, PR_ZAI, 0, 3, false); + assert fo_next_model(FC_LOGIN_REJECTED, PR_ZAI, 0, 3, false); + // on its last model, a profile with one model, a row that names no key, or at the cap: no model step + assert fo_next_model(FC_NO_BALANCE, PR_ZAI, 1, 3, false) == false; + assert fo_next_model(FC_NO_BALANCE, PR_ANTHROPIC, 0, 3, false) == false; + assert fo_next_model(FC_NO_BALANCE, PR_NONE, 0, 3, false) == false; + assert fo_next_model(FC_SUBSCRIPTION, PR_ZAI, 0, 3, false) == false; + assert fo_next_model(FC_NONE, PR_ZAI, 0, 3, false) == false; + assert fo_next_model(FC_OTHER, PR_ZAI, 0, 3, false) == false; + assert fo_next_model(FC_NO_BALANCE, PR_ZAI, 0, MAX_FAILOVERS, false) == false; + // the cap leaves room for every key of a provider on each model of its tier, then every profile + assert MAX_FAILOVERS >= MAX_KEYS * MODEL_STEPS + PROFILE_COUNT; + var o : [128]u8 = undefined; + const n : usize = fo_why(FC_NO_BALANCE, PR_ZAI, FA_RESPAWN_MODEL, PR_ZAI, 3, o[0..128]); + assert has_bytes(o[0..n], "claude: zai out of balance, every key tried, same profile on the next model"); + assert n < 128; +} + test the_profile_tables_have_one_field_per_profile { assert csv_fields(PROFILE_NAMES) == PROFILE_COUNT; assert csv_fields(MODELS_DEFAULT) == PROFILE_COUNT; assert csv_fields(MODELS_HAIKU) == PROFILE_COUNT; assert csv_fields(MODELS_SONNET) == PROFILE_COUNT; assert csv_fields(MODELS_OPUS) == PROFILE_COUNT; + assert csv_fields(MODELS_SECOND) == PROFILE_COUNT; assert csv_fields(PROFILE_BASE_URLS) == PROFILE_COUNT; assert csv_fields(PROFILE_KEY_VARS) == PROFILE_COUNT; assert csv_fields(TIER_VARS) == TIER_COUNT; diff --git a/specs/t27os/pty_logic.t27 b/specs/t27os/pty_logic.t27 index aee8e51731..35e4bb833f 100644 --- a/specs/t27os/pty_logic.t27 +++ b/specs/t27os/pty_logic.t27 @@ -126,6 +126,12 @@ use t27os::keys::FA_KEEP; use t27os::keys::FA_RESPAWN_NEXT; use t27os::keys::FA_RESPAWN_KEY; use t27os::keys::FA_GIVE_UP; +use t27os::keys::FA_RESPAWN_MODEL; +use t27os::keys::fo_next_model; +use t27os::keys::model_for_profile; +use t27os::keys::tier_model_step; +use t27os::keys::TI_DEFAULT; +use t27os::keys::profile_var_text_step; use t27os::keys::setkey_provider; use t27os::keys::setkey_number; use t27os::editors::TERM_COUNT; @@ -175,6 +181,10 @@ pub const G_START : usize = 227; /// The millisecond the server last read the Keychain itself (pty_server load_keys); 0 = never. Word 228 was /// free: G_START is 227 and the connection tables begin at G_CFD. pub const G_KLOAD : usize = 228; +/// The model step of every pane (keys.t27 MODEL_STEPS, tier_model_step), one nibble per pane, pane i at +/// 16^i: 0 = the default model of the pane's profile, 1 = the second model of its tier (gHashTag/t27#11284). +/// Word 229 was free; the connection tables begin at G_CFD. +pub const G_MSTEP : usize = 229; pub const G_KLAST : usize = 256; pub const G_KREST : usize = 328; pub const G_PANEK : usize = 400; @@ -469,8 +479,20 @@ pub fn key_slot(p: u32, k: u32) -> usize { return O_KEYS + key_index(p, k) * KEY_CAP; } -/// A pane that has chosen nothing: no keys, no profile, no failures, nothing to skip, nothing waiting. +/// The model step of pane i (0 = the default model of its profile). +pub fn pane_mstep_get(st: []const u64, i: usize) -> u32 { + return @as(u32, @intCast(st[G_MSTEP] / pow16(@as(u32, @intCast(i))) % 16)); +} + +pub fn pane_mstep_set(st: []u64, i: usize, v: u32) -> void { + const m : u64 = pow16(@as(u32, @intCast(i))); + st[G_MSTEP] = st[G_MSTEP] - @as(u64, pane_mstep_get(st, i)) * m + @as(u64, v) * m; +} + +/// A pane that has chosen nothing: no keys, no profile, no failures, nothing to skip, nothing waiting, and the +/// default model of its tier (a new pane, and a pane restored from its record, start on it). pub fn pane_keys_reset(st: []u64, i: usize) -> void { + pane_mstep_set(st, i, 0); const pk : usize = G_PANEK + i * PK; st[pk + PK_KEYS] = 0; st[pk + PK_PROF] = PK_UNSET; @@ -548,6 +570,17 @@ pub fn key_pick_free(st: []const u64, p: u32, skip: u32, now_ms: u64) -> u32 { return rotate_pick_except(MAX_KEYS, skip, st[G_KLAST + at .. G_KLAST + at + 8], st[G_KREST + at .. G_KREST + at + 8], now_ms / 1000); } +/// Every key of provider p is free again: a rest was earned on the model a pane ran, and the next model of the +/// tier answers on its own (z.ai: the same key that is overloaded on glm-4.7-flash answers glm-4.5-flash). +/// A slot that holds no key stays empty. +pub fn key_rests_lift(st: []u64, p: u32) -> void { + var k : u32 = 0; + while (k < MAX_KEYS) { + if (key_present(st, p, k)) { st[G_KREST + key_index(p, k)] = 0; } + k = k + 1; + } +} + pub fn key_used(st: []u64, p: u32, k: u32, now_ms: u64) -> void { st[G_KLAST + key_index(p, k)] = now_ms; } @@ -638,11 +671,24 @@ pub fn plan_pane_keys(st: []u64, i: usize, a: u32, now_ms: u64) -> void { } } +/// How long a give-up leaves the pane's screen unread: at least GIVEUP_QUIET_MS, and at least as long as the +/// failed key rests (keys.t27 key_rest_after: ten minutes for an empty account, an hour for a rejected key). +/// Until a rest ends there is no key to move to, so reading the same error row again would only write the +/// same why line again: the give-up is said once. +pub fn giveup_quiet_ms(class: u32) -> u64 { + const rest : u64 = key_rest_after(class) * 1000; + if (rest > GIVEUP_QUIET_MS) { return rest; } + return GIVEUP_QUIET_MS; +} + /// The failover for pane i whose screen showed failure `class` at `now_ms`. The key the pane used rests by the /// kind of failure (keys.t27 key_rest_after), for every pane, not just this one. fo_action decides; on a respawn /// the pane's profile, bad mask, failover count and key to skip are set for plan_pane_keys, and the why line /// (keys.t27 fo_why, 128 bytes are enough) goes to `why` with its length in wl[0]. After a give-up the screen -/// is not read again for GIVEUP_QUIET_MS. Returns the action; FA_RESPAWN_* means "stop the process, launch again". +/// is not read again for giveup_quiet_ms (never retrying a key that rests). Returns the action; FA_RESPAWN_* +/// means "stop the process, launch again": FA_RESPAWN_KEY on the next key of the provider (the failed one +/// skipped), FA_RESPAWN_MODEL on the next model of the tier with every key free again (no key was left on this +/// model, fo_next_model), FA_RESPAWN_NEXT on the next profile of the chain (model step 0 again). pub fn plan_failover(st: []u64, i: usize, class: u32, now_ms: u64, why: []u8, wl: []u64) -> u32 { const pk : usize = G_PANEK + i * PK; var pr : u32 = PR_NONE; @@ -660,25 +706,49 @@ pub fn plan_failover(st: []u64, i: usize, class: u32, now_ms: u64, why: []u8, wl if (cur < MAX_KEYS and rest > 0) { key_rest(st, prov, cur, now_ms / 1000 + rest); } alt = key_pick_free(st, prov, cur, now_ms) != MAX_KEYS; } - const act : u32 = fo_action(class, pr, stored, bad, done, alt); + const step : u32 = pane_mstep_get(st, i); + var act : u32 = fo_action(class, pr, stored, bad, done, alt); + if (fo_next_model(class, pr, step, done, alt)) { act = FA_RESPAWN_MODEL; } var nxt : u32 = PR_NONE; if (act == FA_RESPAWN_KEY) { nxt = pr; st[pk + PK_SKIP] = @as(u64, cur) + 1; } + if (act == FA_RESPAWN_MODEL) { + nxt = pr; + pane_mstep_set(st, i, step + 1); + key_rests_lift(st, PROFILE_PROVIDER[pr]); + } if (act == FA_RESPAWN_NEXT) { nxt = fo_target(class, pr, stored, bad, done, alt); st[pk + PK_BAD] = @as(u64, fo_bad(class, pr, bad)); + pane_mstep_set(st, i, 0); } - if (act == FA_RESPAWN_KEY or act == FA_RESPAWN_NEXT) { + if (act == FA_RESPAWN_KEY or act == FA_RESPAWN_NEXT or act == FA_RESPAWN_MODEL) { st[pk + PK_PROF] = @as(u64, nxt); st[pk + PK_DONE] = @as(u64, done) + 1; } - if (act == FA_GIVE_UP) { st[pk + PK_STOP] = now_ms + GIVEUP_QUIET_MS; } + if (act == FA_GIVE_UP) { st[pk + PK_STOP] = now_ms + giveup_quiet_ms(class); } wl[0] = @as(u64, fo_why(class, pr, act, nxt, done, why)); return act; } +/// The model pane i (agent a) starts with, written into out (64 bytes are enough); 0 = pass no model. Call it +/// after plan_pane_keys, which chose the pane's profile. A pane that is restored from its record, started +/// again after a failover, or relaunched after a resume that died gets the model a fresh spawn of its +/// profile computes (keys.t27 model_for_profile at the pane's model step), never the one its record holds: the +/// record may name a model the provider does not serve to these keys (glm-5.2 answers 1113 on all of them), +/// and a model of the profile it just left has no place on the next one. `first` is the first start of a +/// SPAWN, whose payload may name a model on purpose; that one, and the model of every agent but the claude +/// cell (no profile), stay as they are. +pub fn launch_model(st: []const u64, i: usize, a: u32, first: bool, saved: []const u8, out: []u8) -> usize { + if (profile_agent(a) == false or first) { return put_bytes(out, 0, saved); } + var pr : u32 = PR_NONE; + const w : u64 = st[G_PANEK + i * PK + PK_PROF]; + if (w < @as(u64, PROFILE_COUNT)) { pr = @as(u32, @intCast(w)); } + return model_for_profile(pr, pane_mstep_get(st, i), saved, out); +} + // ---- reading a pane's screen: plain text lines out of terminal bytes ---- fn put_break(out: []u8, n: usize) -> usize { @@ -1231,6 +1301,9 @@ test the_server_tables_and_the_arena_line_up_for_eight_keys_a_provider { assert G_START > G_BINLEN; assert G_KLOAD > G_START; assert G_KLOAD < G_CFD; + assert G_MSTEP > G_KLOAD; + assert G_MSTEP < G_CFD; + assert MAX_PANES <= 16; assert 16 * PF <= G_LISTEN; assert key_slot(0, 0) == O_KEYS; assert key_slot(4, 2) == O_KEYS + (4 * 8 + 2) * KEY_CAP; @@ -1359,7 +1432,7 @@ test an_overloaded_key_swaps_to_the_next_key_rests_and_is_passed_over_by_other_p assert st[G_KREST + 34] == 1011 + 30; } -test when_every_z_ai_key_is_overloaded_the_pane_moves_to_the_next_profile_then_gives_up { +test when_every_z_ai_key_is_overloaded_the_pane_takes_the_next_model_then_the_next_profile_then_gives_up { var st : [512]u64 = undefined; var why : [128]u8 = undefined; var wl : [2]u64 = undefined; @@ -1374,37 +1447,59 @@ test when_every_z_ai_key_is_overloaded_the_pane_moves_to_the_next_profile_then_g assert a == FA_RESPAWN_KEY; plan_pane_keys(st[0..512], 0, 0, 1002500); assert pane_key_get(st[0..512], 0, P_ZAI) == 3; - // the third key is overloaded too and the other two still rest: next profile, z.ai marked bad + // the third key is overloaded too and the other two still rest: no key is left on the first model, so + // the pane takes the second model of the tier, and every key is free again for it a = plan_failover(st[0..512], 0, FC_OVERLOADED, 1003000, why[0..128], wl[0..2]); + assert a == FA_RESPAWN_MODEL; + assert t_prof(st[0..512], 0) == PR_ZAI; + assert pane_mstep_get(st[0..512], 0) == 1; + assert st[G_PANEK + PK_BAD] == 0; + assert st[G_KREST + 32] == 0 and st[G_KREST + 33] == 0 and st[G_KREST + 34] == 0; + assert has_bytes(why[0 .. @as(usize, @intCast(wl[0]))], "zai overloaded (529), every key tried, same profile on the next model"); + plan_pane_keys(st[0..512], 0, 0, 1003500); + assert pane_key_get(st[0..512], 0, P_ZAI) == 1; + // the same walk over the keys on the second model + a = plan_failover(st[0..512], 0, FC_OVERLOADED, 1004000, why[0..128], wl[0..2]); + assert a == FA_RESPAWN_KEY; + plan_pane_keys(st[0..512], 0, 0, 1004500); + assert pane_key_get(st[0..512], 0, P_ZAI) == 2; + a = plan_failover(st[0..512], 0, FC_OVERLOADED, 1005000, why[0..128], wl[0..2]); + assert a == FA_RESPAWN_KEY; + plan_pane_keys(st[0..512], 0, 0, 1005500); + assert pane_key_get(st[0..512], 0, P_ZAI) == 3; + // every key failed on the last model of the tier: next profile, z.ai marked bad, the model step is 0 again + a = plan_failover(st[0..512], 0, FC_OVERLOADED, 1006000, why[0..128], wl[0..2]); assert a == FA_RESPAWN_NEXT; assert t_prof(st[0..512], 0) == PR_ANTHROPIC; + assert pane_mstep_get(st[0..512], 0) == 0; assert st[G_PANEK + PK_BAD] == 16; assert has_bytes(why[0 .. @as(usize, @intCast(wl[0]))], "zai overloaded (529), respawn on anthropic-api"); - plan_pane_keys(st[0..512], 0, 0, 1003500); + plan_pane_keys(st[0..512], 0, 0, 1006500); assert pane_key_get(st[0..512], 0, P_ANTHROPIC) == 1; assert pane_key_get(st[0..512], 0, P_ZAI) == 0; // the Anthropic key is overloaded: no profile is left, the pane stays and its screen is not read for a minute - a = plan_failover(st[0..512], 0, FC_OVERLOADED, 1004000, why[0..128], wl[0..2]); + a = plan_failover(st[0..512], 0, FC_OVERLOADED, 1007000, why[0..128], wl[0..2]); assert a == FA_GIVE_UP; - assert st[G_PANEK + PK_DONE] == 3; - assert st[G_PANEK + PK_STOP] == 1004000 + GIVEUP_QUIET_MS; + assert st[G_PANEK + PK_DONE] == 6; + assert st[G_PANEK + PK_STOP] == 1007000 + GIVEUP_QUIET_MS; assert has_bytes(why[0 .. @as(usize, @intCast(wl[0]))], "no profile left in the chain"); - assert scan_ready(st[0..512], 0, 1003500, 1020000) == false; - assert scan_ready(st[0..512], 0, 1003500, 1064001); + assert scan_ready(st[0..512], 0, 1006500, 1020000) == false; + assert scan_ready(st[0..512], 0, 1006500, 1067001); } -test an_empty_balance_leaves_the_provider_at_once_and_the_key_rests_ten_minutes { +test an_empty_balance_swaps_to_the_next_key_and_the_key_rests_ten_minutes { var st : [512]u64 = undefined; var why : [128]u8 = undefined; var wl : [2]u64 = undefined; t_zai3(st[0..512]); plan_pane_keys(st[0..512], 0, 0, 1000000); const a : u32 = plan_failover(st[0..512], 0, FC_NO_BALANCE, 1005000, why[0..128], wl[0..2]); - assert a == FA_RESPAWN_NEXT; + assert a == FA_RESPAWN_KEY; assert st[G_KREST + 32] == 1005 + 600; - assert st[G_PANEK + PK_BAD] == 16; - assert t_prof(st[0..512], 0) == PR_ANTHROPIC; - assert has_bytes(why[0 .. @as(usize, @intCast(wl[0]))], "zai out of balance, respawn on anthropic-api"); + assert st[G_PANEK + PK_BAD] == 0; + assert st[G_PANEK + PK_SKIP] == 1; + assert t_prof(st[0..512], 0) == PR_ZAI; + assert has_bytes(why[0 .. @as(usize, @intCast(wl[0]))], "zai out of balance, same profile with the next key"); // the failure cap ends any loop st[G_PANEK + PK_DONE] = @as(u64, MAX_FAILOVERS); const g : u32 = plan_failover(st[0..512], 0, FC_OVERLOADED, 1006000, why[0..128], wl[0..2]); @@ -1449,6 +1544,252 @@ test a_new_spawn_starts_with_a_clean_pane_record { assert t_prof(st[0..512], 2) == PR_ZAI; } +test the_model_step_of_a_pane_is_its_own_nibble_and_a_reset_pane_starts_on_the_default_model { + var st : [512]u64 = undefined; + t_table(st[0..512]); + assert st[G_MSTEP] == 0; + pane_mstep_set(st[0..512], 0, 1); + pane_mstep_set(st[0..512], 15, 1); + pane_mstep_set(st[0..512], 3, 2); + assert pane_mstep_get(st[0..512], 0) == 1; + assert pane_mstep_get(st[0..512], 15) == 1; + assert pane_mstep_get(st[0..512], 3) == 2; + assert pane_mstep_get(st[0..512], 7) == 0; + pane_mstep_set(st[0..512], 3, 0); + assert pane_mstep_get(st[0..512], 3) == 0; + assert pane_mstep_get(st[0..512], 0) == 1; + // a new spawn and a restored pane both start from pane_keys_reset: the default model of the tier + pane_keys_reset(st[0..512], 0); + assert pane_mstep_get(st[0..512], 0) == 0; + assert pane_mstep_get(st[0..512], 15) == 1; + keys_init(st[0..512]); + assert pane_mstep_get(st[0..512], 15) == 0; + assert st[G_MSTEP] == 0; +} + +// ---- a restored pane (gHashTag/t27#11284): its record held key 1 and the model glm-5.2, and every key answers +// 1113 for glm-5.2. It must start as a fresh spawn would (the free flash model, the key rotation's pick) and +// fail over exactly as a spawned pane does. restore_all leaves a pane in the state pane_keys_reset gives it, +// with the sid and cwd of the record and the saved model in its slot; these tests start there. + +/// Pane i the way restore_all leaves it just before its launch: nothing chosen yet. +fn t_restored(st: []u64, i: usize) -> void { + pane_keys_reset(st, i); +} + +test a_restored_pane_takes_the_key_a_fresh_spawn_would_take_not_the_one_its_record_held { + var st : [512]u64 = undefined; + t_zai3(st[0..512]); + // key 1 earned a rest, key 3 was used lately: the key used longest ago and not resting is key 2 + st[G_KREST + 32] = 1000 + 600; + st[G_KLAST + 32] = 900000; + st[G_KLAST + 33] = 100; + st[G_KLAST + 34] = 500000; + const fresh : u32 = key_pick(st[0..512], P_ZAI, MAX_KEYS, 1000000); + assert fresh == 1; + t_restored(st[0..512], 3); + plan_pane_keys(st[0..512], 3, 0, 1000000); + assert t_prof(st[0..512], 3) == PR_ZAI; + assert pane_key_get(st[0..512], 3, P_ZAI) == fresh + 1; + assert pane_key_get(st[0..512], 3, P_ZAI) == 2; + assert st[G_KLAST + 33] == 1000000; + // with no key in the table at all a restored pane has none, as a spawned one + var empty : [512]u64 = undefined; + t_table(empty[0..512]); + t_restored(empty[0..512], 0); + plan_pane_keys(empty[0..512], 0, 0, 1000000); + assert t_prof(empty[0..512], 0) == PR_NONE; +} + +test a_restored_pane_resumes_its_session_on_the_free_flash_model_not_the_one_its_record_held { + var st : [512]u64 = undefined; + var o : [64]u8 = undefined; + var cmd : [200]u8 = undefined; + t_zai3(st[0..512]); + t_restored(st[0..512], 0); + plan_pane_keys(st[0..512], 0, 0, 1000000); + // the record said glm-5.2; the model the pane starts with is the free flash tier's default + const n : usize = launch_model(st[0..512], 0, 0, false, "glm-5.2", o[0..64]); + assert n == 13; + assert has_bytes(o[0..n], "glm-4.5-flash"); + assert has_bytes(o[0..n], "glm-5.2") == false; + // the session id of the record still resumes; the command names the free model and never the stale one + const c : usize = pane_command(0, R_RESUME, "5f1c9a00-aaaa-bbbb-cccc-0123456789ab", o[0..n], "/h", cmd[0..200]); + assert pl_count(cmd[0..c], "claude --resume 5f1c9a00-aaaa-bbbb-cccc-0123456789ab --model 'glm-4.5-flash' --settings /h/.t27os/hooks.json") == 1; + assert has_bytes(cmd[0..c], "glm-5.2") == false; + // a fresh spawn of the same agent computes the same default: the environment's model variable + t_restored(st[0..512], 1); + plan_pane_keys(st[0..512], 1, 0, 1000100); + const f : usize = tier_model_step(PR_ZAI, TI_DEFAULT, pane_mstep_get(st[0..512], 1), o[0..64]); + assert f == 13; + assert has_bytes(o[0..f], "glm-4.5-flash"); + const e : usize = profile_var_text_step(PR_ZAI, 2, pane_mstep_get(st[0..512], 1), o[0..64]); + assert e == 13 and has_bytes(o[0..e], "glm-4.5-flash"); + // its first start passes no model of its own on the command line + assert launch_model(st[0..512], 1, 0, true, "", o[0..64]) == 0; + // one a SPAWN names on purpose stays on that first start + const x : usize = launch_model(st[0..512], 1, 0, true, "glm-4.7-flash", o[0..64]); + assert x == 13 and has_bytes(o[0..x], "glm-4.7-flash"); + // but a restart of that pane is a restore: the profile's model again + const r : usize = launch_model(st[0..512], 1, 0, false, "glm-4.7-flash", o[0..64]); + assert r == 13 and has_bytes(o[0..r], "glm-4.5-flash"); + // another agent has no profile: its saved model stays + const g : usize = launch_model(st[0..512], 2, 1, false, "gpt-5.1", o[0..64]); + assert g == 7 and has_bytes(o[0..g], "gpt-5.1"); +} + +test a_pane_that_fails_over_to_another_profile_does_not_carry_the_model_of_the_one_it_left { + var st : [512]u64 = undefined; + var o : [64]u8 = undefined; + var why : [128]u8 = undefined; + var wl : [2]u64 = undefined; + t_table(st[0..512]); + t_key(st[0..512], P_ZAI, 0); + t_key(st[0..512], P_ANTHROPIC, 0); + t_restored(st[0..512], 0); + plan_pane_keys(st[0..512], 0, 0, 1000000); + assert t_prof(st[0..512], 0) == PR_ZAI; + // the only z.ai key is empty: the second model of the tier first, then the next profile + assert plan_failover(st[0..512], 0, FC_NO_BALANCE, 1001000, why[0..128], wl[0..2]) == FA_RESPAWN_MODEL; + plan_pane_keys(st[0..512], 0, 0, 1001500); + assert plan_failover(st[0..512], 0, FC_NO_BALANCE, 1002000, why[0..128], wl[0..2]) == FA_RESPAWN_NEXT; + plan_pane_keys(st[0..512], 0, 0, 1002500); + assert t_prof(st[0..512], 0) == PR_ANTHROPIC; + assert pane_mstep_get(st[0..512], 0) == 0; + // the pane's slot still says glm-4.7-flash; the Anthropic API does not serve it, so no model is passed + assert launch_model(st[0..512], 0, 0, false, "glm-4.7-flash", o[0..64]) == 0; + // a model of Anthropic's own that a person chose is kept + const n : usize = launch_model(st[0..512], 0, 0, false, "claude-opus-5-5", o[0..64]); + assert n == 15 and has_bytes(o[0..n], "claude-opus-5-5"); +} + +test a_restored_pane_fails_over_on_an_empty_or_rejected_key_like_a_spawned_one { + var st : [512]u64 = undefined; + var why : [128]u8 = undefined; + var wl : [2]u64 = undefined; + t_zai3(st[0..512]); + t_restored(st[0..512], 0); + plan_pane_keys(st[0..512], 0, 0, 1000000); + assert pane_key_get(st[0..512], 0, P_ZAI) == 1; + // a restored claude pane's screen is read once its grace has passed, like any other pane's + assert scan_applies(0); + assert scan_ready(st[0..512], 0, 1000000, 1000000 + SCAN_GRACE_MS - 1) == false; + assert scan_ready(st[0..512], 0, 1000000, 1000000 + SCAN_GRACE_MS); + // an empty account: the key rests ten minutes and the pane respawns on the next key, the failed one skipped + var a : u32 = plan_failover(st[0..512], 0, FC_NO_BALANCE, 1020000, why[0..128], wl[0..2]); + assert a == FA_RESPAWN_KEY; + assert st[G_KREST + 32] == 1020 + 600; + assert t_prof(st[0..512], 0) == PR_ZAI; + assert st[G_PANEK + PK_DONE] == 1; + assert has_bytes(why[0 .. @as(usize, @intCast(wl[0]))], "claude: zai out of balance, same profile with the next key"); + plan_pane_keys(st[0..512], 0, 0, 1021000); + assert pane_key_get(st[0..512], 0, P_ZAI) == 2; + // a rejected key rests an hour and the pane moves on to the third key + a = plan_failover(st[0..512], 0, FC_LOGIN_REJECTED, 1030000, why[0..128], wl[0..2]); + assert a == FA_RESPAWN_KEY; + assert st[G_KREST + 33] == 1030 + 3600; + plan_pane_keys(st[0..512], 0, 0, 1031000); + assert pane_key_get(st[0..512], 0, P_ZAI) == 3; + // neither resting key is picked by the next pane that starts + t_restored(st[0..512], 1); + plan_pane_keys(st[0..512], 1, 0, 1032000); + assert pane_key_get(st[0..512], 1, P_ZAI) == 3; +} + +test the_retry_row_of_an_empty_account_is_a_failure_at_once_so_a_stuck_pane_fails_over { + var t : [300]u8 = undefined; + // what the owner saw: HTTP 429 with code 1113, Claude Code's retry row, attempt 9 of 10 + var n : usize = put_bytes(t[0..300], 0, "API Error: 429 {\"error\":{\"code\":\"1113\",\"message\":\"Insufficient balance or no resource package. Please recharge.\"}} - Retrying in 4 seconds (attempt 9/10)"); + t[n] = 10; + assert text_class(t[0 .. n + 1]) == FC_NO_BALANCE; + // a plain throttle in flight still waits for its final row + n = put_bytes(t[0..300], 0, "API Error: 429 rate limit reached - Retrying in 4 seconds (attempt 3/10)"); + t[n] = 10; + assert text_class(t[0 .. n + 1]) == FC_NONE; + // a diff that shows the row is no failure + n = put_bytes(t[0..300], 0, "+ API Error: 429 1113 Insufficient balance - Retrying in 4 seconds (attempt 9/10)"); + t[n] = 10; + assert text_class(t[0 .. n + 1]) == FC_NONE; +} + +test with_one_key_an_empty_pane_tries_the_next_model_once_then_gives_up_and_says_so_once { + var st : [512]u64 = undefined; + var why : [128]u8 = undefined; + var wl : [2]u64 = undefined; + var o : [64]u8 = undefined; + t_table(st[0..512]); + t_key(st[0..512], P_ZAI, 0); + t_restored(st[0..512], 0); + plan_pane_keys(st[0..512], 0, 0, 1000000); + assert pane_key_get(st[0..512], 0, P_ZAI) == 1; + // the only key is empty on the default model: nothing to swap to, so the second model of the tier + var a : u32 = plan_failover(st[0..512], 0, FC_NO_BALANCE, 1020000, why[0..128], wl[0..2]); + assert a == FA_RESPAWN_MODEL; + assert pane_mstep_get(st[0..512], 0) == 1; + plan_pane_keys(st[0..512], 0, 0, 1021000); + assert pane_key_get(st[0..512], 0, P_ZAI) == 1; + const n : usize = launch_model(st[0..512], 0, 0, false, "glm-4.5-flash", o[0..64]); + assert n == 13 and has_bytes(o[0..n], "glm-4.7-flash"); + // empty on the second model too: no key, no model and no profile left, the pane stays + a = plan_failover(st[0..512], 0, FC_NO_BALANCE, 1040000, why[0..128], wl[0..2]); + assert a == FA_GIVE_UP; + assert has_bytes(why[0 .. @as(usize, @intCast(wl[0]))], "claude: zai out of balance, no profile left in the chain, the pane stays"); + assert st[G_PANEK + PK_DONE] == 1; + // the give-up is said once: the screen is not read again until the key's rest is over + assert st[G_PANEK + PK_STOP] == 1040000 + 600000; + assert scan_ready(st[0..512], 0, 1021000, 1100000) == false; + assert scan_ready(st[0..512], 0, 1021000, 1639999) == false; + assert scan_ready(st[0..512], 0, 1021000, 1640000); + assert giveup_quiet_ms(FC_NO_BALANCE) == 600000; + assert giveup_quiet_ms(FC_LOGIN_REJECTED) == 3600000; + assert giveup_quiet_ms(FC_RATE_LIMITED) == GIVEUP_QUIET_MS; + assert giveup_quiet_ms(FC_SUBSCRIPTION) == GIVEUP_QUIET_MS; +} + +test every_key_resting_for_an_empty_balance_moves_the_pane_on_and_no_dead_key_is_picked_twice_per_model { + var st : [512]u64 = undefined; + var why : [128]u8 = undefined; + var wl : [2]u64 = undefined; + t_zai3(st[0..512]); + t_restored(st[0..512], 0); + var t : u64 = 1000000; + plan_pane_keys(st[0..512], 0, 0, t); + var seen : [2]u32 = [1, 0]; + var respawns : u32 = 0; + var act : u32 = FA_KEEP; + var guard : u32 = 0; + var gave_up : bool = false; + while (guard < 30 and gave_up == false) { + t = t + 1000; + act = plan_failover(st[0..512], 0, FC_NO_BALANCE, t, why[0..128], wl[0..2]); + if (act == FA_GIVE_UP) { + gave_up = true; + } else { + respawns = respawns + 1; + t = t + 500; + plan_pane_keys(st[0..512], 0, 0, t); + if (t_prof(st[0..512], 0) == PR_ZAI) { + const kn : u32 = pane_key_get(st[0..512], 0, P_ZAI) - 1; + const sp : usize = @as(usize, pane_mstep_get(st[0..512], 0)); + // each z.ai key is tried once on each model of the tier, never twice on the same one + assert (seen[sp] >> kn) % 2 == 0; + seen[sp] = seen[sp] + (1 << kn); + } + } + guard = guard + 1; + } + // three keys on glm-4.5-flash, then the same three on glm-4.7-flash, then the Anthropic key, then done + assert gave_up; + assert seen[0] == 7; + assert seen[1] == 7; + assert respawns == 6; + assert st[G_PANEK + PK_DONE] == 6; + assert t_prof(st[0..512], 0) == PR_ANTHROPIC; + assert pane_key_get(st[0..512], 0, P_ANTHROPIC) == 1; + assert st[G_PANEK + PK_BAD] == 16; + assert st[G_PANEK + PK_STOP] == t + 600000; +} + test a_list_entry_carries_the_key_number_in_the_high_nibble { const w : u32 = pane_step(0, P_SPAWNED); assert list_flags_key(true, w, 0) == list_flags(true, w); diff --git a/specs/t27os/pty_server.t27 b/specs/t27os/pty_server.t27 index 7df1bd7012..ffb9f5a575 100644 --- a/specs/t27os/pty_server.t27 +++ b/specs/t27os/pty_server.t27 @@ -185,6 +185,8 @@ use t27os::pty_logic::list_flags_key; use t27os::pty_logic::key_put; use t27os::pty_logic::plan_pane_keys; use t27os::pty_logic::plan_failover; +use t27os::pty_logic::launch_model; +use t27os::pty_logic::pane_mstep_get; use t27os::pty_logic::screen_text; use t27os::pty_logic::text_class; use t27os::pty_logic::carry_cut; @@ -218,7 +220,7 @@ use t27os::keys::PROFILE_PROVIDER; use t27os::keys::profile_var_count; use t27os::keys::profile_var_name; use t27os::keys::profile_var_is_key; -use t27os::keys::profile_var_text; +use t27os::keys::profile_var_text_step; use t27os::pty_client::sockaddr_un; use t27os::pty_client::client_connect; use t27os::pty_client::client_list; @@ -464,7 +466,7 @@ fn inject_profile(st: []const u64, mem: []u8, i: usize) -> void { if (profile_var_is_key(pr, k)) { sys_setenv(mem[O_TMP2 .. O_TMP2 + nl + 1], mem[at + 1 .. at + 2 + @as(usize, mem[at])], 1); } else { - const vl : usize = profile_var_text(pr, k, mem[O_TMP2 + 64 .. O_TMP2 + 200]); + const vl : usize = profile_var_text_step(pr, k, pane_mstep_get(st, i), mem[O_TMP2 + 64 .. O_TMP2 + 200]); mem[O_TMP2 + 64 + vl] = 0; sys_setenv(mem[O_TMP2 .. O_TMP2 + nl + 1], mem[O_TMP2 + 64 .. O_TMP2 + 64 + vl + 1], 1); } @@ -495,8 +497,11 @@ fn inject_keys(st: []const u64, mem: []u8, i: usize, agent: u32) -> void { } } -/// Start the process of pane i (fields already in st/mem) under plan R_RESUME or R_RELAUNCH. -fn launch(st: []u64, mem: []u8, i: usize, plan: u32) -> bool { +/// Start the process of pane i (fields already in st/mem) under plan R_RESUME or R_RELAUNCH. `spawned` is true +/// only for the first start of a SPAWN request, whose payload may name a model on purpose; a pane restored from +/// its record, started again by a failover or relaunched after a resume that died is not "spawned" and runs the +/// model its profile serves (pty_logic launch_model), never the one its record held. +fn launch(st: []u64, mem: []u8, i: usize, plan: u32, spawned: bool) -> bool { const base : usize = i * PF; const agent : u32 = @as(u32, @intCast(st[base + PF_AGENT])); if (agent == 0 and plan == R_RELAUNCH) { @@ -507,6 +512,10 @@ fn launch(st: []u64, mem: []u8, i: usize, plan: u32) -> bool { // the keys of this start: the profile and the key number of each provider the agent reads, by rotation // over the numbered keys (a failover has set what to skip); the unfinished screen row of the old process goes plan_pane_keys(st, i, agent, now_ms(mem)); + // the model of this start, from the profile just chosen; the pane's slot (and so its record) takes it + const mn : usize = launch_model(st, i, agent, spawned, model_of(mem, i), mem[O_TMP2 + 2048 .. O_TMP2 + 2112]); + mem[O_MODEL + i * MODEL_CAP] = 0; + if (mn > 0) { set_model(mem, i, O_TMP2 + 2048, mn); } mem[O_CARRY + i * CARRY_SLOT] = 0; const home : []const u8 = home_slice(st, mem); const cn : usize = launch_command(agent, plan, sid_of(st, mem, i), model_of(mem, i), home, cwd_of(st, mem, i), mem[O_TMP1 + 4096 .. O_TMP1 + 8000]); @@ -569,7 +578,7 @@ fn reap(st: []u64, mem: []u8, i: usize) -> void { st[base + PF_LIVE] = 0; if (resume_failed(@as(u32, @intCast(st[base + PF_PLAN])), st[base + PF_BORN], now_ms(mem), ws[0])) { // nothing to resume: start the agent fresh in the same cwd - launch(st, mem, i, R_RELAUNCH); + launch(st, mem, i, R_RELAUNCH, false); save_pane(st, mem, i); return; } @@ -605,7 +614,7 @@ fn kill_pane(st: []u64, mem: []u8, i: usize) -> void { fn respawn_pane(st: []u64, mem: []u8, i: usize, plan: u32) -> void { const total : u64 = st[i * PF + PF_TOTAL]; stop_pane_process(st, i); - launch(st, mem, i, plan); + launch(st, mem, i, plan, false); st[i * PF + PF_TOTAL] = total; save_pane(st, mem, i); } @@ -792,7 +801,7 @@ fn handle_frame(st: []u64, mem: []u8, fd: i32, off: usize) -> void { // Keychain for the providers the agent reads, unless that was done in the last minute (keys_reload_due) const spawn_agent : u32 = @as(u32, mem[p0]); if (keys_reload_due(st, spawn_agent, now_ms(mem), st[G_KLOAD])) { load_keys(st, mem, load_mask_agent(spawn_agent)); } - launch(st, mem, @as(usize, slot), R_RELAUNCH); + launch(st, mem, @as(usize, slot), R_RELAUNCH, true); save_pane(st, mem, @as(usize, slot)); reply(fd, mem, RE_OK, @as(u8, @intCast(slot)), 0); return; @@ -940,7 +949,7 @@ fn wait_tick(st: []u64, mem: []u8) -> void { if (waiting(w)) { if (restore_release(st, ag, now, st[G_START])) { st[wk] = after_wait(st, ag); - launch(st, mem, i, wait_plan(w)); + launch(st, mem, i, wait_plan(w), false); save_pane(st, mem, i); } } else if (st[base + PF_LIVE] != 0 and keyless_respawn(w, st, ag)) { @@ -1020,7 +1029,7 @@ fn restore_all(st: []u64, mem: []u8) -> u32 { st[G_PANEK + @as(usize, i) * PK + PK_WAIT] = wait_word(plan); back = back + 1; } else { - if (launch(st, mem, @as(usize, i), plan)) { back = back + 1; } + if (launch(st, mem, @as(usize, i), plan, false)) { back = back + 1; } save_pane(st, mem, @as(usize, i)); } }