diff --git a/.trinity/seals/ml_thread_model.json b/.trinity/seals/ml_thread_model.json new file mode 100644 index 0000000000..86289e6181 --- /dev/null +++ b/.trinity/seals/ml_thread_model.json @@ -0,0 +1,28 @@ +{ + "built_by": "t27c-bootstrap@0.5.2+353cf1df0", + "closure": [], + "config": "gen=c,rust,verilog,zig;tests=zig", + "gen_hash_c": "sha256:f8a801fb1cab50b79d71e48f082d9a20631cc186a95ff556c0c95e28d3f673ff", + "gen_hash_rust": "sha256:0259f9a227a8fa9e4d44d206ce067466968359dda7faf078b6dc2eff2c98ecb9", + "gen_hash_verilog": "sha256:0c848643da8632ad81793d4fc7c75b7ca0bf90cbc1e188816579329c89f53389", + "gen_hash_zig": "sha256:a0ea2b547770300e1d014bebb7155f36285b2897d81053c71e8b112037851590", + "module": "thread_model", + "ring": 12, + "seal_schema": 2, + "sealed_at": "2026-10-19T05:50:29Z", + "sealed_by": "t27c-bootstrap@0.5.2", + "spec_hash": "sha256:4496b1af159ece6df6faa55f966b8a25f87ef7e20fc896607bbf9507ec91704e", + "spec_path": "specs/ml/thread_model.t27", + "tests": { + "failed": 0, + "failing": [], + "forced": false, + "passed": 6, + "total": 6, + "vacuous": 0 + }, + "toolchain": { + "test_runner": "sha256:8512b80da533b79289110fca4a65f3336abd7a059ca4b72ea595d5e33ff8ad10", + "zig": "0.16.0" + } +} \ No newline at end of file diff --git a/.trinity/seals/ml_train_hslm.json b/.trinity/seals/ml_train_hslm.json index 0cd2ef83ca..066dc275c6 100644 --- a/.trinity/seals/ml_train_hslm.json +++ b/.trinity/seals/ml_train_hslm.json @@ -1,5 +1,5 @@ { - "built_by": "t27c-bootstrap@0.5.2+589e8e088", + "built_by": "t27c-bootstrap@0.5.2+353cf1df0", "closure": [ { "spec": "specs/base/types.t27", @@ -19,23 +19,23 @@ } ], "config": "gen=c,rust,verilog,zig;tests=zig", - "gen_hash_c": "sha256:d1aab3569e7f81a5c9dde33f807f336fdf635a444123851f3af2337054e78adc", - "gen_hash_rust": "sha256:bc122390cb0e4fbb5874cb9a3b673821dac1ec2688218810ceeed920723680e0", - "gen_hash_verilog": "sha256:62ccb084432145a73480df7b3341a8e850ef4f3f09d48cbb063d392d9f6dd285", - "gen_hash_zig": "sha256:1173b9ccf2990e4d0bd8478d24c85f5fd40c7d5c0c410535a403f3dc0ccd9518", + "gen_hash_c": "sha256:822f833d6c18b7e4934d3b2d642f2c148fe1f474a913a8d3ae8b5d66a8574e3e", + "gen_hash_rust": "sha256:e51f59949e0422e8c502718227650dc98a439ef672bf30f57cb81845fca2eed9", + "gen_hash_verilog": "sha256:9e721a0fd0edb9a087beff8610b322cf6c08058be2eae63646179ebac438d0b7", + "gen_hash_zig": "sha256:932f799ed156fede9141759f9f8d0f06c23700006b5362756b8e67d4a6d51b5e", "module": "train_hslm", "ring": 12, "seal_schema": 2, - "sealed_at": "2026-10-19T05:47:55Z", + "sealed_at": "2026-10-19T05:50:26Z", "sealed_by": "t27c-bootstrap@0.5.2", - "spec_hash": "sha256:b4c888de5e9f260644eb98b0374fa55820be75a5a01b7ec988097b1cc9c65ddc", + "spec_hash": "sha256:84db90108f99fb68a750e82e6cd104f669900631495eafa9ae50574f956e23c4", "spec_path": "specs/ml/train_hslm.t27", "tests": { "failed": 0, "failing": [], "forced": false, - "passed": 29, - "total": 29, + "passed": 30, + "total": 30, "vacuous": 0 }, "toolchain": { diff --git a/bootstrap/src/compiler.rs b/bootstrap/src/compiler.rs index e29f1967c3..0c80f341c0 100644 --- a/bootstrap/src/compiler.rs +++ b/bootstrap/src/compiler.rs @@ -139,12 +139,6 @@ impl Node { children: Vec::new(), } } - - #[allow(dead_code)] - pub fn with_child(mut self, child: Node) -> Self { - self.children.push(child); - self - } } // ============================================================================ @@ -1332,11 +1326,6 @@ impl Parser { self.current.kind == kind } - #[allow(dead_code)] - fn check_peek(&self, kind: TokenKind) -> bool { - self.peek.kind == kind - } - fn expect(&mut self, kind: TokenKind) -> Result<(), String> { if !self.check(kind) { return Err(format!( @@ -2038,6 +2027,15 @@ the parser used to read it as `{}` followed by a negation", continue; } + // #10876: `threadlocal var` -- one copy of the variable per thread (Zig `threadlocal var`). + if self.current.kind == TokenKind::Ident && self.current.lexeme == "threadlocal" && self.peek.kind == TokenKind::KwVar { + self.advance(); + let mut decl = self.parse_var_decl(false)?; + decl.extra_kind = "threadlocal".to_string(); + module.children.push(decl); + continue; + } + // W458: allow bare expression / assignment statements at module // level so functions with array parameters can be called from the // module body (e.g. `read_mem(rom_array, idx);`). These are emitted @@ -7352,6 +7350,26 @@ fn __t27_host_write(fd: i32, b: []const u8) i32 { return __t27_host_ret(__t27_po fn __t27_host_close(fd: i32) i32 { return __t27_host_ret(__t27_posix.system.close(fd)); } "#; +/// #10876: OS threads in 256 numbered slots. thread_spawn(slot, f, arg) runs f(arg) on a new thread +/// and returns false when the slot is busy or the OS refuses; thread_join(slot) waits for it. +const THREAD_ZIG: &str = r#" +const __t27_Thread = @import("std").Thread; +var __t27_threads: [256]?__t27_Thread = [_]?__t27_Thread{null} ** 256; +fn __t27_thread_spawn(slot: usize, comptime f: anytype, arg: usize) bool { + if (slot >= __t27_threads.len or __t27_threads[slot] != null) return false; + __t27_threads[slot] = __t27_Thread.spawn(.{}, f, .{arg}) catch return false; + return true; +} +fn __t27_thread_join(slot: usize) bool { + if (slot >= __t27_threads.len) return false; + const t = __t27_threads[slot] orelse return false; + t.join(); + __t27_threads[slot] = null; + return true; +} +fn __t27_thread_cores() usize { return __t27_Thread.getCpuCount() catch 1; } +"#; + pub struct Codegen { output: String, indent: u32, @@ -8243,6 +8261,11 @@ impl Codegen { if host { self.write(HOST_IO_ZIG); } + let mut threads = false; + Self::visit_calls(&ast.children, &mut |c| threads |= matches!(c.name.as_str(), "thread_spawn" | "thread_join" | "thread_cores")); + if threads { + self.write(THREAD_ZIG); + } } /// Generate Zig code with project-aware import resolution. @@ -8418,6 +8441,9 @@ impl Codegen { // module-level name emitted `const` and Zig refused the assignment. // gen-verilog got this right all along -- it emits a `reg`. let kw = if node.extra_mutable { "var" } else { "const" }; + if node.extra_kind == "threadlocal" { + self.write("threadlocal "); + } self.write(&format!("{} {}", kw, node.name)); if !node.extra_type.is_empty() { @@ -10423,6 +10449,9 @@ impl Codegen { "host_read" => Some("__t27_host_read"), "host_write" => Some("__t27_host_write"), "host_close" => Some("__t27_host_close"), + "thread_spawn" => Some("__t27_thread_spawn"), + "thread_join" => Some("__t27_thread_join"), + "thread_cores" => Some("__t27_thread_cores"), _ => None, }; if let Some(b) = builtin { @@ -23595,15 +23624,6 @@ impl Compiler { Self::ast_is_icarus_lowerable(&ast, &structs, &functions, false) } - /// W534: reason string version of the structural classifier. - pub fn icarus_lowerability_reason(source: &str) -> Result { - match Self::is_icarus_lowerable(source) { - Ok(true) => Ok(String::new()), - Ok(false) => Err("not lowerable".to_string()), - Err(e) => Err(e), - } - } - fn collect_function_names( node: &Node, functions: &mut std::collections::HashSet, @@ -30993,18 +31013,6 @@ impl AsmConfig { } } -pub fn align_address(addr: u32, alignment: u32) -> u32 { - if alignment == 0 { - return addr; - } - let remainder = addr % alignment; - if remainder == 0 { - addr - } else { - addr + alignment - remainder - } -} - #[derive(Debug, Clone)] pub struct HirAssembler { pub config: AsmConfig, @@ -31195,10 +31203,6 @@ impl TbResetCfg { duration_cycles: duration, } } - - pub fn reset_end_cycle(&self) -> u32 { - self.delay_cycles.saturating_add(self.duration_cycles) - } } #[derive(Debug, Clone)] @@ -32325,17 +32329,6 @@ impl TimingModel { } } -pub fn path_delay(arcs: &[TimingArc]) -> u32 { - arcs.iter().map(|a| a.delay_ps).sum() -} - -pub fn worst_path_delay(paths: &[TimingPath]) -> u32 { - if paths.is_empty() { - return 0; - } - paths.iter().map(|p| p.total_delay_ps).max().unwrap_or(0) -} - #[derive(Debug, Clone)] pub struct PowerDomain { pub name: String, diff --git a/bootstrap/stage0/FROZEN_HASH b/bootstrap/stage0/FROZEN_HASH index bf541a096a..3ec2801889 100644 --- a/bootstrap/stage0/FROZEN_HASH +++ b/bootstrap/stage0/FROZEN_HASH @@ -1 +1 @@ -88b8cb73ceda740a7dd5f29d895dcce4da3a3df9a9b2fc8571b43a023af7a263 bootstrap/src/compiler.rs +509db03c8f4d1f846c0a15e07208b0564886c65e2c0eeb4b5c013e4533fe1f6c bootstrap/src/compiler.rs diff --git a/specs/ml/thread_model.t27 b/specs/ml/thread_model.t27 new file mode 100644 index 0000000000..a9457ab6c2 --- /dev/null +++ b/specs/ml/thread_model.t27 @@ -0,0 +1,199 @@ +// SPDX-License-Identifier: Apache-2.0 +; specs/ml/thread_model.t27 -- OS threads and per-thread variables under `t27c --backend ml`. +; Issue #10876 (lever (a) of specs/t27t/t27t.t27: the windows of a training step on N cores). + +; WHY: a training step of specs/ml/train_hslm.t27 is 66 independent windows; their gradients are +; summed at the end. Running them on N cores needs two things t27 did not have: a way to start a +; thread and wait for it, and variables of which every thread has its own copy (the forward and +; backward of nn/hslm.t27 keep activations, KV caches and gradients in module-level variables). +; This spec is the contract of both, and its tests are the proof that t27c lowers them right. + +; THE FORM (parsed by t27c, emitted for --backend ml): +; threadlocal var name : T = init; one copy per thread. A new thread's copy starts at init, +; NOT at the value the spawning thread holds. + +; HOST CALLS (lowered by t27c to Zig std.Thread; nothing here is implemented in this file): +; thread_spawn(slot, f, arg) starts f(arg) on a new thread in slot 0..255; f is a function +; fn(usize) -> void named directly. Returns false when the slot is +; taken (spawned and not joined) or the OS refuses the thread. +; thread_join(slot) waits for the slot's thread and frees the slot; false when the +; slot holds no thread. +; thread_cores() the CPU count the OS reports, 1 when it reports nothing. + +; Not claimed: no mutex, no atomics, no thread pool. Threads may share data only by writing +; disjoint elements; a result that has to be the same bits for any thread count is combined by +; the spawning thread, in a fixed order, after every join (the last test shows the method). +; phi^2 + 1/phi^2 = 3 | TRINITY + +module thread_model; + +pub const MAX_WORKERS : usize = 16; +pub const PARTS : usize = 64; +pub const PART_LEN : usize = 1000; + +; Shared: every worker writes its own elements only. +var tm_out : [MAX_WORKERS]u64 = undefined; +var tm_seen_init : [MAX_WORKERS]u64 = undefined; +var tm_part : [PARTS]f64 = undefined; +var tm_nworkers : usize = 1; + +; One copy per thread. +threadlocal var tm_own : u64 = 5; +threadlocal var tm_acc : [PART_LEN]f64 = undefined; + +/// Worker for the copy tests: records the copy it started with, then makes the copy its own and +/// reads it back after a long loop that would be disturbed by any other thread writing it +pub fn tm_own_worker(id: usize) -> void { + tm_seen_init[id] = tm_own; + tm_own = @as(u64, id) * 1000003; + var spin : u64 = 0; + var i : u64 = 0; + while (i < 200000) { + spin = spin + (tm_own ^ i); + i = i + 1; + } + tm_out[id] = tm_own + (spin & 0); +} + +/// Start n workers in slots 0..n-1 and join them all; the number that started +pub fn tm_run(n: usize) -> usize { + var started : usize = 0; + var s : usize = 0; + while (s < n) { + if (thread_spawn(s, tm_own_worker, s)) { + started = started + 1; + } + s = s + 1; + } + s = 0; + while (s < n) { + if (!thread_join(s)) { + started = started + 1000; + } + s = s + 1; + } + return started; +} + +/// The float value of part p: a sum in a thread-local scratch, so two threads computing two parts +/// at once cannot see each other's partial sums +pub fn tm_part_value(p: usize) -> f64 { + var i : usize = 0; + while (i < PART_LEN) { + tm_acc[i] = 1.0 / @as(f64, @floatFromInt(p * PART_LEN + i + 1)); + i = i + 1; + } + var s : f64 = 0.0; + i = 0; + while (i < PART_LEN) { + s = s + tm_acc[i]; + i = i + 1; + } + return s; +} + +/// Worker id of tm_nworkers takes parts id, id + n, id + 2n, ... +pub fn tm_part_worker(id: usize) -> void { + var p : usize = id; + while (p < PARTS) { + tm_part[p] = tm_part_value(p); + p = p + tm_nworkers; + } +} + +/// The parts on n threads (n = 1 runs on the caller), then their sum in part order 0..PARTS-1 +pub fn tm_parts_sum(n: usize) -> f64 { + tm_nworkers = n; + if (n == 1) { + tm_part_worker(0); + } else { + var spawned : [MAX_WORKERS]bool = undefined; + var s : usize = 0; + while (s < n) { + spawned[s] = thread_spawn(s, tm_part_worker, s); + if (!spawned[s]) { + tm_part_worker(s); + } + s = s + 1; + } + s = 0; + while (s < n) { + if (spawned[s]) { + assert thread_join(s); + } + s = s + 1; + } + } + var sum : f64 = 0.0; + var p : usize = 0; + while (p < PARTS) { + sum = sum + tm_part[p]; + p = p + 1; + } + return sum; +} + +/// The same 64000 terms summed in one running sum, the order a serial loop would use without parts +pub fn tm_running_sum() -> f64 { + var sum : f64 = 0.0; + var k : usize = 0; + while (k < PARTS * PART_LEN) { + sum = sum + 1.0 / @as(f64, @floatFromInt(k + 1)); + k = k + 1; + } + return sum; +} + +pub fn main() -> void { + assert tm_run(8) == 8; + assert @as(u64, @bitCast(tm_parts_sum(8))) == @as(u64, @bitCast(tm_parts_sum(1))); +} + +test every_thread_gets_its_own_copy { + assert tm_run(8) == 8; + var id : usize = 0; + while (id < 8) { + assert tm_out[id] == @as(u64, id) * 1000003; + id = id + 1; + } +} + +test a_new_thread_starts_at_the_initializer_not_at_the_spawners_value { + tm_own = 99; + assert tm_run(4) == 4; + var id : usize = 0; + while (id < 4) { + assert tm_seen_init[id] == 5; + id = id + 1; + } + assert tm_own == 99; +} + +test a_taken_slot_is_refused_and_a_free_slot_cannot_be_joined { + assert thread_spawn(3, tm_own_worker, 3); + assert !thread_spawn(3, tm_own_worker, 3); + assert thread_join(3); + assert !thread_join(3); + assert !thread_spawn(256, tm_own_worker, 0); +} + +test the_fixed_order_sum_has_the_same_bits_on_1_2_3_8_and_16_threads { + const one : u64 = @bitCast(tm_parts_sum(1)); + assert @as(u64, @bitCast(tm_parts_sum(2))) == one; + assert @as(u64, @bitCast(tm_parts_sum(3))) == one; + assert @as(u64, @bitCast(tm_parts_sum(8))) == one; + assert @as(u64, @bitCast(tm_parts_sum(16))) == one; +} + +test the_part_order_is_not_the_running_order { + // the fixed part order is a different association of the same terms: the bits differ, so a + // serial path that has to match the threaded one must sum in parts too + const parts : u64 = @bitCast(tm_parts_sum(1)); + const running : u64 = @bitCast(tm_running_sum()); + assert parts != running; + assert abs(tm_parts_sum(1) - tm_running_sum()) < 1.0e-12; +} + +test the_os_reports_at_least_one_core { + assert thread_cores() >= 1; +} diff --git a/specs/ml/train_hslm.t27 b/specs/ml/train_hslm.t27 index 8b52ec9f22..e15749eef8 100644 --- a/specs/ml/train_hslm.t27 +++ b/specs/ml/train_hslm.t27 @@ -90,69 +90,71 @@ const HEADER_BYTES : usize = 16; ; THE STATE OF nn/hslm.t27. use_resolve splices fns, structs, enums and consts but not module-level ; `var`; the spliced forward and backward bind to these, which repeat nn/hslm.t27 section 2 (names, ; types, initial values) and nothing else. A variable added there and missing here stops the build -; with "use of undeclared identifier"; it cannot drift silently. +; with "use of undeclared identifier"; it cannot drift silently. What one window writes (caches, +; activations, gradients, scratch, gate counters) is `threadlocal`: each thread of a step has its +; own copy (section 3b). The weights, nudge and vsa_on are read by every thread and written by none. var model : HSLMWeights = undefined; -var kv_k : [NUM_BLOCKS * CONTEXT_LEN * EMBED_DIM]f64 = undefined; -var kv_v : [NUM_BLOCKS * CONTEXT_LEN * EMBED_DIM]f64 = undefined; -var act_token : usize = 0; -var act_pos : [NUM_BLOCKS]usize = undefined; -var act_x : [NUM_BLOCKS * EMBED_DIM]f64 = undefined; -var act_n1 : [NUM_BLOCKS * EMBED_DIM]f64 = undefined; -var act_rms1 : [NUM_BLOCKS]f64 = undefined; -var act_q : [NUM_BLOCKS * EMBED_DIM]f64 = undefined; -var act_attn : [NUM_BLOCKS * NUM_HEADS * CONTEXT_LEN]f64 = undefined; -var act_ctx : [NUM_BLOCKS * EMBED_DIM]f64 = undefined; -var act_r : [NUM_BLOCKS * EMBED_DIM]f64 = undefined; -var act_pre : [NUM_BLOCKS * FF_DIM]f64 = undefined; -var act_h : [NUM_BLOCKS * FF_DIM]f64 = undefined; -var act_out : [NUM_BLOCKS * EMBED_DIM]f64 = undefined; -var act_hn : [EMBED_DIM]f64 = undefined; -var act_hrms : f64 = 1.0; -var gw_q : [NUM_BLOCKS * EMBED_DIM * EMBED_DIM]f64 = undefined; -var gw_k : [NUM_BLOCKS * EMBED_DIM * EMBED_DIM]f64 = undefined; -var gw_v : [NUM_BLOCKS * EMBED_DIM * EMBED_DIM]f64 = undefined; -var gw_o : [NUM_BLOCKS * EMBED_DIM * EMBED_DIM]f64 = undefined; -var gw_up : [NUM_BLOCKS * EMBED_DIM * FF_DIM]f64 = undefined; -var gw_down : [NUM_BLOCKS * FF_DIM * EMBED_DIM]f64 = undefined; -var gw_out : [EMBED_DIM * VOCAB_SIZE]f64 = undefined; -var g_gamma1 : [NUM_BLOCKS * EMBED_DIM]f64 = undefined; -var g_b_up : [NUM_BLOCKS * FF_DIM]f64 = undefined; -var g_b_down : [NUM_BLOCKS * EMBED_DIM]f64 = undefined; -var g_b_out : [VOCAB_SIZE]f64 = undefined; -var g_embed : [VOCAB_SIZE * EMBED_DIM]f64 = undefined; -var bw_out : [EMBED_DIM]f64 = undefined; -var bw_h : [FF_DIM]f64 = undefined; -var bw_pre : [FF_DIM]f64 = undefined; -var bw_sf : [FF_DIM]f64 = undefined; -var bw_se : [EMBED_DIM]f64 = undefined; -var bw_n : [EMBED_DIM]f64 = undefined; -var bw_tmp : [EMBED_DIM]f64 = undefined; -var bw_r : [EMBED_DIM]f64 = undefined; -var bw_ctx : [EMBED_DIM]f64 = undefined; -var bw_q : [EMBED_DIM]f64 = undefined; -var bw_k : [EMBED_DIM]f64 = undefined; -var bw_v : [EMBED_DIM]f64 = undefined; -var bw_da : [CONTEXT_LEN]f64 = undefined; -var bw_x : [EMBED_DIM]f64 = undefined; -var bw_dz : [VOCAB_SIZE]f64 = undefined; +threadlocal var kv_k : [NUM_BLOCKS * CONTEXT_LEN * EMBED_DIM]f64 = undefined; +threadlocal var kv_v : [NUM_BLOCKS * CONTEXT_LEN * EMBED_DIM]f64 = undefined; +threadlocal var act_token : usize = 0; +threadlocal var act_pos : [NUM_BLOCKS]usize = undefined; +threadlocal var act_x : [NUM_BLOCKS * EMBED_DIM]f64 = undefined; +threadlocal var act_n1 : [NUM_BLOCKS * EMBED_DIM]f64 = undefined; +threadlocal var act_rms1 : [NUM_BLOCKS]f64 = undefined; +threadlocal var act_q : [NUM_BLOCKS * EMBED_DIM]f64 = undefined; +threadlocal var act_attn : [NUM_BLOCKS * NUM_HEADS * CONTEXT_LEN]f64 = undefined; +threadlocal var act_ctx : [NUM_BLOCKS * EMBED_DIM]f64 = undefined; +threadlocal var act_r : [NUM_BLOCKS * EMBED_DIM]f64 = undefined; +threadlocal var act_pre : [NUM_BLOCKS * FF_DIM]f64 = undefined; +threadlocal var act_h : [NUM_BLOCKS * FF_DIM]f64 = undefined; +threadlocal var act_out : [NUM_BLOCKS * EMBED_DIM]f64 = undefined; +threadlocal var act_hn : [EMBED_DIM]f64 = undefined; +threadlocal var act_hrms : f64 = 1.0; +threadlocal var gw_q : [NUM_BLOCKS * EMBED_DIM * EMBED_DIM]f64 = undefined; +threadlocal var gw_k : [NUM_BLOCKS * EMBED_DIM * EMBED_DIM]f64 = undefined; +threadlocal var gw_v : [NUM_BLOCKS * EMBED_DIM * EMBED_DIM]f64 = undefined; +threadlocal var gw_o : [NUM_BLOCKS * EMBED_DIM * EMBED_DIM]f64 = undefined; +threadlocal var gw_up : [NUM_BLOCKS * EMBED_DIM * FF_DIM]f64 = undefined; +threadlocal var gw_down : [NUM_BLOCKS * FF_DIM * EMBED_DIM]f64 = undefined; +threadlocal var gw_out : [EMBED_DIM * VOCAB_SIZE]f64 = undefined; +threadlocal var g_gamma1 : [NUM_BLOCKS * EMBED_DIM]f64 = undefined; +threadlocal var g_b_up : [NUM_BLOCKS * FF_DIM]f64 = undefined; +threadlocal var g_b_down : [NUM_BLOCKS * EMBED_DIM]f64 = undefined; +threadlocal var g_b_out : [VOCAB_SIZE]f64 = undefined; +threadlocal var g_embed : [VOCAB_SIZE * EMBED_DIM]f64 = undefined; +threadlocal var bw_out : [EMBED_DIM]f64 = undefined; +threadlocal var bw_h : [FF_DIM]f64 = undefined; +threadlocal var bw_pre : [FF_DIM]f64 = undefined; +threadlocal var bw_sf : [FF_DIM]f64 = undefined; +threadlocal var bw_se : [EMBED_DIM]f64 = undefined; +threadlocal var bw_n : [EMBED_DIM]f64 = undefined; +threadlocal var bw_tmp : [EMBED_DIM]f64 = undefined; +threadlocal var bw_r : [EMBED_DIM]f64 = undefined; +threadlocal var bw_ctx : [EMBED_DIM]f64 = undefined; +threadlocal var bw_q : [EMBED_DIM]f64 = undefined; +threadlocal var bw_k : [EMBED_DIM]f64 = undefined; +threadlocal var bw_v : [EMBED_DIM]f64 = undefined; +threadlocal var bw_da : [CONTEXT_LEN]f64 = undefined; +threadlocal var bw_x : [EMBED_DIM]f64 = undefined; +threadlocal var bw_dz : [VOCAB_SIZE]f64 = undefined; var nudge_which : u8 = MAT_NONE; var nudge_block : usize = 0; var nudge_index : usize = 0; var nudge_delta : f64 = 0.0; var vsa_on : bool = true; -var vsa_in : [NUM_BLOCKS * CONTEXT_LEN * VSA_DIM]Trit = undefined; -var vsa_out : [NUM_BLOCKS * VSA_DIM]Trit = undefined; -var vsa_ctx : [VSA_DIM]Trit = undefined; -var vsa_reasoned : [VSA_DIM]Trit = undefined; -var vsa_temp : [VSA_DIM]Trit = undefined; -var vsa_temp2 : [VSA_DIM]Trit = undefined; -var vsa_sim : [CONTEXT_LEN]f64 = undefined; -var vsa_accum : [VSA_DIM]i64 = undefined; -var vsa_max_sim : [NUM_BLOCKS]f64 = undefined; -var vsa_fired : [NUM_BLOCKS]bool = undefined; -var gate_total : [NUM_BLOCKS]usize = undefined; -var gate_conscious : [NUM_BLOCKS]usize = undefined; -var gate_ema : [NUM_BLOCKS]f64 = undefined; +threadlocal var vsa_in : [NUM_BLOCKS * CONTEXT_LEN * VSA_DIM]Trit = undefined; +threadlocal var vsa_out : [NUM_BLOCKS * VSA_DIM]Trit = undefined; +threadlocal var vsa_ctx : [VSA_DIM]Trit = undefined; +threadlocal var vsa_reasoned : [VSA_DIM]Trit = undefined; +threadlocal var vsa_temp : [VSA_DIM]Trit = undefined; +threadlocal var vsa_temp2 : [VSA_DIM]Trit = undefined; +threadlocal var vsa_sim : [CONTEXT_LEN]f64 = undefined; +threadlocal var vsa_accum : [VSA_DIM]i64 = undefined; +threadlocal var vsa_max_sim : [NUM_BLOCKS]f64 = undefined; +threadlocal var vsa_fired : [NUM_BLOCKS]bool = undefined; +threadlocal var gate_total : [NUM_BLOCKS]usize = undefined; +threadlocal var gate_conscious : [NUM_BLOCKS]usize = undefined; +threadlocal var gate_ema : [NUM_BLOCKS]f64 = undefined; ; THE TRAINING CONSTANTS (the issue's numbers). Steps, schedule length, warmup, intervals, seed, ; speed level and resume step can be overridden in /tmp/hslm/train.cfg; these are the defaults. @@ -241,12 +243,12 @@ var ntrain : usize = 0; var nvalid : usize = 0; var data_synthetic : bool = false; var fixed_starts : [8]usize = undefined; -var win_logits : [VOCAB_SIZE]f64 = undefined; -var win_dl : [VOCAB_SIZE]f64 = undefined; -var win_gin : [EMBED_DIM]f64 = undefined; +threadlocal var win_logits : [VOCAB_SIZE]f64 = undefined; +threadlocal var win_dl : [VOCAB_SIZE]f64 = undefined; +threadlocal var win_gin : [EMBED_DIM]f64 = undefined; var ref_logits : [VOCAB_SIZE]f64 = undefined; var ref_gin : [EMBED_DIM]f64 = undefined; -var kv_n1 : [EMBED_DIM]f64 = undefined; +threadlocal var kv_n1 : [EMBED_DIM]f64 = undefined; var io_chunk : [IO_CHUNK]u8 = undefined; var io_back : [IO_CHUNK]u8 = undefined; @@ -277,6 +279,7 @@ var cfg_eval_windows : u64 = DEFAULT_EVAL_WINDOWS; var cfg_loss_positions : u64 = DEFAULT_LOSS_POSITIONS; var cfg_gradcheck : u64 = 0; var cfg_profile : u64 = 0; +var cfg_threads : u64 = 0; ; THE PROFILE (profile=1 or 2 in train.cfg; gHashTag/t27#10850). Where a training step's time goes, ; read off the monotonic clock (1) or the thread's CPU clock (2) with interval attribution: every @@ -309,8 +312,8 @@ var prof_on : bool = false; var prof_ns : [PR_PARTS]u64 = undefined; var prof_calls : [PR_PARTS]u64 = undefined; var prof_last : u64 = 0; -var prof_fcall : usize = 0; -var prof_bcall : usize = 0; +threadlocal var prof_fcall : usize = 0; +threadlocal var prof_bcall : usize = 0; var prof_sink : u64 = 0; var prof_cpu : bool = false; @@ -738,37 +741,38 @@ const TR3_LAB_S2_TRAIN_S_AGAIN : f64 = 55.62; const TR3_LAB_S3_TRAIN_S_AGAIN : f64 = 37.27; const TR3_LAB_SPEEDUP : f64 = 1.478; const TR3_TILE : usize = 81; -var b3_tok : [CONTEXT_LEN]usize = undefined; -var b3_x : [NUM_BLOCKS * CONTEXT_LEN * EMBED_DIM]f64 = undefined; -var b3_n1 : [NUM_BLOCKS * CONTEXT_LEN * EMBED_DIM]f64 = undefined; -var b3_rms1 : [NUM_BLOCKS * CONTEXT_LEN]f64 = undefined; -var b3_q : [NUM_BLOCKS * CONTEXT_LEN * EMBED_DIM]f64 = undefined; -var b3_attn : [NUM_BLOCKS * CONTEXT_LEN * NUM_HEADS * CONTEXT_LEN]f64 = undefined; -var b3_ctx : [NUM_BLOCKS * CONTEXT_LEN * EMBED_DIM]f64 = undefined; -var b3_r : [NUM_BLOCKS * CONTEXT_LEN * EMBED_DIM]f64 = undefined; -var b3_pre : [NUM_BLOCKS * CONTEXT_LEN * FF_DIM]f64 = undefined; -var b3_h : [NUM_BLOCKS * CONTEXT_LEN * FF_DIM]f64 = undefined; -var b3_out : [NUM_BLOCKS * CONTEXT_LEN * EMBED_DIM]f64 = undefined; -var b3_hn : [CONTEXT_LEN * EMBED_DIM]f64 = undefined; -var b3_hrms : [CONTEXT_LEN]f64 = undefined; -var b3_logits : [CONTEXT_LEN * VOCAB_SIZE]f64 = undefined; -var b3_y : [CONTEXT_LEN * EMBED_DIM]f64 = undefined; -var b3_wt : [VOCAB_SIZE]f64 = undefined; -var b3_dl : [CONTEXT_LEN * VOCAB_SIZE]f64 = undefined; -var b3_dz : [CONTEXT_LEN * VOCAB_SIZE]f64 = undefined; -var b3_bo : [CONTEXT_LEN * EMBED_DIM]f64 = undefined; -var b3_br : [CONTEXT_LEN * EMBED_DIM]f64 = undefined; -var b3_bctx : [CONTEXT_LEN * EMBED_DIM]f64 = undefined; -var b3_bq : [CONTEXT_LEN * EMBED_DIM]f64 = undefined; -var b3_bk : [CONTEXT_LEN * EMBED_DIM]f64 = undefined; -var b3_bv : [CONTEXT_LEN * EMBED_DIM]f64 = undefined; -var b3_bn : [CONTEXT_LEN * EMBED_DIM]f64 = undefined; -var b3_tmp : [CONTEXT_LEN * EMBED_DIM]f64 = undefined; -var b3_se : [CONTEXT_LEN * EMBED_DIM]f64 = undefined; -var b3_bh : [CONTEXT_LEN * FF_DIM]f64 = undefined; -var b3_bpre : [CONTEXT_LEN * FF_DIM]f64 = undefined; -var b3_sf : [CONTEXT_LEN * FF_DIM]f64 = undefined; -var b3_da : [CONTEXT_LEN]f64 = undefined; +; threadlocal (#10876): the windows of one step run on several threads, each with its own scratch +threadlocal var b3_tok : [CONTEXT_LEN]usize = undefined; +threadlocal var b3_x : [NUM_BLOCKS * CONTEXT_LEN * EMBED_DIM]f64 = undefined; +threadlocal var b3_n1 : [NUM_BLOCKS * CONTEXT_LEN * EMBED_DIM]f64 = undefined; +threadlocal var b3_rms1 : [NUM_BLOCKS * CONTEXT_LEN]f64 = undefined; +threadlocal var b3_q : [NUM_BLOCKS * CONTEXT_LEN * EMBED_DIM]f64 = undefined; +threadlocal var b3_attn : [NUM_BLOCKS * CONTEXT_LEN * NUM_HEADS * CONTEXT_LEN]f64 = undefined; +threadlocal var b3_ctx : [NUM_BLOCKS * CONTEXT_LEN * EMBED_DIM]f64 = undefined; +threadlocal var b3_r : [NUM_BLOCKS * CONTEXT_LEN * EMBED_DIM]f64 = undefined; +threadlocal var b3_pre : [NUM_BLOCKS * CONTEXT_LEN * FF_DIM]f64 = undefined; +threadlocal var b3_h : [NUM_BLOCKS * CONTEXT_LEN * FF_DIM]f64 = undefined; +threadlocal var b3_out : [NUM_BLOCKS * CONTEXT_LEN * EMBED_DIM]f64 = undefined; +threadlocal var b3_hn : [CONTEXT_LEN * EMBED_DIM]f64 = undefined; +threadlocal var b3_hrms : [CONTEXT_LEN]f64 = undefined; +threadlocal var b3_logits : [CONTEXT_LEN * VOCAB_SIZE]f64 = undefined; +threadlocal var b3_y : [CONTEXT_LEN * EMBED_DIM]f64 = undefined; +threadlocal var b3_wt : [VOCAB_SIZE]f64 = undefined; +threadlocal var b3_dl : [CONTEXT_LEN * VOCAB_SIZE]f64 = undefined; +threadlocal var b3_dz : [CONTEXT_LEN * VOCAB_SIZE]f64 = undefined; +threadlocal var b3_bo : [CONTEXT_LEN * EMBED_DIM]f64 = undefined; +threadlocal var b3_br : [CONTEXT_LEN * EMBED_DIM]f64 = undefined; +threadlocal var b3_bctx : [CONTEXT_LEN * EMBED_DIM]f64 = undefined; +threadlocal var b3_bq : [CONTEXT_LEN * EMBED_DIM]f64 = undefined; +threadlocal var b3_bk : [CONTEXT_LEN * EMBED_DIM]f64 = undefined; +threadlocal var b3_bv : [CONTEXT_LEN * EMBED_DIM]f64 = undefined; +threadlocal var b3_bn : [CONTEXT_LEN * EMBED_DIM]f64 = undefined; +threadlocal var b3_tmp : [CONTEXT_LEN * EMBED_DIM]f64 = undefined; +threadlocal var b3_se : [CONTEXT_LEN * EMBED_DIM]f64 = undefined; +threadlocal var b3_bh : [CONTEXT_LEN * FF_DIM]f64 = undefined; +threadlocal var b3_bpre : [CONTEXT_LEN * FF_DIM]f64 = undefined; +threadlocal var b3_sf : [CONTEXT_LEN * FF_DIM]f64 = undefined; +threadlocal var b3_da : [CONTEXT_LEN]f64 = undefined; /// y[p] = x[p] W for rows p_lo..p_hi (x rows in_dim wide, y rows out_dim wide). Each weight row is /// read once for all rows; a column tile of TR3_TILE keeps that tile of every row in L1. Per output @@ -1705,33 +1709,177 @@ fn tr_train_step(t: u64, batch: usize, lr_fixed: f64, fixed: bool) -> f64 { return last_loss; } -/// tr_train_step under the k-position objective: the same windows, averaging, clip, LAMB and -/// requantize; each window's loss and gradient come from tr_window_grad_k -fn tr_train_step_k(t: u64, batch: usize, lr_fixed: f64, fixed: bool, k: usize) -> f64 { - if (prof_on) { - prof_mark(PR_STEP_OTHER); +; 3b. THE WINDOWS ON N THREADS (lever (a) of specs/t27t/t27t.t27, gHashTag/t27#10876). The windows +; of a step are independent; only their gradients meet. The starts are drawn first, on the calling +; thread, in the order a serial loop draws them. Worker id then takes windows id, id + n, ...: each +; from zero gradients in the worker's own `threadlocal` copy, packed in segment order into the +; window's own slot of win_g. After the join, g_val[e] = (((0 + slot 0) + slot 1) + ...) * (1 / +; batch), the windows in order 0, 1, ..., on n threads by element range; the loss is the windows' +; losses added in the same order. The sum of an element is the same chain of additions for every +; n, so 1 thread and n threads end in the same bits (a test runs 1, 2 and 3). Change stated: before +; this, one running sum took every position of every window, so at k > 1 the serial bits are not +; those of the older code (the mean is; the last digits are a different association). At k = 1 +; each window adds one term per element, the two chains are one, and the old step still matches +; to the bit (loss_positions_one_is_the_last_position_step_to_the_bit). threads=N in train.cfg; 0, +; the default, is the cores the OS reports; at most the batch; 1 while the profile is on, whose +; marks are one thread's clock. A thread the OS refuses runs its windows on the calling thread. +const MAX_THREADS : usize = 66; +var win_start : [BATCH]usize = undefined; +var win_loss : [BATCH]f64 = undefined; +; zero, not undefined: a Debug build writes undefined as 0xaa into .data, 1.1 GB for this array +var win_g : [BATCH * NPARAM]f64 = [0.0] ** (BATCH * NPARAM); +var par_spawned : [MAX_THREADS]bool = undefined; +var par_n : usize = 1; +var par_batch : usize = 0; +var par_k : usize = 1; +var par_inv : f64 = 1.0; +var par_phase : u8 = 0; +var par_started : u64 = 0; + +/// The thread count of a step of `batch` windows +fn tr_threads(batch: usize) -> usize { + var n : usize = @intCast(cfg_threads); + if (n == 0) { + n = thread_cores(); + } + if (n > batch) { + n = batch; + } + if (n > MAX_THREADS) { + n = MAX_THREADS; + } + if (n < 1 or prof_on) { + n = 1; } + return n; +} + +/// This thread's gradient into window w's slot, segment by segment, zeroing it for the next window +fn tr_pack(w: usize) -> void { + const base : usize = w * NPARAM; + var s : usize = 0; + while (s < NSEG) { + const src = tr_grad_of(s); + const off : usize = base + seg_off[s]; + var i : usize = 0; + while (i < src.len) { + win_g[off + i] = src[i]; + src[i] = 0.0; + i = i + 1; + } + s = s + 1; + } +} + +/// Windows id, id + par_n, ...: loss into win_loss, gradient into win_g +fn tr_window_worker(id: usize) -> void { zero_weight_gradients(); if (prof_on) { prof_mark(PR_ZERO); } - var loss_sum : f64 = 0.0; + var w : usize = id; + while (w < par_batch) { + win_loss[w] = tr_window_grad_k(train_tok[0..ntrain], win_start[w], par_k); + if (prof_on) { + prof_mark(PR_STEP_OTHER); + } + tr_pack(w); + if (prof_on) { + prof_mark(PR_GATHER_CLIP); + } + w = w + par_n; + } +} + +/// Elements [id * NPARAM / par_n, (id + 1) * NPARAM / par_n) of g_val: the slots added in window +/// order from zero, then scaled by par_inv +fn tr_reduce_worker(id: usize) -> void { + const lo : usize = id * NPARAM / par_n; + const hi : usize = (id + 1) * NPARAM / par_n; + var e : usize = lo; + while (e < hi) { + g_val[e] = 0.0; + e = e + 1; + } + var w : usize = 0; + while (w < par_batch) { + const base : usize = w * NPARAM; + e = lo; + while (e < hi) { + g_val[e] = g_val[e] + win_g[base + e]; + e = e + 1; + } + w = w + 1; + } + e = lo; + while (e < hi) { + g_val[e] = g_val[e] * par_inv; + e = e + 1; + } +} + +fn tr_par_worker(id: usize) -> void { + if (par_phase == 0) { + tr_window_worker(id); + } else { + tr_reduce_worker(id); + } +} + +/// Phase `phase` (0 windows, 1 reduce) as par_n workers: 1..par_n-1 on threads, 0 here; every +/// thread joined before it returns +fn tr_par_run(phase: u8) -> void { + par_phase = phase; + var s : usize = 1; + while (s < par_n) { + par_spawned[s] = thread_spawn(s, tr_par_worker, s); + if (!par_spawned[s]) { + tr_par_worker(s); + } + s = s + 1; + } + tr_par_worker(0); + s = 1; + while (s < par_n) { + if (par_spawned[s]) { + if (thread_join(s)) { + par_started = par_started + 1; + } else { + io_fail = io_fail + 1; + } + } + s = s + 1; + } +} + +/// tr_train_step under the k-position objective, its windows on tr_threads(batch) threads (3b): +/// the same windows, averaging, clip, LAMB and requantize +fn tr_train_step_k(t: u64, batch: usize, lr_fixed: f64, fixed: bool, k: usize) -> f64 { + if (prof_on) { + prof_mark(PR_STEP_OTHER); + } var wi : usize = 0; while (wi < batch) { - var start : usize = 0; if (fixed) { - start = fixed_starts[wi]; + win_start[wi] = fixed_starts[wi]; } else { - start = tr_next_start(ntrain); + win_start[wi] = tr_next_start(ntrain); } - loss_sum = loss_sum + tr_window_grad_k(train_tok[0..ntrain], start, k); wi = wi + 1; } - if (prof_on) { - prof_mark(PR_STEP_OTHER); - } + par_n = tr_threads(batch); + par_batch = batch; + par_k = k; const inv : f64 = 1.0 / @as(f64, @floatFromInt(batch)); - tr_gather(inv); + par_inv = inv; + tr_par_run(0); + var loss_sum : f64 = 0.0; + wi = 0; + while (wi < batch) { + loss_sum = loss_sum + win_loss[wi]; + wi = wi + 1; + } + tr_par_run(1); last_gnorm = tr_clip_global(g_val[0..NPARAM], CLIP_NORM); if (prof_on) { prof_mark(PR_GATHER_CLIP); @@ -2036,6 +2184,8 @@ fn tr_cfg_set(off: usize, len: usize, value: u64) -> void { cfg_gradcheck = value; } else if (tr_key_is(off, len, "profile")) { cfg_profile = value; + } else if (tr_key_is(off, len, "threads")) { + cfg_threads = value; } } @@ -2820,6 +2970,32 @@ fn tr_prof_digest_on() -> u64 { return tr_prof_step_digest(true); } +/// One fixed k = 27 step of three windows of the tiny slice on n threads at a speed level: a digest of every +/// parameter bit and the loss's bits afterwards +fn tr_thread_step_digest(n: u64, speed: u8) -> u64 { + speed_level = speed; + tr_tiny_setup(11); + fixed_starts[0] = 300; + fixed_starts[1] = 1700; + fixed_starts[2] = 3100; + cfg_threads = n; + const seen : f64 = tr_train_step_k(0, 3, 0.01, true, CONTEXT_LEN); + cfg_threads = 0; + speed_level = 2; + var h : u64 = @as(u64, @bitCast(seen)) % 4294967291; + var i : usize = 0; + while (i < NPARAM) { + h = (h * 31 + (@as(u64, @bitCast(p_val[i])) % 4294967291)) % 4294967291; + i = i + 1; + } + return h; +} + +/// The threads the steps so far started and joined +fn tr_threads_started() -> u64 { + return par_started; +} + /// After tr_prof_digest_on: the marks charged to `part` (2 windows x 27 positions) fn tr_prof_marks(part: usize) -> u64 { return prof_calls[part]; @@ -3803,3 +3979,18 @@ test the_profile_moves_no_bit_and_sees_the_call_order and af == 162 and ab == 162 and la == 1 + +test one_two_and_three_threads_give_one_step_the_same_bits_at_speed_2_and_3 + given one = tr_thread_step_digest(1, 2) + and before = tr_threads_started() + and two = tr_thread_step_digest(2, 2) + and three = tr_thread_step_digest(3, 2) + and after = tr_threads_started() + and one3 = tr_thread_step_digest(1, 3) + and three3 = tr_thread_step_digest(3, 3) + then one == two + and one == three + and one != 0 + and after - before == 6 + and one3 == three3 + and one3 == one